Compare commits
No files matched your search
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Bash(git status:*)", "Bash(git diff:*)", "Bash(git log:*)", "Bash(git show:*)",
|
||||
"Bash(git rev-parse:*)", "Bash(git add:*)", "Bash(git commit:*)",
|
||||
"Bash(go mod download:*)", "Bash(go build:*)", "Bash(go test:*)", "Bash(go vet:*)",
|
||||
"Bash(gofmt -l:*)",
|
||||
"Bash(golangci-lint run:*)"
|
||||
],
|
||||
"ask": [
|
||||
"Bash(git push:*)", "Bash(git merge:*)", "Bash(git rebase:*)", "Bash(git switch:*)",
|
||||
"Bash(git restore:*)", "Bash(git commit --amend:*)", "Bash(rm:*)",
|
||||
"Bash(go get:*)", "Bash(go mod tidy:*)", "Bash(go install:*)",
|
||||
"Bash(make:*)", "Bash(docker:*)"
|
||||
],
|
||||
"deny": [
|
||||
"Bash(git push --force:*)", "Bash(git push -f:*)", "Bash(git push --force-with-lease:*)",
|
||||
"Bash(git reset --hard:*)", "Bash(git clean:*)", "Bash(git add -f:*)", "Bash(git add --force:*)",
|
||||
"Bash(git commit --no-verify:*)", "Bash(git commit -n:*)",
|
||||
"Bash(rm -r:*)", "Bash(rm -R:*)", "Bash(rm -rf:*)", "Bash(rm -fr:*)", "Bash(rm -Rf:*)",
|
||||
"Bash(rm -fR:*)", "Bash(rm --recursive:*)", "Bash(sudo:*)",
|
||||
"Read(./.env)", "Read(./.env.*)", "Read(./secrets/**)"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
sandbox_mode = "workspace-write"
|
||||
approval_policy = "on-request"
|
||||
@@ -0,0 +1,20 @@
|
||||
# Rendered from docs/PERMISSIONS.md - change both in the same commit.
|
||||
# Deny
|
||||
prefix_rule(pattern = ["git", "push", ["--force", "-f", "--force-with-lease"]], decision = "forbidden",
|
||||
justification = "Force-push rewrites shared history.")
|
||||
prefix_rule(pattern = ["git", "reset", "--hard"], decision = "forbidden", justification = "Discards work.")
|
||||
prefix_rule(pattern = ["git", "clean"], decision = "forbidden", justification = "Deletes untracked files.")
|
||||
prefix_rule(pattern = ["git", "add", ["-f", "--force"]], decision = "forbidden",
|
||||
justification = "Stages ignored files, e.g. secrets.")
|
||||
prefix_rule(pattern = ["git", "commit", ["--no-verify", "-n"]], decision = "forbidden", justification = "Skips hooks.")
|
||||
prefix_rule(pattern = ["rm", ["-r", "-R", "-rf", "-fr", "-Rf", "-fR", "--recursive"]], decision = "forbidden",
|
||||
justification = "Recursive delete.")
|
||||
prefix_rule(pattern = ["sudo"], decision = "forbidden", justification = "No privilege escalation.")
|
||||
# Ask
|
||||
prefix_rule(pattern = ["git", ["push", "merge", "rebase", "switch", "restore"]], decision = "prompt")
|
||||
prefix_rule(pattern = ["git", "commit", "--amend"], decision = "prompt")
|
||||
prefix_rule(pattern = ["rm"], decision = "prompt")
|
||||
prefix_rule(pattern = ["go", ["get", "install"]], decision = "prompt")
|
||||
prefix_rule(pattern = ["go", "mod", "tidy"], decision = "prompt")
|
||||
prefix_rule(pattern = ["make"], decision = "prompt")
|
||||
prefix_rule(pattern = ["docker"], decision = "prompt")
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"tools": {
|
||||
"allowed": [
|
||||
"run_shell_command(git status)", "run_shell_command(git diff)", "run_shell_command(git log)",
|
||||
"run_shell_command(git show)", "run_shell_command(git branch --list)",
|
||||
"run_shell_command(git add)", "run_shell_command(git commit)",
|
||||
"run_shell_command(go mod download)", "run_shell_command(go build)",
|
||||
"run_shell_command(go test)", "run_shell_command(go vet)", "run_shell_command(gofmt -l)",
|
||||
"run_shell_command(golangci-lint run)"
|
||||
],
|
||||
"exclude": [
|
||||
"run_shell_command(git push --force)", "run_shell_command(git push -f)",
|
||||
"run_shell_command(git push --force-with-lease)", "run_shell_command(git reset --hard)",
|
||||
"run_shell_command(git clean)", "run_shell_command(git add -f)", "run_shell_command(git add --force)",
|
||||
"run_shell_command(git commit --no-verify)", "run_shell_command(git commit -n)",
|
||||
"run_shell_command(rm -r)", "run_shell_command(rm -R)", "run_shell_command(rm -rf)",
|
||||
"run_shell_command(rm -fr)", "run_shell_command(rm -Rf)", "run_shell_command(sudo)"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
# Fork CI for the Proton Drive backend. Every step is hermetic: the
|
||||
# tests run against the fake server in backend/protondrive/protontest.
|
||||
name: ci
|
||||
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
|
||||
env:
|
||||
# Never pick up a config holding a real Proton remote.
|
||||
RCLONE_CONFIG: /notfound
|
||||
|
||||
jobs:
|
||||
protondrive:
|
||||
runs-on: ubuntu-latest
|
||||
# The race suite takes about 6 minutes on a fast machine and 7 on
|
||||
# two cores; the slow runner may take several times that.
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
# The line the code was verified with, not go.mod's minimum,
|
||||
# so govulncheck sees a patched standard library.
|
||||
go-version: '1.27.x'
|
||||
# Caches the module and build caches, keyed on go.sum.
|
||||
cache: true
|
||||
|
||||
- name: Build
|
||||
run: go build ./...
|
||||
|
||||
- name: Vet
|
||||
run: go vet ./backend/protondrive/...
|
||||
|
||||
- name: Lint
|
||||
uses: golangci/golangci-lint-action@v8
|
||||
with:
|
||||
version: v2.15.0
|
||||
args: ./backend/protondrive/...
|
||||
|
||||
- name: Test
|
||||
# Go's default 10 minute limit per test binary is too short
|
||||
# for the race suite on a slow runner.
|
||||
run: go test -race -timeout 40m ./backend/protondrive/...
|
||||
|
||||
- name: Fuzz
|
||||
run: |
|
||||
for target in $(go test -list '^Fuzz' ./backend/protondrive/ | grep '^Fuzz'); do
|
||||
go test -run '^$' -fuzz "^${target}\$" -fuzztime 20s ./backend/protondrive/
|
||||
done
|
||||
|
||||
- name: Vulnerability check
|
||||
run: go run golang.org/x/vuln/cmd/govulncheck@v1.9.0 ./backend/protondrive/...
|
||||
@@ -19,5 +19,12 @@ Thumbs.db
|
||||
__pycache__
|
||||
.DS_Store
|
||||
resource_windows_*.syso
|
||||
/librclone.h
|
||||
.devcontainer
|
||||
/cmd/bisync/testdata/bisync_vscode_debuggers_launch.json
|
||||
.env
|
||||
.env.*
|
||||
CLAUDE.local.md
|
||||
.claude/settings.local.json
|
||||
.worktrees/
|
||||
.superpowers/
|
||||
@@ -134,3 +134,16 @@ Uses golangci-lint v2 with config in `.golangci.yml`. Enabled linters: errcheck,
|
||||
- Command docs are in the command source code (e.g., `cmd/ls/ls.go`).
|
||||
- Don't commit autogenerated doc changes from `make backenddocs` or `make commanddocs`.
|
||||
- Website docs are in `docs/content/` as markdown, built with Hugo (`make serve` to preview).
|
||||
|
||||
## Fork Workflow
|
||||
|
||||
This fork carries agent tooling that upstream does not. Never include `docs/specs/`, `docs/plans/`, `docs/{ARCHITECTURE,DECISIONS,TODO,PERMISSIONS}.md`, `GEMINI.md`, `.claude/`, `.codex/`, `.gemini/` or this section in a PR to rclone/rclone.
|
||||
|
||||
- Required skill: `superpowers` - check at session start, install if missing. There is no product UI here, so `impeccable` is not needed.
|
||||
- Specs go in `docs/specs/NNN-slug.md` (permanent). Plans go in `docs/plans/NNN-slug.md` and are removed with `git rm` before merge. This overrides the superpowers default paths.
|
||||
- Paused or in-flight work is listed under "Now" in `docs/TODO.md` with the handoff to read first.
|
||||
- Non-trivial work: brainstorm → spec → plan → TDD → review. For trivial fixes say "trivial, skipping spec" and proceed.
|
||||
- Log things noticed but deliberately not fixed in `docs/TODO.md`, record decisions in `docs/DECISIONS.md`, and keep `docs/ARCHITECTURE.md` in step with structural changes - in the same commit as the code.
|
||||
- `docs/PERMISSIONS.md` is the source for the `.claude/`, `.codex/` and `.gemini/` permission files - change them together.
|
||||
- Never push, force-push or commit directly to `master`. Ask before adding or upgrading a dependency (`go get`, `go mod tidy`).
|
||||
- Done means run in this session and seen passing: `go build`, `go test` on the touched packages, and `golangci-lint run` on them.
|
||||
@@ -0,0 +1,8 @@
|
||||
# GEMINI.md
|
||||
|
||||
**`AGENTS.md` is the single source of truth for this repository.** Read
|
||||
`./AGENTS.md` before doing anything else and follow it exactly.
|
||||
|
||||
This file exists only so Gemini CLI finds the rules. It contains no project
|
||||
rules of its own - add those to `AGENTS.md`. If anything here ever conflicts
|
||||
with `AGENTS.md`, `AGENTS.md` wins.
|
||||
@@ -285,7 +285,7 @@ endif
|
||||
|
||||
# Fetch the binary builds from GitHub actions
|
||||
fetch_binaries:
|
||||
rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
|
||||
rclone -P sync --exclude "/testbuilds/**" --exclude "/index.{html,json}" --delete-excluded $(BETA_UPLOAD) build/
|
||||
|
||||
serve: website
|
||||
cd docs && hugo server --logLevel info -w --disableFastRender --ignoreCache
|
||||
|
||||
@@ -24,6 +24,7 @@ directories to and from different cloud storage providers.
|
||||
## Storage providers
|
||||
|
||||
- 1Fichier [:page_facing_up:](https://rclone.org/fichier/)
|
||||
- Abrha Storage [:page_facing_up:](https://rclone.org/s3/#abrhastorage)
|
||||
- Akamai Netstorage [:page_facing_up:](https://rclone.org/netstorage/)
|
||||
- Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
|
||||
- Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
|
||||
|
||||
+7
-3
@@ -32,10 +32,14 @@ This file describes how to make the various kinds of releases
|
||||
- git commit -a -v -m "Version v1.XX.0"
|
||||
- make check # if this fails, fix, git commit --amend and run it again
|
||||
- make retag # moves the tag onto the Version commit
|
||||
- git push origin # without --follow-tags so it doesn't push the tag if it fails
|
||||
- git push origin # without --follow-tags so it doesn't push the tag yet
|
||||
- Wait for the GitHub actions build of master to complete and check it is
|
||||
green - this is the first time CI sees any commits which were only local,
|
||||
such as security fixes, so it must pass before the tag is pushed
|
||||
- if it fails: fix, git commit, make retag, git push origin and wait again
|
||||
- git push --follow-tags origin
|
||||
- \# Wait for the GitHub builds to complete then...
|
||||
- \# Don't commit anything until after make upload_github as these steps
|
||||
- \# Wait for the GitHub builds of the tag to complete then...
|
||||
- \# Don't commit or push anything until after make upload_github as these steps
|
||||
use the tag on HEAD to find the version
|
||||
- make fetch_binaries
|
||||
- make tarball
|
||||
|
||||
@@ -634,6 +634,27 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (_ fs.Objec
|
||||
"x-archive-filemeta-rclone-update-track": updateTracker,
|
||||
}
|
||||
|
||||
mdata, err := fs.GetMetadataOptions(ctx, f, src, fs.MetadataAsOpenOptions(ctx))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if mdata != nil {
|
||||
// IA ignores file metadata headers with the COPY directive.
|
||||
headers["x-amz-metadata-directive"] = "REPLACE"
|
||||
}
|
||||
for key, value := range mdata {
|
||||
key = strings.ToLower(key)
|
||||
if key == "mtime" {
|
||||
key = "rclone-mtime"
|
||||
} else if strings.HasPrefix(key, "rclone-") {
|
||||
// Keep the new copy tracker and mapped mtime, not the source internals.
|
||||
continue
|
||||
} else if _, readOnly := roMetadataKey[key]; readOnly {
|
||||
continue
|
||||
}
|
||||
headers["x-archive-filemeta-"+key] = value
|
||||
}
|
||||
|
||||
// make a PUT request at (IAS3)/:item/:path without body
|
||||
var resp *http.Response
|
||||
opts := rest.Opts{
|
||||
|
||||
@@ -597,6 +597,12 @@ func TestMkdirMetadataThroughPlantedSymlinkBlocked(t *testing.T) {
|
||||
// TestRemoveThroughPlantedSymlinkBlocked checks Remove and Rmdir can't
|
||||
// delete anything outside the root through a planted symlink.
|
||||
func TestRemoveThroughPlantedSymlinkBlocked(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
// os.Root.Symlink makes a file symlink to a directory outside
|
||||
// the root, which Windows can't traverse, so the planted link
|
||||
// can't reach the outside files.
|
||||
t.Skip("planted directory symlinks can't be traversed on Windows")
|
||||
}
|
||||
skipIfNoSymlinks(t)
|
||||
ctx := context.Background()
|
||||
|
||||
|
||||
@@ -0,0 +1,362 @@
|
||||
package protondrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"sync"
|
||||
|
||||
"github.com/ProtonMail/gopenpgp/v3/crypto"
|
||||
"github.com/rclone/rclone/backend/protondrive/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
// unlockMu is held while unlocking account keys. Unlocking is memory
|
||||
// hungry, so unlocks for different sessions run one at a time.
|
||||
var unlockMu sync.Mutex
|
||||
|
||||
// address is one of the user's addresses with its unlocked keys.
|
||||
type address struct {
|
||||
ID string
|
||||
Email string
|
||||
keyRing *crypto.KeyRing // unlocked address keys in the server's order, primary first
|
||||
primaryKeyID string // ID of the first key of keyRing
|
||||
}
|
||||
|
||||
// account is the signed in user's unlocked keys and addresses.
|
||||
type account struct {
|
||||
userKR *crypto.KeyRing // unlocked user keys
|
||||
addresses []*address // the primary address is first
|
||||
|
||||
mu sync.Mutex
|
||||
foreign map[string]*publicKeyLookup // public keys of other users by emailKey
|
||||
}
|
||||
|
||||
// publicKeyLookup is a fetch of another user's public keys, done when
|
||||
// done is closed.
|
||||
type publicKeyLookup struct {
|
||||
done chan struct{}
|
||||
keyRing *crypto.KeyRing
|
||||
err error
|
||||
}
|
||||
|
||||
// loadAccount fetches the user and addresses of the session and unlocks
|
||||
// their keys with keyPassword.
|
||||
//
|
||||
// User keys which don't unlock are skipped, but an error is returned
|
||||
// if none unlocks or if the primary address has no key which unlocks.
|
||||
// Other addresses without a key which unlocks are left out.
|
||||
func loadAccount(ctx context.Context, s *session, keyPassword []byte) (*account, error) {
|
||||
var user api.UserResponse
|
||||
if _, err := s.call(ctx, &rest.Opts{Method: "GET", Path: "/core/v4/users"}, nil, &user); err != nil {
|
||||
return nil, fmt.Errorf("load account: get user: %w", err)
|
||||
}
|
||||
var addrs api.AddressesResponse
|
||||
// The SDK fetches only the first page of 50 addresses.
|
||||
opts := &rest.Opts{Method: "GET", Path: "/core/v4/addresses", Parameters: url.Values{"Page": {"0"}, "PageSize": {"50"}}}
|
||||
if _, err := s.call(ctx, opts, nil, &addrs); err != nil {
|
||||
return nil, fmt.Errorf("load account: get addresses: %w", err)
|
||||
}
|
||||
if len(addrs.Addresses) == 0 {
|
||||
return nil, errors.New("load account: the account has no addresses")
|
||||
}
|
||||
|
||||
unlockMu.Lock()
|
||||
defer unlockMu.Unlock()
|
||||
if s.unlockHook != nil {
|
||||
s.unlockHook()
|
||||
}
|
||||
a := &account{foreign: make(map[string]*publicKeyLookup)}
|
||||
var err error
|
||||
a.userKR, err = unlockUserKeys(user.User.Keys, keyPassword)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load account: unlock user keys: %w", err)
|
||||
}
|
||||
var verifyTime int64
|
||||
if t := s.serverTime(); !t.IsZero() {
|
||||
verifyTime = t.Unix()
|
||||
}
|
||||
for i, addr := range addrs.Addresses {
|
||||
kr, primaryKeyID, err := unlockAddressKeys(addr.Keys, keyPassword, a.userKR, verifyTime)
|
||||
if err != nil {
|
||||
if i == 0 {
|
||||
a.close()
|
||||
return nil, fmt.Errorf("load account: unlock keys of primary address: %w", err)
|
||||
}
|
||||
fs.Debugf(nil, "protondrive: skipping address %s: %v", addr.ID, err)
|
||||
continue
|
||||
}
|
||||
a.addresses = append(a.addresses, &address{ID: addr.ID, Email: addr.Email, keyRing: kr, primaryKeyID: primaryKeyID})
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// unlockUserKeys returns a key ring of the active user keys which
|
||||
// unlock with keyPassword.
|
||||
func unlockUserKeys(keys []api.UserKey, keyPassword []byte) (*crypto.KeyRing, error) {
|
||||
kr, err := crypto.NewKeyRing(nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var lastErr error
|
||||
for _, k := range keys {
|
||||
if k.Active == 0 {
|
||||
continue
|
||||
}
|
||||
unlocked, err := unlockKey(k.PrivateKey, keyPassword)
|
||||
if err == nil {
|
||||
if err = kr.AddKey(unlocked); err != nil {
|
||||
unlocked.ClearPrivateParams()
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
fs.Debugf(nil, "protondrive: can't unlock user key %s: %v", k.ID, err)
|
||||
lastErr = err
|
||||
}
|
||||
}
|
||||
if kr.CountEntities() == 0 {
|
||||
if lastErr == nil {
|
||||
return nil, errors.New("no active user key")
|
||||
}
|
||||
return nil, fmt.Errorf("no user key unlocks, is the password right?: %w", lastErr)
|
||||
}
|
||||
return kr, nil
|
||||
}
|
||||
|
||||
// unlockAddressKeys returns a key ring of the active address keys
|
||||
// which unlock, in the order given, and the ID of the first of them.
|
||||
//
|
||||
// A key with a Token is locked with the passphrase in it, encrypted to
|
||||
// and signed by the user keys in userKR; the signature is checked as
|
||||
// at verifyTime (Unix seconds, or now if zero). A key without a Token
|
||||
// is locked with keyPassword.
|
||||
func unlockAddressKeys(keys []api.AddressKey, keyPassword []byte, userKR *crypto.KeyRing, verifyTime int64) (kr *crypto.KeyRing, firstID string, err error) {
|
||||
kr, err = crypto.NewKeyRing(nil)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
var lastErr error
|
||||
for _, k := range keys {
|
||||
if k.Active == 0 {
|
||||
continue
|
||||
}
|
||||
unlocked, err := unlockAddressKey(k, keyPassword, userKR, verifyTime)
|
||||
if err == nil {
|
||||
if err = kr.AddKey(unlocked); err != nil {
|
||||
unlocked.ClearPrivateParams()
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
fs.Debugf(nil, "protondrive: can't unlock address key %s: %v", k.ID, err)
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
if firstID == "" {
|
||||
firstID = k.ID
|
||||
}
|
||||
}
|
||||
if kr.CountEntities() == 0 {
|
||||
if lastErr == nil {
|
||||
return nil, "", errors.New("no active address key")
|
||||
}
|
||||
return nil, "", fmt.Errorf("no address key unlocks: %w", lastErr)
|
||||
}
|
||||
return kr, firstID, nil
|
||||
}
|
||||
|
||||
// unlockAddressKey unlocks one address key as unlockAddressKeys does.
|
||||
func unlockAddressKey(k api.AddressKey, keyPassword []byte, userKR *crypto.KeyRing, verifyTime int64) (*crypto.Key, error) {
|
||||
if k.Token == "" {
|
||||
return unlockKey(k.PrivateKey, keyPassword)
|
||||
}
|
||||
dh, err := crypto.PGP().Decryption().DecryptionKeys(userKR).New()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, err := dh.Decrypt([]byte(k.Token), crypto.Armor)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decrypt token: %w", err)
|
||||
}
|
||||
passphrase := res.Bytes()
|
||||
defer clear(passphrase)
|
||||
vb := crypto.PGP().Verify().VerificationKeys(userKR)
|
||||
if verifyTime != 0 {
|
||||
vb = vb.VerifyTime(verifyTime)
|
||||
}
|
||||
vh, err := vb.New()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// As the SDK does, refuse a token without a valid signature.
|
||||
verified, err := vh.VerifyDetached(passphrase, []byte(k.Signature), crypto.Armor)
|
||||
if err == nil {
|
||||
err = verified.SignatureError()
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("verify token: %w", err)
|
||||
}
|
||||
return unlockKey(k.PrivateKey, passphrase)
|
||||
}
|
||||
|
||||
// unlockKey returns the armored private key unlocked with passphrase.
|
||||
func unlockKey(armored string, passphrase []byte) (*crypto.Key, error) {
|
||||
locked, err := crypto.NewKeyFromArmored(armored)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return locked.Unlock(passphrase)
|
||||
}
|
||||
|
||||
// signingKeyRing returns a key ring holding only the first key of the
|
||||
// address, which the SDK calls the primary address key.
|
||||
func (a *address) signingKeyRing() (*crypto.KeyRing, error) {
|
||||
return crypto.NewKeyRing(a.keyRing.GetKeys()[0])
|
||||
}
|
||||
|
||||
// primary returns the primary address.
|
||||
func (a *account) primary() *address {
|
||||
return a.addresses[0]
|
||||
}
|
||||
|
||||
// byID returns the address with the ID id, or nil.
|
||||
func (a *account) byID(id string) *address {
|
||||
for _, addr := range a.addresses {
|
||||
if addr.ID == id {
|
||||
return addr
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// emailKey returns email with ASCII letters lowercased, which is how
|
||||
// emails are compared: Proton treats addresses case-insensitively, but
|
||||
// Unicode case folding would let a non-ASCII letter, such as the Kelvin
|
||||
// sign, stand for an ASCII one.
|
||||
func emailKey(email string) string {
|
||||
b := []byte(email)
|
||||
for i, c := range b {
|
||||
if 'A' <= c && c <= 'Z' {
|
||||
b[i] = c + 'a' - 'A'
|
||||
}
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// byEmail returns the address email, compared as emailKey does, or
|
||||
// nil.
|
||||
func (a *account) byEmail(email string) *address {
|
||||
key := emailKey(email)
|
||||
for _, addr := range a.addresses {
|
||||
if emailKey(addr.Email) == key {
|
||||
return addr
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// owns reports whether email is one of the account's addresses whose
|
||||
// keys it holds, compared as emailKey does.
|
||||
func (a *account) owns(email string) bool {
|
||||
return a.byEmail(email) != nil
|
||||
}
|
||||
|
||||
// apiCaller makes an API call with the semantics of session.call,
|
||||
// without returning the response; client.apiCall is one which paces
|
||||
// and retries.
|
||||
type apiCaller func(ctx context.Context, opts *rest.Opts, request, response any) error
|
||||
|
||||
// publicKeys returns the public keys of the address email, which the
|
||||
// caller must not modify.
|
||||
//
|
||||
// The keys of the user's own addresses, in any ASCII case, come from
|
||||
// the account, never the server. Other users' keys are fetched once
|
||||
// per emailKey and kept for the life of the account; an email without
|
||||
// a Proton address has an empty key ring.
|
||||
func (a *account) publicKeys(ctx context.Context, call apiCaller, email string) (*crypto.KeyRing, error) {
|
||||
if addr := a.byEmail(email); addr != nil {
|
||||
return publicKeyRing(addr.keyRing)
|
||||
}
|
||||
key := emailKey(email)
|
||||
a.mu.Lock()
|
||||
l := a.foreign[key]
|
||||
if l == nil {
|
||||
l = &publicKeyLookup{done: make(chan struct{})}
|
||||
a.foreign[key] = l
|
||||
a.mu.Unlock()
|
||||
l.keyRing, l.err = fetchPublicKeys(ctx, call, email)
|
||||
if l.err != nil {
|
||||
// Let the next caller try again.
|
||||
a.mu.Lock()
|
||||
delete(a.foreign, key)
|
||||
a.mu.Unlock()
|
||||
}
|
||||
close(l.done)
|
||||
} else {
|
||||
a.mu.Unlock()
|
||||
}
|
||||
select {
|
||||
case <-l.done:
|
||||
return l.keyRing, l.err
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// publicKeyRing returns the public parts of the keys in kr.
|
||||
func publicKeyRing(kr *crypto.KeyRing) (*crypto.KeyRing, error) {
|
||||
pub, err := crypto.NewKeyRing(nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, key := range kr.GetKeys() {
|
||||
pk, err := key.ToPublic()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := pub.AddKey(pk); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return pub, nil
|
||||
}
|
||||
|
||||
// fetchPublicKeys fetches the public keys of the Proton address email
|
||||
// with call.
|
||||
func fetchPublicKeys(ctx context.Context, call apiCaller, email string) (*crypto.KeyRing, error) {
|
||||
kr, err := crypto.NewKeyRing(nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out api.PublicKeysResponse
|
||||
opts := &rest.Opts{Method: "GET", Path: "/core/v4/keys/all", Parameters: url.Values{"Email": {email}, "InternalOnly": {"1"}}}
|
||||
err = call(ctx, opts, nil, &out)
|
||||
var apiErr *api.Error
|
||||
if errors.As(err, &apiErr) && (apiErr.Code == api.CodeAddressMissing || apiErr.Code == api.CodeDomainExternal) {
|
||||
return kr, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get public keys of %s: %w", email, err)
|
||||
}
|
||||
for _, k := range out.Address.Keys {
|
||||
key, err := crypto.NewKeyFromArmored(k.PublicKey)
|
||||
if err == nil {
|
||||
err = kr.AddKey(key)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("public key of %s: %w", email, err)
|
||||
}
|
||||
}
|
||||
return kr, nil
|
||||
}
|
||||
|
||||
// close clears the private key material of the account.
|
||||
func (a *account) close() {
|
||||
if a.userKR != nil {
|
||||
a.userKR.ClearPrivateParams()
|
||||
}
|
||||
for _, addr := range a.addresses {
|
||||
addr.keyRing.ClearPrivateParams()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
package protondrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ProtonMail/gopenpgp/v3/crypto"
|
||||
"github.com/rclone/rclone/backend/protondrive/protontest"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// newAccountSession returns a session for a new session of u.
|
||||
func newAccountSession(srv *protontest.Server, u *protontest.User) *session {
|
||||
tokens := srv.NewSession(u)
|
||||
store := &memoryStore{creds: credentials{
|
||||
UID: tokens.UID,
|
||||
AccessToken: tokens.AccessToken,
|
||||
RefreshToken: tokens.RefreshToken,
|
||||
KeyPassword: u.KeyPassword(),
|
||||
}}
|
||||
return newSession(srv.Client(), srv.URL(), testAppVersion, store)
|
||||
}
|
||||
|
||||
// sessionCaller returns an apiCaller making unpaced calls with s.
|
||||
func sessionCaller(s *session) apiCaller {
|
||||
return func(ctx context.Context, opts *rest.Opts, request, response any) error {
|
||||
_, err := s.call(ctx, opts, request, response)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// decryptWith decrypts the armored message with kr.
|
||||
func decryptWith(t *testing.T, kr *crypto.KeyRing, armored string) string {
|
||||
t.Helper()
|
||||
dh, err := crypto.PGP().Decryption().DecryptionKeys(kr).New()
|
||||
require.NoError(t, err)
|
||||
res, err := dh.Decrypt([]byte(armored), crypto.Armor)
|
||||
require.NoError(t, err)
|
||||
return string(res.Bytes())
|
||||
}
|
||||
|
||||
func TestLoadAccount(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
u.AddAddress("alias@proton.test")
|
||||
s := newAccountSession(srv, u)
|
||||
|
||||
a, err := loadAccount(context.Background(), s, []byte(u.KeyPassword()))
|
||||
require.NoError(t, err)
|
||||
defer a.close()
|
||||
|
||||
require.NotNil(t, a.userKR)
|
||||
require.Len(t, a.addresses, 2)
|
||||
primary := a.primary()
|
||||
require.NotNil(t, primary)
|
||||
assert.Equal(t, u.Email(), primary.Email)
|
||||
assert.NotEmpty(t, primary.ID)
|
||||
assert.Same(t, primary, a.byID(primary.ID))
|
||||
assert.Same(t, primary, a.byEmail(u.Email()))
|
||||
assert.Nil(t, a.byID("unknown"))
|
||||
assert.Nil(t, a.byEmail("unknown@proton.test"))
|
||||
|
||||
for _, email := range []string{u.Email(), "alias@proton.test"} {
|
||||
addr := a.byEmail(email)
|
||||
require.NotNil(t, addr, email)
|
||||
assert.Equal(t, "secret for "+email, decryptWith(t, addr.keyRing, u.EncryptToAddress(email, []byte("secret for "+email))))
|
||||
}
|
||||
// The address keys are distinct.
|
||||
dh, err := crypto.PGP().Decryption().DecryptionKeys(primary.keyRing).New()
|
||||
require.NoError(t, err)
|
||||
_, err = dh.Decrypt([]byte(u.EncryptToAddress("alias@proton.test", []byte("x"))), crypto.Armor)
|
||||
assert.Error(t, err, "primary address key decrypted a message for the alias")
|
||||
}
|
||||
|
||||
func TestLoadAccountTwoPassword(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddTwoPasswordUser("user", "login", "mailbox")
|
||||
want, err := computeKeyPassword([]byte("mailbox"), u.KeySalt)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, string(want), u.KeyPassword())
|
||||
|
||||
a, err := loadAccount(context.Background(), newAccountSession(srv, u), want)
|
||||
require.NoError(t, err)
|
||||
a.close()
|
||||
}
|
||||
|
||||
func TestLoadAccountWrongKeyPassword(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
s := newAccountSession(srv, u)
|
||||
|
||||
for _, keyPassword := range [][]byte{[]byte("wrong"), nil} {
|
||||
a, err := loadAccount(context.Background(), s, keyPassword)
|
||||
require.Error(t, err)
|
||||
assert.Nil(t, a)
|
||||
assert.Contains(t, err.Error(), "unlock user keys")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAccountRotatedUserKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
u.AddUserKey()
|
||||
|
||||
// The old user key no longer unlocks with the key password, which
|
||||
// must not stop the account loading.
|
||||
a, err := loadAccount(context.Background(), newAccountSession(srv, u), []byte(u.KeyPassword()))
|
||||
require.NoError(t, err)
|
||||
defer a.close()
|
||||
assert.Equal(t, "msg", decryptWith(t, a.primary().keyRing, u.EncryptToAddress(u.Email(), []byte("msg"))))
|
||||
}
|
||||
|
||||
func TestLoadAccountBadAddressKeySignature(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
empty bool
|
||||
}{
|
||||
{"impostor", false},
|
||||
{"empty", true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
|
||||
// On the primary address the account doesn't load.
|
||||
u := srv.AddUser("user", "pass")
|
||||
srv.CorruptAddressKeySignature(u, u.Email(), test.empty)
|
||||
a, err := loadAccount(context.Background(), newAccountSession(srv, u), []byte(u.KeyPassword()))
|
||||
require.Error(t, err)
|
||||
assert.Nil(t, a)
|
||||
assert.Contains(t, err.Error(), "verify token")
|
||||
|
||||
// A secondary address is left out.
|
||||
u = srv.AddUser("user2", "pass")
|
||||
u.AddAddress("alias@proton.test")
|
||||
srv.CorruptAddressKeySignature(u, "alias@proton.test", test.empty)
|
||||
a, err = loadAccount(context.Background(), newAccountSession(srv, u), []byte(u.KeyPassword()))
|
||||
require.NoError(t, err)
|
||||
defer a.close()
|
||||
assert.Nil(t, a.byEmail("alias@proton.test"))
|
||||
assert.Len(t, a.addresses, 1)
|
||||
assert.Equal(t, u.Email(), a.primary().Email)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAccountSerialised(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
s := newAccountSession(srv, u)
|
||||
|
||||
var active, peak, calls atomic.Int32
|
||||
s.unlockHook = func() {
|
||||
n := active.Add(1)
|
||||
for {
|
||||
p := peak.Load()
|
||||
if n <= p || peak.CompareAndSwap(p, n) {
|
||||
break
|
||||
}
|
||||
}
|
||||
calls.Add(1)
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
active.Add(-1)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for range 16 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
a, err := loadAccount(context.Background(), s, []byte(u.KeyPassword()))
|
||||
if assert.NoError(t, err) {
|
||||
a.close()
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
assert.Equal(t, int32(16), calls.Load())
|
||||
assert.Equal(t, int32(1), peak.Load(), "unlocks overlapped")
|
||||
}
|
||||
|
||||
func TestAccountClose(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
a, err := loadAccount(context.Background(), newAccountSession(srv, u), []byte(u.KeyPassword()))
|
||||
require.NoError(t, err)
|
||||
a.close()
|
||||
|
||||
for _, kr := range []*crypto.KeyRing{a.userKR, a.primary().keyRing} {
|
||||
for _, key := range kr.GetKeys() {
|
||||
assert.False(t, key.IsPrivate(), "private key material left after close")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicKeysCached(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
other := srv.AddUser("other", "pass")
|
||||
s := newAccountSession(srv, u)
|
||||
a, err := loadAccount(context.Background(), s, []byte(u.KeyPassword()))
|
||||
require.NoError(t, err)
|
||||
defer a.close()
|
||||
otherAccount, err := loadAccount(context.Background(), newAccountSession(srv, other), []byte(other.KeyPassword()))
|
||||
require.NoError(t, err)
|
||||
defer otherAccount.close()
|
||||
|
||||
const route = "GET /core/v4/keys/all"
|
||||
srv.ResetRequests()
|
||||
for range 5 {
|
||||
kr, err := a.publicKeys(context.Background(), sessionCaller(s), other.Email())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, kr.CountEntities())
|
||||
assert.False(t, kr.GetKeys()[0].IsPrivate())
|
||||
// A message encrypted to the key decrypts with other's address key.
|
||||
eh, err := crypto.PGP().Encryption().Recipients(kr).New()
|
||||
require.NoError(t, err)
|
||||
msg, err := eh.Encrypt([]byte("hello"))
|
||||
require.NoError(t, err)
|
||||
armored, err := msg.ArmorBytes()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "hello", decryptWith(t, otherAccount.primary().keyRing, string(armored)))
|
||||
}
|
||||
assert.Equal(t, 1, srv.Requests()[route])
|
||||
|
||||
srv.ResetRequests()
|
||||
for range 5 {
|
||||
kr, err := a.publicKeys(context.Background(), sessionCaller(s), u.Email())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, kr.CountEntities())
|
||||
assert.False(t, kr.GetKeys()[0].IsPrivate())
|
||||
}
|
||||
assert.Zero(t, srv.Requests()[route])
|
||||
|
||||
// An email without a Proton account has no keys, and that is cached too.
|
||||
for range 3 {
|
||||
kr, err := a.publicKeys(context.Background(), sessionCaller(s), "nobody@example.com")
|
||||
require.NoError(t, err)
|
||||
assert.Zero(t, kr.CountEntities())
|
||||
}
|
||||
assert.Equal(t, 1, srv.Requests()[route])
|
||||
|
||||
// Case variants share a lookup, and an own address in any case
|
||||
// never reaches the server.
|
||||
srv.ResetRequests()
|
||||
for _, email := range []string{"OTHER@proton.test", "Other@Proton.Test", strings.ToUpper(u.Email())} {
|
||||
_, err := a.publicKeys(context.Background(), sessionCaller(s), email)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
assert.Zero(t, srv.Requests()[route])
|
||||
}
|
||||
|
||||
func TestPublicKeysErrorNotCached(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
other := srv.AddUser("other", "pass")
|
||||
s := newAccountSession(srv, u)
|
||||
a, err := loadAccount(context.Background(), s, []byte(u.KeyPassword()))
|
||||
require.NoError(t, err)
|
||||
defer a.close()
|
||||
|
||||
srv.Inject(protontest.Fault{Pattern: "/core/v4/keys/all", Times: 1, Status: 500})
|
||||
_, err = a.publicKeys(context.Background(), sessionCaller(s), other.Email())
|
||||
require.Error(t, err)
|
||||
kr, err := a.publicKeys(context.Background(), sessionCaller(s), other.Email())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, kr.CountEntities())
|
||||
}
|
||||
|
||||
func TestSameEmail(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
a, b string
|
||||
same bool
|
||||
}{
|
||||
{"user@proton.me", "user@proton.me", true},
|
||||
{"User@Proton.Me", "uSER@pROTON.mE", true},
|
||||
{"user@proton.me", "user@proton.mee", false},
|
||||
// Unicode case folding would match these to ASCII letters.
|
||||
{"kim@proton.me", "\u212aim@proton.me", false},
|
||||
{"sam@proton.me", "\u017fam@proton.me", false},
|
||||
{"élan@proton.me", "Élan@proton.me", false},
|
||||
} {
|
||||
assert.Equal(t, tc.same, emailKey(tc.a) == emailKey(tc.b), "%q %q", tc.a, tc.b)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,661 @@
|
||||
// Package api has the type definitions for the Proton Drive API.
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Response codes returned in the Code field of every API response.
|
||||
//
|
||||
// The names follow ErrorCode in the Proton Drive SDK
|
||||
// (client/js/src/internal/apiService/errorCodes.ts) unless noted.
|
||||
const (
|
||||
CodeOK = 1000
|
||||
CodeOKMany = 1001
|
||||
CodeOKAsync = 1002
|
||||
CodeInvalidRequirements = 2000
|
||||
CodeInvalidValue = 2001
|
||||
CodeNotEnoughPermissions = 2011
|
||||
CodeNotEnoughPermissionsToGrantPermissions = 2026
|
||||
CodeAlreadyExists = 2500
|
||||
CodeNotExists = 2501
|
||||
CodeInsufficientQuota = 200001
|
||||
CodeInsufficientSpace = 200002
|
||||
CodeMaxFileSizeForFreeUser = 200003
|
||||
CodeMaxPublicEditModeForFreeUser = 200004
|
||||
CodeInsufficientVolumeQuota = 200100
|
||||
CodeInsufficientDeviceQuota = 200101
|
||||
CodeAlreadyMemberOfShareInVolumeWithAnotherAddress = 200201
|
||||
CodeTooManyChildren = 200300
|
||||
CodeNestingTooDeep = 200301
|
||||
CodeInsufficientInvitationQuota = 200600
|
||||
CodeInsufficientShareQuota = 200601
|
||||
CodeInsufficientShareJoinedQuota = 200602
|
||||
CodeInsufficientBookmarksQuota = 200800
|
||||
|
||||
// CodeFeatureBlocked is returned when a feature is disabled,
|
||||
// rate-limited or blocked because of abuse (driveTypes.ts).
|
||||
CodeFeatureBlocked = 2028
|
||||
// CodeAppVersionMissing is returned when the request has no app
|
||||
// version (go-proton-api AppVersionMissingCode).
|
||||
CodeAppVersionMissing = 5001
|
||||
// CodeAppVersionBad is returned when the app version is no longer
|
||||
// accepted: "You need to update this app" (coreTypes.ts,
|
||||
// go-proton-api AppVersionBadCode).
|
||||
CodeAppVersionBad = 5003
|
||||
// CodeInvalidSection is returned when a request header such as
|
||||
// x-pm-appversion is malformed.
|
||||
CodeInvalidSection = 2064
|
||||
// CodePasswordWrong is returned when the login credentials are
|
||||
// incorrect (go-proton-api PasswordWrong).
|
||||
CodePasswordWrong = 8002
|
||||
// CodeHumanVerification is returned when a CAPTCHA must be solved.
|
||||
CodeHumanVerification = 9001
|
||||
// CodeRefreshTokenInvalid is returned when the refresh token is
|
||||
// rejected and the session must be established again.
|
||||
CodeRefreshTokenInvalid = 10013
|
||||
// CodeAddressMissing is returned by GET /core/v4/keys/all when no
|
||||
// Proton address has the email (account accountApi.ts).
|
||||
CodeAddressMissing = 33102
|
||||
// CodeDomainExternal is returned by GET /core/v4/keys/all when the
|
||||
// email's domain is not hosted by Proton (account accountApi.ts).
|
||||
CodeDomainExternal = 33103
|
||||
)
|
||||
|
||||
// Error is the body of an unsuccessful API response.
|
||||
type Error struct {
|
||||
Status int `json:"-"` // HTTP status of the response
|
||||
Code int `json:"Code"` // Proton response code
|
||||
Message string `json:"Error"` // human readable message
|
||||
Details json.RawMessage `json:"Details,omitempty"` // request specific details
|
||||
}
|
||||
|
||||
// Error returns the message with its code and HTTP status.
|
||||
func (e *Error) Error() string {
|
||||
return fmt.Sprintf("%s (Code=%d, Status=%d)", e.Message, e.Code, e.Status)
|
||||
}
|
||||
|
||||
// AuthRefreshRequest is the body of POST /auth/v4/refresh.
|
||||
type AuthRefreshRequest struct {
|
||||
ResponseType string `json:"ResponseType"` // always "token"
|
||||
GrantType string `json:"GrantType"` // always "refresh_token"
|
||||
RefreshToken string `json:"RefreshToken"`
|
||||
}
|
||||
|
||||
// AuthRefreshResponse is the response to POST /auth/v4/refresh.
|
||||
type AuthRefreshResponse struct {
|
||||
Code int `json:"Code"`
|
||||
UID string `json:"UID"`
|
||||
AccessToken string `json:"AccessToken"`
|
||||
RefreshToken string `json:"RefreshToken"`
|
||||
}
|
||||
|
||||
// AuthInfoRequest is the body of POST /core/v4/auth/info.
|
||||
type AuthInfoRequest struct {
|
||||
Intent string `json:"Intent"` // always "Proton"
|
||||
Username string `json:"Username"`
|
||||
}
|
||||
|
||||
// AuthInfoResponse is the response to POST /core/v4/auth/info: the
|
||||
// server's SRP challenge.
|
||||
type AuthInfoResponse struct {
|
||||
Code int `json:"Code"`
|
||||
Version int `json:"Version"` // SRP password hashing version
|
||||
Modulus string `json:"Modulus"` // signed base64 SRP modulus
|
||||
ServerEphemeral string `json:"ServerEphemeral"` // base64
|
||||
Salt string `json:"Salt"` // base64 SRP salt
|
||||
SRPSession string `json:"SRPSession"`
|
||||
Username string `json:"Username"`
|
||||
}
|
||||
|
||||
// AuthRequest is the body of POST /core/v4/auth.
|
||||
type AuthRequest struct {
|
||||
Username string `json:"Username"`
|
||||
ClientEphemeral string `json:"ClientEphemeral"` // base64
|
||||
ClientProof string `json:"ClientProof"` // base64
|
||||
SRPSession string `json:"SRPSession"`
|
||||
PersistentCookies int `json:"PersistentCookies"`
|
||||
Payload map[string]string `json:"Payload"`
|
||||
}
|
||||
|
||||
// Values of TwoFAInfo.Enabled, which may be combined.
|
||||
const (
|
||||
TwoFATOTP = 1
|
||||
TwoFAFIDO2 = 2
|
||||
)
|
||||
|
||||
// Values of AuthResponse.PasswordMode.
|
||||
const (
|
||||
PasswordModeOne = 1 // the login password also unlocks the keys
|
||||
PasswordModeTwo = 2 // a separate mailbox password unlocks the keys
|
||||
)
|
||||
|
||||
// TwoFAInfo describes the second factors an account requires.
|
||||
type TwoFAInfo struct {
|
||||
Enabled int `json:"Enabled"` // bitmask of TwoFATOTP and TwoFAFIDO2
|
||||
}
|
||||
|
||||
// AuthResponse is the response to POST /core/v4/auth.
|
||||
type AuthResponse struct {
|
||||
Code int `json:"Code"`
|
||||
UID string `json:"UID"`
|
||||
UserID string `json:"UserID"`
|
||||
AccessToken string `json:"AccessToken"`
|
||||
RefreshToken string `json:"RefreshToken"`
|
||||
TokenType string `json:"TokenType"`
|
||||
ServerProof string `json:"ServerProof"` // base64
|
||||
Scopes []string `json:"Scopes"`
|
||||
PasswordMode int `json:"PasswordMode"`
|
||||
TwoFA TwoFAInfo `json:"2FA"`
|
||||
}
|
||||
|
||||
// Auth2FARequest is the body of POST /core/v4/auth/2fa.
|
||||
type Auth2FARequest struct {
|
||||
TwoFactorCode string `json:"TwoFactorCode"`
|
||||
}
|
||||
|
||||
// KeySalt is the salt of one user key's password.
|
||||
type KeySalt struct {
|
||||
ID string `json:"ID"` // user key ID
|
||||
KeySalt string `json:"KeySalt"` // base64
|
||||
}
|
||||
|
||||
// KeySaltsResponse is the response to GET /core/v4/keys/salts.
|
||||
type KeySaltsResponse struct {
|
||||
Code int `json:"Code"`
|
||||
KeySalts []KeySalt `json:"KeySalts"`
|
||||
}
|
||||
|
||||
// ForkInitResponse is the response to GET /auth/v4/sessions/forks.
|
||||
type ForkInitResponse struct {
|
||||
Code int `json:"Code"`
|
||||
Selector string `json:"Selector"`
|
||||
UserCode string `json:"UserCode"`
|
||||
}
|
||||
|
||||
// ForkStatusResponse is the response to GET
|
||||
// /auth/v4/sessions/forks/{selector} once the user has signed in.
|
||||
type ForkStatusResponse struct {
|
||||
Code int `json:"Code"`
|
||||
Payload string `json:"Payload"` // base64 AES-256-GCM encrypted key password
|
||||
UID string `json:"UID"`
|
||||
AccessToken string `json:"AccessToken"`
|
||||
RefreshToken string `json:"RefreshToken"`
|
||||
}
|
||||
|
||||
// UserKey is one of the user's keys.
|
||||
type UserKey struct {
|
||||
ID string `json:"ID"`
|
||||
Version int `json:"Version"`
|
||||
PrivateKey string `json:"PrivateKey"` // armored, locked with the key password
|
||||
Primary int `json:"Primary"` // 1 for the primary key
|
||||
Active int `json:"Active"` // 0 for a key which must not be unlocked
|
||||
}
|
||||
|
||||
// User is the account of the signed in user.
|
||||
type User struct {
|
||||
ID string `json:"ID"`
|
||||
Name string `json:"Name"`
|
||||
Email string `json:"Email"`
|
||||
Keys []UserKey `json:"Keys"`
|
||||
|
||||
UsedSpace int64 `json:"UsedSpace"` // bytes used by all products
|
||||
MaxSpace int64 `json:"MaxSpace"` // bytes of storage all products share
|
||||
}
|
||||
|
||||
// UserResponse is the response to GET /core/v4/users.
|
||||
type UserResponse struct {
|
||||
Code int `json:"Code"`
|
||||
User User `json:"User"`
|
||||
}
|
||||
|
||||
// AddressKey is one of an address's keys.
|
||||
type AddressKey struct {
|
||||
ID string `json:"ID"`
|
||||
Version int `json:"Version"`
|
||||
PrivateKey string `json:"PrivateKey"` // armored, locked with the passphrase in Token
|
||||
// Token is the armored key passphrase encrypted to the user key,
|
||||
// or empty for a legacy key locked with the key password.
|
||||
Token string `json:"Token"`
|
||||
Signature string `json:"Signature"` // armored detached signature of the passphrase by the user key
|
||||
Primary int `json:"Primary"` // 1 for the primary key
|
||||
Active int `json:"Active"` // 0 for a key which must not be unlocked
|
||||
Flags int `json:"Flags"` // bitmap of key usages
|
||||
}
|
||||
|
||||
// Address is one of the user's email addresses.
|
||||
type Address struct {
|
||||
ID string `json:"ID"`
|
||||
Email string `json:"Email"`
|
||||
Status int `json:"Status"` // 0 disabled, 1 enabled, 2 deleting
|
||||
Type int `json:"Type"` // 1 original, 2 alias, 3 custom, 4 premium, 5 external
|
||||
Priority int `json:"Priority"` // order of the addresses, lowest first
|
||||
Keys []AddressKey `json:"Keys"`
|
||||
}
|
||||
|
||||
// AddressesResponse is the response to GET /core/v4/addresses.
|
||||
type AddressesResponse struct {
|
||||
Code int `json:"Code"`
|
||||
Addresses []Address `json:"Addresses"`
|
||||
}
|
||||
|
||||
// PublicKey is a public key of an address.
|
||||
type PublicKey struct {
|
||||
PublicKey string `json:"PublicKey"` // armored
|
||||
Primary int `json:"Primary"` // 1 for the primary key
|
||||
Flags int `json:"Flags"` // bitmap of key usages
|
||||
Source int `json:"Source"` // 0 for internal keys
|
||||
}
|
||||
|
||||
// PublicKeysResponse is the response to GET /core/v4/keys/all.
|
||||
type PublicKeysResponse struct {
|
||||
Code int `json:"Code"`
|
||||
Address struct {
|
||||
Keys []PublicKey `json:"Keys"`
|
||||
} `json:"Address"`
|
||||
}
|
||||
|
||||
// Link types, as Link.Type encodes them.
|
||||
const (
|
||||
LinkTypeFolder = 1
|
||||
LinkTypeFile = 2
|
||||
LinkTypeAlbum = 3
|
||||
)
|
||||
|
||||
// Link states, as Link.State encodes them.
|
||||
const (
|
||||
LinkStateDraft = 0
|
||||
LinkStateActive = 1
|
||||
LinkStateTrashed = 2
|
||||
)
|
||||
|
||||
// Volume is the summary of a volume in PrimaryRootShareResponseDto.
|
||||
type Volume struct {
|
||||
VolumeID string `json:"VolumeID"`
|
||||
UsedSpace int64 `json:"UsedSpace"`
|
||||
}
|
||||
|
||||
// Share is a share with its key, as ShareDto.
|
||||
type Share struct {
|
||||
ShareID string `json:"ShareID"`
|
||||
CreatorEmail string `json:"CreatorEmail"`
|
||||
Key string `json:"Key"` // armored share key, locked with the passphrase
|
||||
Passphrase string `json:"Passphrase"` // armored, encrypted to the address key
|
||||
PassphraseSignature string `json:"PassphraseSignature"` // armored detached signature by the creator
|
||||
AddressID string `json:"AddressID"` // address whose key decrypts the passphrase
|
||||
}
|
||||
|
||||
// Link is the metadata every node has, as LinkDto. Strings the API
|
||||
// sends as null are empty.
|
||||
type Link struct {
|
||||
LinkID string `json:"LinkID"`
|
||||
Type int `json:"Type"` // LinkTypeFolder, LinkTypeFile or LinkTypeAlbum
|
||||
ParentLinkID string `json:"ParentLinkID"` // empty for a root
|
||||
State int `json:"State"` // LinkStateDraft, LinkStateActive or LinkStateTrashed
|
||||
CreateTime int64 `json:"CreateTime"` // Unix seconds
|
||||
ModifyTime int64 `json:"ModifyTime"` // Unix seconds
|
||||
TrashTime int64 `json:"TrashTime"` // Unix seconds, 0 unless trashed
|
||||
Name string `json:"Name"` // armored, encrypted to the parent key
|
||||
NameHash string `json:"NameHash"` // lower-case hex lookup hash; empty for a root
|
||||
NodeKey string `json:"NodeKey"` // armored, locked with the node passphrase
|
||||
NodePassphrase string `json:"NodePassphrase"`
|
||||
NodePassphraseSignature string `json:"NodePassphraseSignature"`
|
||||
SignatureEmail string `json:"SignatureEmail"` // signer of the key and attributes; empty if anonymous
|
||||
NameSignatureEmail string `json:"NameSignatureEmail"` // signer of the name; empty if anonymous
|
||||
ThirdParty bool `json:"ThirdParty"`
|
||||
Sdk bool `json:"Sdk"`
|
||||
}
|
||||
|
||||
// Folder is the folder part of a node's metadata, as FolderDto.
|
||||
type Folder struct {
|
||||
NodeHashKey string `json:"NodeHashKey"` // armored, encrypted to the node key
|
||||
XAttr string `json:"XAttr"` // armored extended attributes; may be empty
|
||||
IsImported bool `json:"IsImported"`
|
||||
}
|
||||
|
||||
// ActiveRevision is the summary of a file's active revision, as
|
||||
// ActiveRevisionDto.
|
||||
type ActiveRevision struct {
|
||||
RevisionID string `json:"RevisionID"`
|
||||
CreateTime int64 `json:"CreateTime"` // Unix seconds
|
||||
EncryptedSize int64 `json:"EncryptedSize"`
|
||||
ChecksumVerified bool `json:"ChecksumVerified"`
|
||||
ManifestSignature string `json:"ManifestSignature"` // armored
|
||||
XAttr string `json:"XAttr"` // armored extended attributes; may be empty
|
||||
SignatureEmail string `json:"SignatureEmail"` // signer of the revision; empty if anonymous
|
||||
IsImported bool `json:"IsImported"`
|
||||
ThirdParty bool `json:"ThirdParty"`
|
||||
Sdk bool `json:"Sdk"`
|
||||
}
|
||||
|
||||
// File is the file part of a node's metadata, as FileDto.
|
||||
type File struct {
|
||||
ActiveRevision *ActiveRevision `json:"ActiveRevision"` // nil for a draft
|
||||
TotalEncryptedSize int64 `json:"TotalEncryptedSize"`
|
||||
ContentKeyPacket string `json:"ContentKeyPacket"` // base64
|
||||
MediaType string `json:"MediaType"`
|
||||
ContentKeyPacketSignature string `json:"ContentKeyPacketSignature"` // armored; may be empty
|
||||
}
|
||||
|
||||
// LinkDetails is a node's metadata, as FileDetailsDto and
|
||||
// FolderDetailsDto.
|
||||
type LinkDetails struct {
|
||||
Link Link `json:"Link"`
|
||||
Folder *Folder `json:"Folder"` // set for a folder
|
||||
File *File `json:"File"` // set for a file
|
||||
}
|
||||
|
||||
// MyFilesResponse is the response to GET /drive/v2/shares/my-files, as
|
||||
// PrimaryRootShareResponseDto.
|
||||
type MyFilesResponse struct {
|
||||
Code int `json:"Code"`
|
||||
Volume Volume `json:"Volume"`
|
||||
Share Share `json:"Share"`
|
||||
Link LinkDetails `json:"Link"` // the root folder
|
||||
}
|
||||
|
||||
// CreateVolumeRequest is the body of POST /drive/volumes, as
|
||||
// CreateVolumeRequestDto without the deprecated names.
|
||||
type CreateVolumeRequest struct {
|
||||
AddressID string `json:"AddressID"`
|
||||
AddressKeyID string `json:"AddressKeyID"` // the primary key of AddressID
|
||||
ShareKey string `json:"ShareKey"`
|
||||
SharePassphrase string `json:"SharePassphrase"`
|
||||
SharePassphraseSignature string `json:"SharePassphraseSignature"`
|
||||
FolderName string `json:"FolderName"`
|
||||
FolderKey string `json:"FolderKey"`
|
||||
FolderPassphrase string `json:"FolderPassphrase"`
|
||||
FolderPassphraseSignature string `json:"FolderPassphraseSignature"`
|
||||
FolderHashKey string `json:"FolderHashKey"`
|
||||
}
|
||||
|
||||
// ShareReference identifies the main share of a volume and its root,
|
||||
// as ShareReferenceResponseDto.
|
||||
type ShareReference struct {
|
||||
ShareID string `json:"ShareID"`
|
||||
LinkID string `json:"LinkID"`
|
||||
}
|
||||
|
||||
// VolumeDetails is a volume, as VolumeResponseDto.
|
||||
type VolumeDetails struct {
|
||||
ID string `json:"ID"`
|
||||
VolumeID string `json:"VolumeID"`
|
||||
UsedSpace int64 `json:"UsedSpace"`
|
||||
State int `json:"State"`
|
||||
Share ShareReference `json:"Share"`
|
||||
}
|
||||
|
||||
// VolumeResponse is the response to POST /drive/volumes and GET
|
||||
// /drive/volumes/{volumeID}, as GetVolumeResponseDto.
|
||||
type VolumeResponse struct {
|
||||
Code int `json:"Code"`
|
||||
Volume VolumeDetails `json:"Volume"`
|
||||
}
|
||||
|
||||
// LinksRequest is the body of POST /drive/v2/volumes/{volumeID}/links
|
||||
// and /drive/v2/volumes/{volumeID}/trash_multiple, as LinkIDsRequestDto.
|
||||
type LinksRequest struct {
|
||||
LinkIDs []string `json:"LinkIDs"`
|
||||
}
|
||||
|
||||
// LinksResponse is the response to POST
|
||||
// /drive/v2/volumes/{volumeID}/links, as LoadLinkDetailsResponseDto.
|
||||
// Unknown IDs are left out.
|
||||
type LinksResponse struct {
|
||||
Code int `json:"Code"`
|
||||
Links []LinkDetails `json:"Links"`
|
||||
}
|
||||
|
||||
// ChildrenResponse is the response to GET
|
||||
// /drive/v2/volumes/{volumeID}/folders/{linkID}/children, as
|
||||
// ListChildrenResponseDto.
|
||||
type ChildrenResponse struct {
|
||||
Code int `json:"Code"`
|
||||
LinkIDs []string `json:"LinkIDs"`
|
||||
AnchorID string `json:"AnchorID"` // pass as AnchorID for the next page
|
||||
More bool `json:"More"`
|
||||
}
|
||||
|
||||
// CreateFolderRequest is the body of POST
|
||||
// /drive/v2/volumes/{volumeID}/folders, as CreateFolderDto with the
|
||||
// SignatureEmail the SDK sends in place of SignatureAddress.
|
||||
type CreateFolderRequest struct {
|
||||
ParentLinkID string `json:"ParentLinkID"`
|
||||
NodeKey string `json:"NodeKey"`
|
||||
NodeHashKey string `json:"NodeHashKey"`
|
||||
NodePassphrase string `json:"NodePassphrase"`
|
||||
NodePassphraseSignature string `json:"NodePassphraseSignature"`
|
||||
SignatureEmail string `json:"SignatureEmail"`
|
||||
Name string `json:"Name"`
|
||||
Hash string `json:"Hash"`
|
||||
XAttr string `json:"XAttr,omitempty"`
|
||||
}
|
||||
|
||||
// CreateFolderResponse is the response to POST
|
||||
// /drive/v2/volumes/{volumeID}/folders, as CreateFolderResponseDto.
|
||||
type CreateFolderResponse struct {
|
||||
Code int `json:"Code"`
|
||||
Folder struct {
|
||||
ID string `json:"ID"`
|
||||
} `json:"Folder"`
|
||||
}
|
||||
|
||||
// RenameRequest is the body of PUT
|
||||
// /drive/v2/volumes/{volumeID}/links/{linkID}/rename, as
|
||||
// RenameLinkRequestDto.
|
||||
type RenameRequest struct {
|
||||
Name string `json:"Name"` // reusing the session key of the current name
|
||||
Hash string `json:"Hash"`
|
||||
NameSignatureEmail string `json:"NameSignatureEmail"`
|
||||
OriginalHash string `json:"OriginalHash"` // current name hash, which must match
|
||||
}
|
||||
|
||||
// MoveRequest is the body of PUT
|
||||
// /drive/v2/volumes/{volumeID}/links/{linkID}/move, as
|
||||
// MoveLinkV2RequestDto.
|
||||
//
|
||||
// NodePassphraseSignature and SignatureEmail are given if and only if
|
||||
// the node's key is anonymous (has no SignatureEmail).
|
||||
type MoveRequest struct {
|
||||
ParentLinkID string `json:"ParentLinkID"`
|
||||
NodePassphrase string `json:"NodePassphrase"` // reusing the session key of the current passphrase
|
||||
NodePassphraseSignature string `json:"NodePassphraseSignature,omitempty"`
|
||||
SignatureEmail string `json:"SignatureEmail,omitempty"`
|
||||
Name string `json:"Name"` // reusing the session key of the current name
|
||||
NameSignatureEmail string `json:"NameSignatureEmail"`
|
||||
Hash string `json:"Hash"`
|
||||
OriginalHash string `json:"OriginalHash"` // current name hash, which must match
|
||||
}
|
||||
|
||||
// CopyRequest is the body of POST
|
||||
// /drive/volumes/{volumeID}/links/{linkID}/copy, as CopyLinkRequestDto.
|
||||
// The server copies everything else of the node.
|
||||
//
|
||||
// NodePassphraseSignature and SignatureEmail are given if and only if
|
||||
// the node's key is anonymous (has no SignatureEmail).
|
||||
type CopyRequest struct {
|
||||
Name string `json:"Name"` // reusing the session key of the current name
|
||||
NodePassphrase string `json:"NodePassphrase"` // reusing the session key of the current passphrase
|
||||
Hash string `json:"Hash"`
|
||||
TargetVolumeID string `json:"TargetVolumeID"`
|
||||
TargetParentLinkID string `json:"TargetParentLinkID"`
|
||||
NameSignatureEmail string `json:"NameSignatureEmail"`
|
||||
NodePassphraseSignature string `json:"NodePassphraseSignature,omitempty"`
|
||||
SignatureEmail string `json:"SignatureEmail,omitempty"`
|
||||
}
|
||||
|
||||
// CopyResponse is the response to POST
|
||||
// /drive/volumes/{volumeID}/links/{linkID}/copy, as CopyLinkResponseDto.
|
||||
type CopyResponse struct {
|
||||
Code int `json:"Code"`
|
||||
LinkID string `json:"LinkID"` // the new node
|
||||
}
|
||||
|
||||
// LinkResult is the outcome for one link of a request on many.
|
||||
type LinkResult struct {
|
||||
LinkID string `json:"LinkID"`
|
||||
Response struct {
|
||||
Code int `json:"Code"`
|
||||
Error string `json:"Error"`
|
||||
} `json:"Response"`
|
||||
}
|
||||
|
||||
// MultiLinkResponse is the response to a request on many links, such
|
||||
// as POST /drive/v2/volumes/{volumeID}/trash_multiple, as
|
||||
// MultiResponsesPerLinkFactory.
|
||||
type MultiLinkResponse struct {
|
||||
Code int `json:"Code"`
|
||||
Responses []LinkResult `json:"Responses"`
|
||||
}
|
||||
|
||||
// RevisionResponse is the response to GET
|
||||
// /drive/v2/volumes/{volumeID}/files/{linkID}/revisions/{revisionID},
|
||||
// as GetRevisionResponseDto.
|
||||
type RevisionResponse struct {
|
||||
Code int `json:"Code"`
|
||||
Revision Revision `json:"Revision"`
|
||||
}
|
||||
|
||||
// Revision is a revision of a file with a page of its blocks, as
|
||||
// DetailedRevisionResponseDto.
|
||||
type Revision struct {
|
||||
ID string `json:"ID"`
|
||||
ManifestSignature string `json:"ManifestSignature"` // armored
|
||||
Size int64 `json:"Size"` // encrypted size
|
||||
State int `json:"State"`
|
||||
XAttr string `json:"XAttr"` // armored extended attributes; may be empty
|
||||
Blocks []Block `json:"Blocks"`
|
||||
Thumbnails []Thumbnail `json:"Thumbnails"`
|
||||
SignatureEmail string `json:"SignatureEmail"` // signer of the manifest; empty if anonymous
|
||||
CreateTime int64 `json:"CreateTime"` // Unix seconds
|
||||
}
|
||||
|
||||
// Block is a block of a revision with the URL to download it, as
|
||||
// BlockResponseDto.
|
||||
type Block struct {
|
||||
Index int `json:"Index"` // from 1
|
||||
Hash string `json:"Hash"` // base64 SHA-256 of the encrypted block
|
||||
Token string `json:"Token"` // send as pm-storage-token
|
||||
BareURL string `json:"BareURL"` // storage URL of the block
|
||||
}
|
||||
|
||||
// Thumbnail is a thumbnail of a revision, as ThumbnailResponseDto.
|
||||
type Thumbnail struct {
|
||||
ThumbnailID string `json:"ThumbnailID"`
|
||||
Type int `json:"Type"`
|
||||
Hash string `json:"Hash"` // base64 SHA-256 of the encrypted thumbnail
|
||||
Size int64 `json:"Size"`
|
||||
}
|
||||
|
||||
// ConflictDetails are the Details of a CodeAlreadyExists error from
|
||||
// creating a node, naming the node which has the name.
|
||||
type ConflictDetails struct {
|
||||
ConflictLinkID string `json:"ConflictLinkID"`
|
||||
ConflictRevisionID string `json:"ConflictRevisionID"` // active revision; empty for a draft or folder
|
||||
ConflictDraftRevisionID string `json:"ConflictDraftRevisionID"` // draft revision; empty if none
|
||||
ConflictDraftClientUID string `json:"ConflictDraftClientUID"` // ClientUID of the draft revision's creator
|
||||
}
|
||||
|
||||
// CreateFileRequest is the body of POST
|
||||
// /drive/v2/volumes/{volumeID}/files, as CreateFileDto: a draft file.
|
||||
type CreateFileRequest struct {
|
||||
ParentLinkID string `json:"ParentLinkID"`
|
||||
Name string `json:"Name"`
|
||||
Hash string `json:"Hash"`
|
||||
MIMEType string `json:"MIMEType"`
|
||||
ClientUID *string `json:"ClientUID"` // identifies the creator's drafts; null if none
|
||||
IntendedUploadSize *int64 `json:"IntendedUploadSize"` // rounded size for an early quota check; null if unknown
|
||||
NodeKey string `json:"NodeKey"`
|
||||
NodePassphrase string `json:"NodePassphrase"`
|
||||
NodePassphraseSignature string `json:"NodePassphraseSignature"`
|
||||
ContentKeyPacket string `json:"ContentKeyPacket"` // base64
|
||||
ContentKeyPacketSignature string `json:"ContentKeyPacketSignature"` // armored, by the node key
|
||||
SignatureAddress string `json:"SignatureAddress"` // signer of the name and passphrase
|
||||
}
|
||||
|
||||
// CreateFileResponse is the response to POST
|
||||
// /drive/v2/volumes/{volumeID}/files, as CreateDraftFileResponseDto.
|
||||
type CreateFileResponse struct {
|
||||
Code int `json:"Code"`
|
||||
File struct {
|
||||
ID string `json:"ID"`
|
||||
RevisionID string `json:"RevisionID"` // the draft revision
|
||||
} `json:"File"`
|
||||
}
|
||||
|
||||
// CreateRevisionRequest is the body of POST
|
||||
// /drive/v2/volumes/{volumeID}/files/{linkID}/revisions, as
|
||||
// CreateRevisionRequestDto: a draft revision of an active file.
|
||||
type CreateRevisionRequest struct {
|
||||
CurrentRevisionID string `json:"CurrentRevisionID"` // the active revision, which must match
|
||||
ClientUID *string `json:"ClientUID"`
|
||||
IntendedUploadSize *int64 `json:"IntendedUploadSize"`
|
||||
}
|
||||
|
||||
// CreateRevisionResponse is the response to POST
|
||||
// /drive/v2/volumes/{volumeID}/files/{linkID}/revisions.
|
||||
type CreateRevisionResponse struct {
|
||||
Code int `json:"Code"`
|
||||
Revision struct {
|
||||
ID string `json:"ID"`
|
||||
} `json:"Revision"`
|
||||
}
|
||||
|
||||
// VerificationResponse is the response to GET
|
||||
// /drive/v2/volumes/{volumeID}/links/{linkID}/revisions/{revisionID}/verification,
|
||||
// as VerificationData.
|
||||
type VerificationResponse struct {
|
||||
Code int `json:"Code"`
|
||||
VerificationCode string `json:"VerificationCode"` // base64
|
||||
ContentKeyPacket string `json:"ContentKeyPacket"` // base64
|
||||
}
|
||||
|
||||
// BlockUploadRequest is the body of POST /drive/blocks, as
|
||||
// RequestUploadInput without its deprecated fields.
|
||||
type BlockUploadRequest struct {
|
||||
AddressID string `json:"AddressID"` // address signing the blocks
|
||||
VolumeID string `json:"VolumeID"`
|
||||
LinkID string `json:"LinkID"`
|
||||
RevisionID string `json:"RevisionID"`
|
||||
BlockList []BlockUploadInfo `json:"BlockList"`
|
||||
ThumbnailList []struct{} `json:"ThumbnailList"`
|
||||
}
|
||||
|
||||
// BlockUploadInfo is a block to upload, as RequestUploadBlockInput
|
||||
// without its deprecated Size and Hash, which the SDK doesn't send.
|
||||
type BlockUploadInfo struct {
|
||||
Index int `json:"Index"` // from 1, consecutive
|
||||
EncSignature string `json:"EncSignature"` // armored signature of the plaintext, encrypted
|
||||
Verifier BlockVerifier `json:"Verifier"`
|
||||
}
|
||||
|
||||
// BlockVerifier proves a block was encrypted with the right key.
|
||||
type BlockVerifier struct {
|
||||
Token string `json:"Token"` // base64 verification code XOR the start of the encrypted block
|
||||
}
|
||||
|
||||
// BlockUploadResponse is the response to POST /drive/blocks, as
|
||||
// RequestUploadResponse.
|
||||
type BlockUploadResponse struct {
|
||||
Code int `json:"Code"`
|
||||
UploadLinks []UploadLink `json:"UploadLinks"`
|
||||
}
|
||||
|
||||
// UploadLink is the storage URL to upload a block to, as BlockURL.
|
||||
type UploadLink struct {
|
||||
Index int `json:"Index"`
|
||||
BareURL string `json:"BareURL"`
|
||||
Token string `json:"Token"` // send as pm-storage-token
|
||||
}
|
||||
|
||||
// CommitRevisionRequest is the body of PUT
|
||||
// /drive/v2/volumes/{volumeID}/files/{linkID}/revisions/{revisionID},
|
||||
// as CommitRevisionDto without its deprecated fields.
|
||||
type CommitRevisionRequest struct {
|
||||
ManifestSignature string `json:"ManifestSignature"` // armored, of the block hashes in order
|
||||
SignatureAddress string `json:"SignatureAddress"` // signer of the manifest, blocks and attributes
|
||||
XAttr string `json:"XAttr"` // armored extended attributes
|
||||
ChecksumVerified bool `json:"ChecksumVerified"` // whether the SHA1 matched the source's
|
||||
Photo *struct{} `json:"Photo"` // always null outside the photo volume
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
package protondrive_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/protondrive/protontest"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
fssync "github.com/rclone/rclone/fs/sync"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Routes counted by the request budgets.
|
||||
const (
|
||||
linksRoute = "POST /drive/v2/volumes/{volumeID}/links"
|
||||
revisionDraftRoute = "POST /drive/v2/volumes/{volumeID}/files/{linkID}/revisions"
|
||||
verifierRoute = "GET /drive/v2/volumes/{volumeID}/links/{linkID}/revisions/{revisionID}/verification"
|
||||
blockURLsRoute = "POST /drive/blocks"
|
||||
blockPutRoute = "POST /storage/upload/{token}"
|
||||
commitRoute = "PUT /drive/v2/volumes/{volumeID}/files/{linkID}/revisions/{revisionID}"
|
||||
createFolderRoute = "POST /drive/v2/volumes/{volumeID}/folders"
|
||||
)
|
||||
|
||||
// TestMain runs the tests, then fails the run if any of them, through
|
||||
// any fake server, requested a removed share-scoped route.
|
||||
//
|
||||
// The config file is RCLONE_CONFIG's, as on rclone's command line,
|
||||
// whichever tests run: only fstest.Initialise would apply it, so the
|
||||
// default config, which may hold a real Proton remote, would be read.
|
||||
func TestMain(m *testing.M) {
|
||||
if path, ok := os.LookupEnv("RCLONE_CONFIG"); ok {
|
||||
if err := config.SetConfigPath(path); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "FAIL: RCLONE_CONFIG: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
code := m.Run()
|
||||
if legacy := protontest.LegacyRequests(); len(legacy) > 0 {
|
||||
fmt.Fprintf(os.Stderr, "FAIL: TestBudgetNoLegacyRoutes: removed share-scoped routes were requested: %v\n", legacy)
|
||||
code = 1
|
||||
}
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
// totalRequests returns the sum of the counts in requests.
|
||||
func totalRequests(requests map[string]int) int {
|
||||
n := 0
|
||||
for _, count := range requests {
|
||||
n += count
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// ceilDiv returns n/d rounded up.
|
||||
func ceilDiv(n, d int) int {
|
||||
return (n + d - 1) / d
|
||||
}
|
||||
|
||||
// childPages returns the number of child ID pages the fake returns for
|
||||
// a folder of n children with page size pageSize: at least one.
|
||||
func childPages(n, pageSize int) int {
|
||||
return max(1, ceilDiv(n, pageSize))
|
||||
}
|
||||
|
||||
// seedFiles seeds n small files into folder parentID.
|
||||
func seedFiles(u *protontest.User, parentID string, n int) {
|
||||
for i := range n {
|
||||
u.SeedFile(parentID, fmt.Sprintf("file%03d.txt", i), []byte(fmt.Sprintf("file %d", i)), time.Now())
|
||||
}
|
||||
}
|
||||
|
||||
// budgetSizes are the folder sizes the listing budgets are checked at:
|
||||
// around the metadata batch of 100 and the child page of 150.
|
||||
var budgetSizes = []int{0, 1, 100, 101, 250}
|
||||
|
||||
// budgetPageSize is the child ID page size the listing budgets use.
|
||||
const budgetPageSize = 150
|
||||
|
||||
// seedBudgetFolders seeds a folder "nN" of N files for each of sizes.
|
||||
func seedBudgetFolders(srv *protontest.Server, u *protontest.User, sizes []int) {
|
||||
srv.SetChildrenPageSize(budgetPageSize)
|
||||
for _, n := range sizes {
|
||||
seedFiles(u, u.SeedFolder(u.RootID(), fmt.Sprintf("n%d", n)), n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBudgetList checks listing a folder of N files costs one request
|
||||
// per page of child IDs and one per batch of 100 nodes, within spec
|
||||
// 9.4's P + ceil(N/100) + 1. The Fs is rooted at the folder, so finding
|
||||
// it is part of NewFs.
|
||||
//
|
||||
// A second listing costs nothing with enable_caching at its default,
|
||||
// and with it off costs the same as the first, no more. The latter is
|
||||
// checked at the sizes which differ in kind - empty, one batch, several
|
||||
// pages and batches - to save time.
|
||||
//
|
||||
// Measured, per route (children pages + links batches):
|
||||
// N=0: 1+0, N=1: 1+1, N=100: 1+1, N=101: 1+2, N=250: 2+3 - exactly
|
||||
// P + ceil(N/100), so the bound is that, without the spec's +1. Warm: 0
|
||||
// at every N; with caching off the second listing repeats the first.
|
||||
func TestBudgetList(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
remote, srv, u := setupFakeRemote(t)
|
||||
seedBudgetFolders(srv, u, budgetSizes)
|
||||
list := func(t *testing.T, f fs.Fs, n int) map[string]int {
|
||||
t.Helper()
|
||||
srv.ResetRequests()
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, n)
|
||||
return srv.Requests()
|
||||
}
|
||||
for _, n := range budgetSizes {
|
||||
t.Run(fmt.Sprint(n), func(t *testing.T) {
|
||||
root := fmt.Sprintf("n%d", n)
|
||||
f := newFakeFs(t, remote, root)
|
||||
cold := list(t, f, n)
|
||||
pages, batches := childPages(n, budgetPageSize), ceilDiv(n, 100)
|
||||
assert.LessOrEqual(t, cold[childrenRoute], pages, "%v", cold)
|
||||
assert.LessOrEqual(t, cold[linksRoute], batches, "%v", cold)
|
||||
assert.LessOrEqual(t, totalRequests(cold), pages+batches, "%v", cold)
|
||||
|
||||
assert.Empty(t, list(t, f, n), "warm listing")
|
||||
|
||||
if n == 100 || n == 101 {
|
||||
return
|
||||
}
|
||||
t.Setenv(fakeEnvPrefix+"ENABLE_CACHING", "false")
|
||||
uncached := newFakeFs(t, remote, root)
|
||||
assert.Equal(t, cold, list(t, uncached, n), "first listing without caching")
|
||||
assert.Equal(t, cold, list(t, uncached, n), "second listing without caching")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBudgetNoopSync checks a cold sync of a tree already in sync, by
|
||||
// size and modification time and by --checksum, lists each folder once
|
||||
// and transfers nothing: SHA-1s come from the metadata.
|
||||
//
|
||||
// For D = 6 folders (the root and 5 below it) and N = 200 files (40 per
|
||||
// folder), the plan's bound is 2*D + ceil(N/100) + 5 = 19. Measured, per
|
||||
// route, either way: 6 children pages + 6 links batches = 12, one page
|
||||
// and one batch per folder, as no folder has more than 100 children.
|
||||
// The bound is that: 2*D.
|
||||
func TestBudgetNoopSync(t *testing.T) {
|
||||
const (
|
||||
folders = 5
|
||||
filesPerFolder = 40
|
||||
budget = 2 * (folders + 1)
|
||||
)
|
||||
remote, srv, u := setupFakeRemote(t)
|
||||
local := t.TempDir()
|
||||
modTime := time.Now().Add(-time.Hour).Truncate(time.Second)
|
||||
for d := range folders {
|
||||
dir := fmt.Sprintf("dir%d", d)
|
||||
id := u.SeedFolder(u.RootID(), dir)
|
||||
require.NoError(t, os.Mkdir(filepath.Join(local, dir), 0o777))
|
||||
for i := range filesPerFolder {
|
||||
name := fmt.Sprintf("file%02d.txt", i)
|
||||
content := []byte(fmt.Sprintf("content of %s/%s", dir, name))
|
||||
u.SeedFile(id, name, content, modTime)
|
||||
p := filepath.Join(local, dir, name)
|
||||
require.NoError(t, os.WriteFile(p, content, 0o666))
|
||||
require.NoError(t, os.Chtimes(p, modTime, modTime))
|
||||
}
|
||||
}
|
||||
|
||||
for _, checksum := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("checksum=%v", checksum), func(t *testing.T) {
|
||||
ctx, ci := fs.AddConfig(context.Background())
|
||||
ci.CheckSum = checksum
|
||||
src, err := fs.NewFs(ctx, local)
|
||||
require.NoError(t, err)
|
||||
f := newFakeFs(t, remote, "")
|
||||
srv.ResetRequests()
|
||||
require.NoError(t, fssync.Sync(ctx, f, src, false))
|
||||
|
||||
got := srv.Requests()
|
||||
assert.LessOrEqual(t, totalRequests(got), budget, "%v", got)
|
||||
assert.Zero(t, blockTransfers(got), "%v", got)
|
||||
assert.Zero(t, got[draftRoute]+got[revisionDraftRoute], "drafts created: %v", got)
|
||||
for route := range got {
|
||||
assert.NotContains(t, route, "/revisions/", "file content read: %v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBudgetNewObject checks finding a file three folders deep, cold,
|
||||
// costs one listing, a children page and a links batch, per folder on
|
||||
// its path.
|
||||
//
|
||||
// The plan's bound is 2 per level + 1 = 7 for three levels. Measured
|
||||
// for a/b/c/file.txt, per route: 4 children pages + 4 links batches =
|
||||
// 8, as the root is listed too - it is a fourth folder on the path,
|
||||
// and Proton has no route to look a child up by name. The bound is 2
|
||||
// per folder listed: 2*4.
|
||||
func TestBudgetNewObject(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
remote, srv, u := setupFakeRemote(t)
|
||||
parent := u.RootID()
|
||||
for _, name := range []string{"a", "b", "c"} {
|
||||
seedFiles(u, parent, 3)
|
||||
parent = u.SeedFolder(parent, name)
|
||||
}
|
||||
u.SeedFile(parent, "file.txt", []byte("deep"), time.Now())
|
||||
|
||||
f := newFakeFs(t, remote, "")
|
||||
srv.ResetRequests()
|
||||
o, err := f.NewObject(ctx, "a/b/c/file.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(4), o.Size())
|
||||
|
||||
got := srv.Requests()
|
||||
const listed = 4 // the root, a, b and c
|
||||
assert.LessOrEqual(t, got[childrenRoute], listed, "%v", got)
|
||||
assert.LessOrEqual(t, got[linksRoute], listed, "%v", got)
|
||||
assert.LessOrEqual(t, totalRequests(got), 2*listed, "%v", got)
|
||||
}
|
||||
|
||||
// TestBudgetUpload checks the requests of uploading new files to a
|
||||
// folder: one listing of the folder to check the name is free, the
|
||||
// first time only, then per file a draft, its verification code, at
|
||||
// most one block URL request per block, one PUT per block, the commit
|
||||
// and one read of the committed node.
|
||||
//
|
||||
// Measured, per route, into a folder of 3 files:
|
||||
//
|
||||
// small.txt, cold: children 1 + links 1 (the listing), draft 1,
|
||||
// verifier 1, block URLs 1, block PUTs 1, commit 1, links 1 = 8
|
||||
// big.bin, 3 blocks: draft 1, verifier 1, block URLs 1-3 (as blocks
|
||||
// are ready), block PUTs 3, commit 1, links 1 = 8-10
|
||||
// again.txt: as small.txt without the listing = 6
|
||||
func TestBudgetUpload(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
remote, srv, u := setupFakeRemote(t)
|
||||
seedFiles(u, u.SeedFolder(u.RootID(), "dir"), 3)
|
||||
f := newFakeFs(t, remote, "dir")
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
size int
|
||||
blocks int
|
||||
listed bool // the folder is listed to check the name is free
|
||||
}{
|
||||
{"small.txt", 5, 1, true},
|
||||
{"big.bin", 2*blockSize + 5, 3, false},
|
||||
{"again.txt", 5, 1, false},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
srv.ResetRequests()
|
||||
content := patternedContent(test.size, 1)
|
||||
_, err := operations.Copy(ctx, f, nil, test.name, object.NewMemoryObject(test.name, time.Now(), content))
|
||||
require.NoError(t, err)
|
||||
|
||||
got := srv.Requests()
|
||||
listing := 0
|
||||
if test.listed {
|
||||
listing = 2 // a children page and a links batch
|
||||
assert.Equal(t, 1, got[childrenRoute], "%v", got)
|
||||
} else {
|
||||
assert.Zero(t, got[childrenRoute], "folder listed again: %v", got)
|
||||
}
|
||||
assert.Equal(t, 1, got[draftRoute], "%v", got)
|
||||
assert.Equal(t, 1, got[verifierRoute], "%v", got)
|
||||
assert.GreaterOrEqual(t, got[blockURLsRoute], 1, "%v", got)
|
||||
assert.LessOrEqual(t, got[blockURLsRoute], test.blocks, "%v", got)
|
||||
assert.Equal(t, test.blocks, got[blockPutRoute], "%v", got)
|
||||
assert.Equal(t, 1, got[commitRoute], "%v", got)
|
||||
assert.Equal(t, listing/2+1, got[linksRoute], "%v", got)
|
||||
assert.LessOrEqual(t, totalRequests(got), listing+4+2*test.blocks, "%v", got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBudgetMkdir checks making folders side by side lists their
|
||||
// parent once, to check the first name is free, and then costs one
|
||||
// request per folder.
|
||||
//
|
||||
// Measured, per route, into a folder of 3 files: x: children 1 +
|
||||
// links 1 + create 1 = 3; y and z: create 1 each.
|
||||
func TestBudgetMkdir(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
remote, srv, u := setupFakeRemote(t)
|
||||
seedFiles(u, u.SeedFolder(u.RootID(), "dir"), 3)
|
||||
f := newFakeFs(t, remote, "dir")
|
||||
srv.ResetRequests()
|
||||
for _, name := range []string{"x", "y", "z"} {
|
||||
require.NoError(t, f.Mkdir(ctx, name))
|
||||
}
|
||||
got := srv.Requests()
|
||||
assert.Equal(t, map[string]int{childrenRoute: 1, linksRoute: 1, createFolderRoute: 3}, got)
|
||||
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, entries, 6, "the cached listing holds the new folders")
|
||||
}
|
||||
|
||||
// TestBudgetNoLegacyRoutes checks a run of everyday operations requests
|
||||
// no removed share-scoped route. TestMain checks the same of every
|
||||
// test in the package, through every fake server.
|
||||
func TestBudgetNoLegacyRoutes(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
remote, srv, u := setupFakeRemote(t)
|
||||
seedFiles(u, u.SeedFolder(u.RootID(), "dir"), 3)
|
||||
f := newFakeFs(t, remote, "")
|
||||
o, err := operations.Copy(ctx, f, nil, "dir/new.txt", object.NewMemoryObject("dir/new.txt", time.Now(), []byte("new")))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "new", readObject(t, o))
|
||||
_, err = operations.Move(ctx, f, nil, "moved/new.txt", o)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, operations.Purge(ctx, f, "dir"))
|
||||
_, err = f.List(ctx, "moved")
|
||||
require.NoError(t, err)
|
||||
_, err = f.Features().About(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
for route := range srv.Requests() {
|
||||
if strings.Contains(route, "/drive/shares/") {
|
||||
assert.Equal(t, "GET /drive/shares/{shareID}", route)
|
||||
}
|
||||
}
|
||||
assert.Empty(t, protontest.LegacyRequests())
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
package protondrive
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// nodeCache holds decrypted nodes and folder listings in memory.
|
||||
//
|
||||
// A nil nodeCache, as newNodeCache returns when caching is disabled,
|
||||
// holds nothing: its lookups miss and its updates do nothing. Cached
|
||||
// nodes are shared and must not be modified.
|
||||
type nodeCache struct {
|
||||
mu sync.Mutex
|
||||
nodes map[string]*node // by ID
|
||||
listings map[string][]string // IDs of the complete listing of a folder, by folder ID
|
||||
fetching map[string]*fetchState // folders being listed from the server, by folder ID
|
||||
}
|
||||
|
||||
// fetchState tracks the listings of a folder being fetched, so that
|
||||
// one which began before a change to the folder's children isn't
|
||||
// cached over it.
|
||||
type fetchState struct {
|
||||
count int // listings in progress
|
||||
changes uint64 // changes to the folder's children since the first began
|
||||
}
|
||||
|
||||
// newNodeCache returns an empty cache, or nil if enabled is false.
|
||||
func newNodeCache(enabled bool) *nodeCache {
|
||||
if !enabled {
|
||||
return nil
|
||||
}
|
||||
return &nodeCache{
|
||||
nodes: make(map[string]*node),
|
||||
listings: make(map[string][]string),
|
||||
fetching: make(map[string]*fetchState),
|
||||
}
|
||||
}
|
||||
|
||||
// get returns the node id, or nil if it isn't cached.
|
||||
func (c *nodeCache) get(id string) *node {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.nodes[id]
|
||||
}
|
||||
|
||||
// put caches n, replacing any node with its ID. If the cached node had
|
||||
// another parent, the listings of both parents are forgotten.
|
||||
func (c *nodeCache) put(n *node) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if old := c.nodes[n.ID]; old != nil && old.ParentID != n.ParentID {
|
||||
c.childrenChangedLocked(old.ParentID)
|
||||
c.childrenChangedLocked(n.ParentID)
|
||||
}
|
||||
c.nodes[n.ID] = n
|
||||
}
|
||||
|
||||
// children returns the cached listing of folder parentID. It reports
|
||||
// false if the listing, or any node in it, isn't cached.
|
||||
func (c *nodeCache) children(parentID string) ([]*node, bool) {
|
||||
if c == nil {
|
||||
return nil, false
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
ids, ok := c.listings[parentID]
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
out := make([]*node, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
n := c.nodes[id]
|
||||
if n == nil {
|
||||
return nil, false
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
// setChildren caches nodes as the complete listing of folder parentID,
|
||||
// and each of the nodes.
|
||||
func (c *nodeCache) setChildren(parentID string, nodes []*node) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.setChildrenLocked(parentID, nodes)
|
||||
}
|
||||
|
||||
// setChildrenLocked is setChildren with c.mu held.
|
||||
func (c *nodeCache) setChildrenLocked(parentID string, nodes []*node) {
|
||||
ids := make([]string, len(nodes))
|
||||
for i, n := range nodes {
|
||||
ids[i] = n.ID
|
||||
c.nodes[n.ID] = n
|
||||
}
|
||||
c.listings[parentID] = ids
|
||||
}
|
||||
|
||||
// startFetch records that a listing of folder parentID is being read
|
||||
// from the server and returns the token to pass to endFetch, which
|
||||
// must be called.
|
||||
func (c *nodeCache) startFetch(parentID string) (token uint64) {
|
||||
if c == nil {
|
||||
return 0
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
st := c.fetching[parentID]
|
||||
if st == nil {
|
||||
st = &fetchState{}
|
||||
c.fetching[parentID] = st
|
||||
}
|
||||
st.count++
|
||||
return st.changes
|
||||
}
|
||||
|
||||
// endFetch ends a listing of folder parentID begun by startFetch with
|
||||
// token. If complete is set, nodes are cached as the listing, unless
|
||||
// the folder's children changed since it began, when what it read may
|
||||
// be out of date.
|
||||
func (c *nodeCache) endFetch(parentID string, token uint64, nodes []*node, complete bool) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
st := c.fetching[parentID]
|
||||
if complete && st.changes == token {
|
||||
c.setChildrenLocked(parentID, nodes)
|
||||
}
|
||||
st.count--
|
||||
if st.count == 0 {
|
||||
delete(c.fetching, parentID)
|
||||
}
|
||||
}
|
||||
|
||||
// childrenChangedLocked forgets the listing of folder parentID, and
|
||||
// marks its listings being fetched as out of date. c.mu must be held.
|
||||
func (c *nodeCache) childrenChangedLocked(parentID string) {
|
||||
delete(c.listings, parentID)
|
||||
if st := c.fetching[parentID]; st != nil {
|
||||
st.changes++
|
||||
}
|
||||
}
|
||||
|
||||
// add caches n, a node new to its parent, and adds it to the cached
|
||||
// listing of its parent, if there is one.
|
||||
func (c *nodeCache) add(n *node) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if old := c.nodes[n.ID]; old != nil && old.ParentID != n.ParentID {
|
||||
c.childrenChangedLocked(old.ParentID)
|
||||
c.childrenChangedLocked(n.ParentID)
|
||||
}
|
||||
c.nodes[n.ID] = n
|
||||
if st := c.fetching[n.ParentID]; st != nil {
|
||||
st.changes++
|
||||
}
|
||||
if ids, ok := c.listings[n.ParentID]; ok && !slices.Contains(ids, n.ID) {
|
||||
c.listings[n.ParentID] = append(ids, n.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// invalidate forgets the node id, its listing if it is a folder, and
|
||||
// the listing of its parent.
|
||||
func (c *nodeCache) invalidate(id string) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if n := c.nodes[id]; n != nil {
|
||||
c.childrenChangedLocked(n.ParentID)
|
||||
} else {
|
||||
// Its parent is unknown, and any listing being fetched may
|
||||
// hold it.
|
||||
for _, st := range c.fetching {
|
||||
st.changes++
|
||||
}
|
||||
}
|
||||
delete(c.nodes, id)
|
||||
c.childrenChangedLocked(id)
|
||||
}
|
||||
|
||||
// invalidateChildren forgets the listing of folder parentID, but not
|
||||
// the nodes in it.
|
||||
func (c *nodeCache) invalidateChildren(parentID string) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.childrenChangedLocked(parentID)
|
||||
}
|
||||
|
||||
// clear forgets everything.
|
||||
func (c *nodeCache) clear() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
clear(c.nodes)
|
||||
clear(c.listings)
|
||||
for _, st := range c.fetching {
|
||||
st.changes++
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
package protondrive
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNodeCacheDisabled(t *testing.T) {
|
||||
c := newNodeCache(false)
|
||||
require.Nil(t, c)
|
||||
n := &node{ID: "a", ParentID: "root"}
|
||||
c.put(n)
|
||||
c.setChildren("root", []*node{n})
|
||||
assert.Nil(t, c.get("a"))
|
||||
_, ok := c.children("root")
|
||||
assert.False(t, ok)
|
||||
c.invalidate("a")
|
||||
c.invalidateChildren("root")
|
||||
c.clear()
|
||||
}
|
||||
|
||||
func TestNodeCache(t *testing.T) {
|
||||
c := newNodeCache(true)
|
||||
dir := &node{ID: "dir", ParentID: "root", IsFolder: true}
|
||||
file := &node{ID: "file", ParentID: "dir"}
|
||||
|
||||
assert.Nil(t, c.get("dir"))
|
||||
_, ok := c.children("root")
|
||||
assert.False(t, ok)
|
||||
|
||||
c.setChildren("root", []*node{dir})
|
||||
c.setChildren("dir", []*node{file})
|
||||
assert.Same(t, dir, c.get("dir"))
|
||||
assert.Same(t, file, c.get("file"))
|
||||
got, ok := c.children("dir")
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, []*node{file}, got)
|
||||
|
||||
// An empty listing is a listing.
|
||||
c.setChildren("empty", nil)
|
||||
got, ok = c.children("empty")
|
||||
assert.True(t, ok)
|
||||
assert.Empty(t, got)
|
||||
|
||||
// put replaces the node in the listing.
|
||||
renamed := &node{ID: "file", ParentID: "dir", Name: "new"}
|
||||
c.put(renamed)
|
||||
got, ok = c.children("dir")
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, []*node{renamed}, got)
|
||||
|
||||
// put of a node with another parent forgets both listings.
|
||||
c.setChildren("other", nil)
|
||||
moved := &node{ID: "file", ParentID: "other"}
|
||||
c.put(moved)
|
||||
assert.Same(t, moved, c.get("file"))
|
||||
_, ok = c.children("dir")
|
||||
assert.False(t, ok)
|
||||
_, ok = c.children("other")
|
||||
assert.False(t, ok)
|
||||
_, ok = c.children("root")
|
||||
assert.True(t, ok)
|
||||
c.setChildren("dir", []*node{renamed})
|
||||
|
||||
// invalidate forgets the node and its parent's listing.
|
||||
c.invalidate("file")
|
||||
assert.Nil(t, c.get("file"))
|
||||
_, ok = c.children("dir")
|
||||
assert.False(t, ok)
|
||||
_, ok = c.children("root")
|
||||
assert.True(t, ok)
|
||||
|
||||
// and a folder's own listing.
|
||||
c.setChildren("dir", []*node{file})
|
||||
c.invalidate("dir")
|
||||
_, ok = c.children("dir")
|
||||
assert.False(t, ok)
|
||||
_, ok = c.children("root")
|
||||
assert.False(t, ok)
|
||||
assert.Same(t, file, c.get("file"))
|
||||
|
||||
c.setChildren("dir", []*node{file})
|
||||
c.invalidateChildren("dir")
|
||||
_, ok = c.children("dir")
|
||||
assert.False(t, ok)
|
||||
assert.Same(t, file, c.get("file"))
|
||||
|
||||
c.setChildren("dir", []*node{file})
|
||||
c.clear()
|
||||
assert.Nil(t, c.get("file"))
|
||||
_, ok = c.children("dir")
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestNodeCacheAdd(t *testing.T) {
|
||||
c := newNodeCache(true)
|
||||
a := &node{ID: "a", ParentID: "dir"}
|
||||
b := &node{ID: "b", ParentID: "dir"}
|
||||
|
||||
// Without a listing of its parent, only the node is cached.
|
||||
c.add(a)
|
||||
assert.Same(t, a, c.get("a"))
|
||||
_, ok := c.children("dir")
|
||||
assert.False(t, ok)
|
||||
|
||||
c.setChildren("dir", []*node{a})
|
||||
c.add(b)
|
||||
got, ok := c.children("dir")
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, []*node{a, b}, got)
|
||||
|
||||
// A node already listed is replaced, not listed twice.
|
||||
b2 := &node{ID: "b", ParentID: "dir", Name: "new"}
|
||||
c.add(b2)
|
||||
got, ok = c.children("dir")
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, []*node{a, b2}, got)
|
||||
|
||||
newNodeCache(false).add(a)
|
||||
}
|
||||
|
||||
func TestNodeCacheFetch(t *testing.T) {
|
||||
a := &node{ID: "a", ParentID: "dir"}
|
||||
listed := func(c *nodeCache) bool {
|
||||
_, ok := c.children("dir")
|
||||
return ok
|
||||
}
|
||||
|
||||
c := newNodeCache(true)
|
||||
token := c.startFetch("dir")
|
||||
c.endFetch("dir", token, []*node{a}, true)
|
||||
assert.True(t, listed(c), "unchanged")
|
||||
assert.Empty(t, c.fetching)
|
||||
|
||||
c = newNodeCache(true)
|
||||
token = c.startFetch("dir")
|
||||
c.endFetch("dir", token, []*node{a}, false)
|
||||
assert.False(t, listed(c), "incomplete")
|
||||
assert.Empty(t, c.fetching)
|
||||
|
||||
for name, change := range map[string]func(c *nodeCache){
|
||||
"add": func(c *nodeCache) { c.add(&node{ID: "b", ParentID: "dir"}) },
|
||||
"invalidate child": func(c *nodeCache) { c.invalidate("a") },
|
||||
"invalidate unknown": func(c *nodeCache) { c.invalidate("unknown") },
|
||||
"invalidate folder": func(c *nodeCache) { c.invalidate("dir") },
|
||||
"invalidateChildren": func(c *nodeCache) { c.invalidateChildren("dir") },
|
||||
"put moved in": func(c *nodeCache) { c.put(&node{ID: "other", ParentID: "dir"}) },
|
||||
"put moved out": func(c *nodeCache) { c.put(&node{ID: "a", ParentID: "elsewhere"}) },
|
||||
"clear": func(c *nodeCache) { c.clear() },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
c := newNodeCache(true)
|
||||
c.put(a)
|
||||
c.put(&node{ID: "other", ParentID: "elsewhere"})
|
||||
first := c.startFetch("dir")
|
||||
second := c.startFetch("dir")
|
||||
change(c)
|
||||
c.endFetch("dir", first, []*node{a}, true)
|
||||
assert.False(t, listed(c), "first listing cached after a change")
|
||||
// A fetch begun after the change is cached.
|
||||
third := c.startFetch("dir")
|
||||
c.endFetch("dir", second, []*node{a}, true)
|
||||
assert.False(t, listed(c), "second listing cached after a change")
|
||||
c.endFetch("dir", third, []*node{a}, true)
|
||||
assert.True(t, listed(c))
|
||||
assert.Empty(t, c.fetching, "fetch state kept")
|
||||
})
|
||||
}
|
||||
|
||||
// Changes elsewhere don't matter.
|
||||
c = newNodeCache(true)
|
||||
c.put(&node{ID: "x", ParentID: "other"})
|
||||
token = c.startFetch("dir")
|
||||
c.add(&node{ID: "y", ParentID: "other"})
|
||||
c.invalidate("x")
|
||||
c.invalidateChildren("other")
|
||||
c.endFetch("dir", token, []*node{a}, true)
|
||||
assert.True(t, listed(c))
|
||||
|
||||
var disabled *nodeCache
|
||||
disabled.endFetch("dir", disabled.startFetch("dir"), nil, true)
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,786 @@
|
||||
package protondrive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"github.com/ProtonMail/gopenpgp/v3/crypto"
|
||||
"github.com/rclone/rclone/backend/protondrive/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
var (
|
||||
// errIntegrity is returned when a file's content doesn't match
|
||||
// what its uploader signed or claimed.
|
||||
errIntegrity = errors.New("integrity check failed")
|
||||
|
||||
// errBlockHash is returned when a block doesn't match its hash
|
||||
// however many times it is downloaded.
|
||||
errBlockHash = errors.New("block hash mismatch")
|
||||
|
||||
// errDownloadClosed is returned by Read after Close.
|
||||
errDownloadClosed = errors.New("download closed")
|
||||
)
|
||||
|
||||
// signatureHint ends an error for a signature which fails, telling the
|
||||
// user how to read the file anyway.
|
||||
const signatureHint = "set ignore_signature_errors to read the file anyway"
|
||||
|
||||
const (
|
||||
// defaultBlockSize is the plaintext size of every block but the
|
||||
// last of a file the client uploads.
|
||||
defaultBlockSize = 4 << 20
|
||||
|
||||
// blocksPerPage is the most blocks one revision request lists.
|
||||
blocksPerPage = 20
|
||||
|
||||
// maxBlockSize is the largest block accepted, encrypted or
|
||||
// decrypted.
|
||||
maxBlockSize = 8 << 20
|
||||
|
||||
// blockAttempts is how many times a block whose hash doesn't match
|
||||
// is downloaded.
|
||||
blockAttempts = 3
|
||||
|
||||
// downloadAhead is the most blocks of one download fetched or
|
||||
// waiting to be read, besides the one being read.
|
||||
downloadAhead = 4
|
||||
|
||||
// minUnclaimedBlockSize bounds the blocks of a file which doesn't
|
||||
// record its block sizes: there are at most its encrypted size over
|
||||
// this, plus one.
|
||||
minUnclaimedBlockSize = 1 << 20
|
||||
|
||||
// maxFileBlocks is the most blocks a file may list: 2 TiB in
|
||||
// 4 MiB blocks, beyond any one file the largest Proton plan, of a
|
||||
// few TB in all, could hold. A full read holds the metadata of
|
||||
// every block until the manifest is checked, so this bounds how
|
||||
// many whatever the server claims: about 250-350 MiB at the cap
|
||||
// with realistic tokens and URLs. Their length isn't bounded.
|
||||
maxFileBlocks = 1 << 19
|
||||
|
||||
// contentKeyFallbackUntil is the Unix time until which a content
|
||||
// key packet may be signed by any of the account's address keys,
|
||||
// as in the SDK.
|
||||
contentKeyFallbackUntil = 1672531200 // 2023-01-01
|
||||
)
|
||||
|
||||
// open returns a reader of limit bytes of the content of file n from
|
||||
// offset, or to the end if limit < 0.
|
||||
//
|
||||
// A read from offset 0 to the end lists every block and checks the
|
||||
// manifest signature before releasing any byte: if it fails, the first
|
||||
// Read returns errIntegrity, unless IgnoreSignatureErrors is set, when
|
||||
// it is logged. A ranged read checks each block against its listed
|
||||
// hash but can't check the manifest. A node key or content key
|
||||
// signature which fails is errIntegrity from open, on the same terms.
|
||||
// The reader must be closed.
|
||||
func (c *client) open(ctx context.Context, n *node, offset, limit int64) (io.ReadCloser, error) {
|
||||
return c.openRead(ctx, n, offset, limit, offset == 0 && limit < 0)
|
||||
}
|
||||
|
||||
// openWhole is open for limit bytes from offset 0 of a range which
|
||||
// covers the whole file, checked as a full read is.
|
||||
func (c *client) openWhole(ctx context.Context, n *node, limit int64) (io.ReadCloser, error) {
|
||||
return c.openRead(ctx, n, 0, limit, true)
|
||||
}
|
||||
|
||||
// openRead is open, listing every block and checking the manifest
|
||||
// before releasing any byte if full is set, which needs offset 0.
|
||||
func (c *client) openRead(ctx context.Context, n *node, offset, limit int64, full bool) (io.ReadCloser, error) {
|
||||
if n.IsFolder || n.link == nil || n.link.File == nil || n.RevisionID == "" {
|
||||
return nil, fmt.Errorf("open %q: not a file with content", n.Name)
|
||||
}
|
||||
if offset < 0 {
|
||||
return nil, fmt.Errorf("open %q: negative offset %d", n.Name, offset)
|
||||
}
|
||||
sk, err := c.contentKey(n, "reading")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open %q: %w", n.Name, err)
|
||||
}
|
||||
var sizes []int64
|
||||
if n.X != nil {
|
||||
sizes = n.X.BlockSizes
|
||||
}
|
||||
full = full && offset == 0
|
||||
if len(sizes) > c.maxFileBlocks {
|
||||
return nil, fmt.Errorf("open %q: %w: %d blocks recorded, more than %d blocks", n.Name, errIntegrity, len(sizes), c.maxFileBlocks)
|
||||
}
|
||||
if sizes != nil {
|
||||
var total int64
|
||||
for i, size := range sizes {
|
||||
// Bounding each size bounds the total too.
|
||||
if size > maxBlockSize {
|
||||
return nil, fmt.Errorf("open %q: %w: block %d recorded as %d bytes", n.Name, errIntegrity, i+1, size)
|
||||
}
|
||||
total += size
|
||||
}
|
||||
switch {
|
||||
case full:
|
||||
// A full read always lists the blocks, to reach the
|
||||
// manifest.
|
||||
case offset >= total:
|
||||
limit = 0
|
||||
case limit >= 0:
|
||||
limit = min(limit, total-offset)
|
||||
}
|
||||
}
|
||||
if limit == 0 {
|
||||
return io.NopCloser(bytes.NewReader(nil)), nil
|
||||
}
|
||||
// Without recorded block sizes only the sizes learnt by earlier
|
||||
// reads tell which block holds offset, as old clients didn't all
|
||||
// write 4 MiB blocks, so the read starts at the last block they
|
||||
// reach and discards the bytes before offset.
|
||||
first, skip, last := 1, offset, -1
|
||||
var startHash []byte
|
||||
switch {
|
||||
case sizes != nil:
|
||||
first, skip = blockAt(sizes, offset)
|
||||
if limit > 0 && !full {
|
||||
last, _ = blockAt(sizes, offset+limit-1)
|
||||
}
|
||||
case !full:
|
||||
first, skip, last, startHash = c.learnt.locate(n.RevisionID, offset, limit)
|
||||
}
|
||||
maxBlocks := len(sizes)
|
||||
if sizes == nil {
|
||||
maxBlocks = int(min(n.EncryptedSize/minUnclaimedBlockSize+1, 1<<30))
|
||||
}
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
r := &blockReader{
|
||||
c: c,
|
||||
n: n,
|
||||
sk: sk,
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
slots: make(chan chan blockResult, downloadAhead),
|
||||
sizes: sizes,
|
||||
startHash: startHash,
|
||||
skip: skip,
|
||||
remaining: limit,
|
||||
}
|
||||
r.wg.Add(1)
|
||||
go r.list(first, last, maxBlocks, full)
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// blockAt returns the index, from 1, of the block holding plaintext
|
||||
// offset pos of a file with block sizes sizes, and the offset of pos in
|
||||
// it.
|
||||
func blockAt(sizes []int64, pos int64) (index int, offset int64) {
|
||||
for i, size := range sizes {
|
||||
if pos < size {
|
||||
return i + 1, pos
|
||||
}
|
||||
pos -= size
|
||||
}
|
||||
return len(sizes) + 1, pos
|
||||
}
|
||||
|
||||
// contentKey returns the content session key of file n for op,
|
||||
// "reading" or "updating", checking its signature as the SDK's
|
||||
// decryptContentKeyPacket does, except that a missing signature fails
|
||||
// too: anyone with the node's public key can make a content key packet.
|
||||
//
|
||||
// An unverified node key or content key is errIntegrity unless
|
||||
// IgnoreSignatureErrors is set, when it is logged once per node and op:
|
||||
// whoever made the key could read what is written with it and make up
|
||||
// the content read.
|
||||
func (c *client) contentKey(n *node, op string) (*crypto.SessionKey, error) {
|
||||
hint := signatureHint
|
||||
if op != "reading" {
|
||||
hint = "set ignore_signature_errors to update the file anyway"
|
||||
}
|
||||
if n.keys.unverified != nil {
|
||||
if !c.opt.IgnoreSignatureErrors {
|
||||
return nil, fmt.Errorf("node key: %w: %w: %s", errIntegrity, n.keys.unverified, hint)
|
||||
}
|
||||
c.reportUnverified(op, n, "")
|
||||
}
|
||||
f := n.link.File
|
||||
now := c.s.serverTime()
|
||||
sk, verifyErr, err := decryptContentKey(f.ContentKeyPacket, f.ContentKeyPacketSignature, n.keys.keyRing, n.keyVerifyKR, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if verifyErr != nil && !isNotSigned(verifyErr) && n.CreateTime.Unix() <= contentKeyFallbackUntil {
|
||||
// Old clients signed with an address key, and a rename or
|
||||
// move could change the node's signer without signing the
|
||||
// content key packet again.
|
||||
krs := make([]*crypto.KeyRing, 0, len(c.acct.addresses))
|
||||
for _, addr := range c.acct.addresses {
|
||||
krs = append(krs, addr.keyRing)
|
||||
}
|
||||
if allKR, err := joinKeyRings(krs...); err == nil && verifyDetached(sk.Key, f.ContentKeyPacketSignature, allKR, now) == nil {
|
||||
fs.Debugf(nil, "protondrive: %q (ID %s): content key signed by another address of the account", n.Name, n.ID)
|
||||
verifyErr = nil
|
||||
}
|
||||
}
|
||||
if verifyErr != nil {
|
||||
verifyErr = fmt.Errorf("content key: %w: %w", errIntegrity, verifyErr)
|
||||
if !c.opt.IgnoreSignatureErrors {
|
||||
return nil, fmt.Errorf("%w: %s", verifyErr, hint)
|
||||
}
|
||||
c.reportOnce("content key "+op, n.ID, func() {
|
||||
fs.Errorf(nil, "protondrive: %s %q (ID %s): %v", op, n.Name, n.ID, verifyErr)
|
||||
})
|
||||
}
|
||||
return sk, nil
|
||||
}
|
||||
|
||||
// revisionBlocks returns the active revision of file n with at most
|
||||
// pageSize of its blocks from block index from.
|
||||
func (c *client) revisionBlocks(ctx context.Context, n *node, from, pageSize int) (*api.Revision, error) {
|
||||
opts := &rest.Opts{
|
||||
Method: "GET",
|
||||
Path: c.volumePath("/drive/v2/volumes", "files", n.ID, "revisions", n.RevisionID),
|
||||
Parameters: url.Values{
|
||||
"FromBlockIndex": {strconv.Itoa(from)},
|
||||
"PageSize": {strconv.Itoa(pageSize)},
|
||||
},
|
||||
}
|
||||
var resp api.RevisionResponse
|
||||
if err := c.apiCall(ctx, opts, nil, &resp); err != nil {
|
||||
return nil, fmt.Errorf("list blocks: %w", err)
|
||||
}
|
||||
return &resp.Revision, nil
|
||||
}
|
||||
|
||||
// blockURL returns block index of file n with a new download URL.
|
||||
func (c *client) blockURL(ctx context.Context, n *node, index int) (api.Block, error) {
|
||||
rev, err := c.revisionBlocks(ctx, n, index, 1)
|
||||
if err != nil {
|
||||
return api.Block{}, err
|
||||
}
|
||||
if len(rev.Blocks) == 0 || rev.Blocks[0].Index != index {
|
||||
return api.Block{}, fmt.Errorf("list blocks: block %d is missing", index)
|
||||
}
|
||||
return rev.Blocks[0], nil
|
||||
}
|
||||
|
||||
// downloadBlock returns the encrypted block at bareURL, which token
|
||||
// unlocks, and its SHA-256. The block must be closed.
|
||||
func (c *client) downloadBlock(ctx context.Context, bareURL, token string) (enc *pool.RW, sum []byte, err error) {
|
||||
c.downloadTokens.Get()
|
||||
defer c.downloadTokens.Put()
|
||||
opts := &rest.Opts{
|
||||
Method: "GET",
|
||||
RootURL: bareURL,
|
||||
// The storage host gets the block's token, never the
|
||||
// session's credentials, which only session.call adds.
|
||||
ExtraHeaders: map[string]string{"pm-storage-token": token},
|
||||
}
|
||||
err = c.pacedCall(ctx, c.storagePacer, func() (*http.Response, error) {
|
||||
if enc != nil {
|
||||
_ = enc.Close()
|
||||
}
|
||||
enc = pool.NewRW(pool.Global())
|
||||
resp, err := c.s.srv.Call(ctx, opts)
|
||||
if err != nil {
|
||||
return resp, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.ContentLength > maxBlockSize {
|
||||
return nil, fmt.Errorf("block is larger than %d bytes", maxBlockSize)
|
||||
}
|
||||
h := sha256.New()
|
||||
size, err := enc.ReadFrom(io.TeeReader(io.LimitReader(resp.Body, maxBlockSize+1), h))
|
||||
if err != nil {
|
||||
return resp, err
|
||||
}
|
||||
if size > maxBlockSize {
|
||||
return nil, fmt.Errorf("block is larger than %d bytes", maxBlockSize)
|
||||
}
|
||||
sum = h.Sum(nil)
|
||||
return resp, nil
|
||||
})
|
||||
if err != nil {
|
||||
if enc != nil {
|
||||
_ = enc.Close()
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
return enc, sum, nil
|
||||
}
|
||||
|
||||
// blockResult is the outcome of a block of a download, or its end.
|
||||
type blockResult struct {
|
||||
index int
|
||||
hash []byte // the block's SHA-256
|
||||
plain *pool.RW // the decrypted block; nil on error or at the end
|
||||
err error
|
||||
end bool // no more blocks follow; err is the verdict on the file
|
||||
}
|
||||
|
||||
// blockReader reads a range of a file, fetching blocks ahead of the
|
||||
// reader in the background.
|
||||
type blockReader struct {
|
||||
c *client
|
||||
n *node
|
||||
sk *crypto.SessionKey
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
slots chan chan blockResult // results to read, in order
|
||||
wg sync.WaitGroup // the lister and the fetches
|
||||
sizes []int64 // claimed plaintext block sizes; nil if not recorded
|
||||
// startHash is the hash the first block had when its size was
|
||||
// learnt, if it was.
|
||||
startHash []byte
|
||||
|
||||
mu sync.Mutex // held by Read and Close
|
||||
cur *pool.RW // block being read
|
||||
skip int64 // bytes to discard before the range, from the first block on
|
||||
remaining int64 // bytes left to read; negative for all
|
||||
err error // returned by every Read once set
|
||||
closed bool
|
||||
}
|
||||
|
||||
// list lists the blocks first to last (to the end if last < 0) and
|
||||
// fetches them, then sends the end of the download. If verify is set,
|
||||
// the list having started at the first block, it lists every block and
|
||||
// checks the manifest before fetching any.
|
||||
//
|
||||
// A list of more than maxBlocks blocks is an error.
|
||||
func (r *blockReader) list(first, last, maxBlocks int, verify bool) {
|
||||
defer r.wg.Done()
|
||||
end := make(chan blockResult, 1)
|
||||
end <- blockResult{end: true, err: r.listBlocks(first, last, maxBlocks, verify)}
|
||||
select {
|
||||
case r.slots <- end:
|
||||
case <-r.ctx.Done():
|
||||
}
|
||||
}
|
||||
|
||||
// listedBlock is a block of the list with its decoded hash.
|
||||
type listedBlock struct {
|
||||
block api.Block
|
||||
hash []byte
|
||||
}
|
||||
|
||||
// listBlocks does the work of list, returning the verdict on the file.
|
||||
func (r *blockReader) listBlocks(first, last, maxBlocks int, verify bool) error {
|
||||
var (
|
||||
hashes [][]byte
|
||||
listed []listedBlock // blocks waiting for the manifest check, if verify
|
||||
manifest string
|
||||
signer string
|
||||
index = first
|
||||
firstPage = true
|
||||
)
|
||||
for {
|
||||
pageSize := blocksPerPage
|
||||
if last > 0 {
|
||||
pageSize = min(pageSize, last-index+1)
|
||||
}
|
||||
rev, err := r.c.revisionBlocks(r.ctx, r.n, index, pageSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if firstPage && verify {
|
||||
manifest, signer = rev.ManifestSignature, rev.SignatureEmail
|
||||
// As in the SDK, the manifest starts with the thumbnails.
|
||||
for _, th := range rev.Thumbnails {
|
||||
hash, err := decodeBlockHash(th.Hash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("thumbnail: %w", err)
|
||||
}
|
||||
hashes = append(hashes, hash)
|
||||
}
|
||||
}
|
||||
firstPage = false
|
||||
if len(rev.Blocks) == 0 {
|
||||
break
|
||||
}
|
||||
for _, b := range rev.Blocks {
|
||||
switch {
|
||||
case index > r.c.maxFileBlocks:
|
||||
return fmt.Errorf("%w: more than %d blocks listed, more than any file can have", errIntegrity, r.c.maxFileBlocks)
|
||||
case b.Index != index:
|
||||
return fmt.Errorf("%w: block %d listed where block %d belongs", errIntegrity, b.Index, index)
|
||||
case index > maxBlocks && r.sizes != nil:
|
||||
return fmt.Errorf("%w: more blocks listed than the %d the file records", errIntegrity, maxBlocks)
|
||||
case index > maxBlocks:
|
||||
return fmt.Errorf("%w: more blocks listed than the file's encrypted size of %d allows", errIntegrity, r.n.EncryptedSize)
|
||||
}
|
||||
hash, err := decodeBlockHash(b.Hash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("block %d: %w", index, err)
|
||||
}
|
||||
if index == first && r.startHash != nil && !bytes.Equal(hash, r.startHash) {
|
||||
// The sizes placing the range came from another block
|
||||
// list of the same revision.
|
||||
return fmt.Errorf("%w: block %d listed with another hash than when it was read before", errIntegrity, index)
|
||||
}
|
||||
if verify {
|
||||
// Nothing is released before the manifest vouches for
|
||||
// every block's hash and place.
|
||||
hashes = append(hashes, hash)
|
||||
listed = append(listed, listedBlock{block: b, hash: hash})
|
||||
} else if !r.fetch(b, hash) {
|
||||
return r.ctx.Err()
|
||||
}
|
||||
if index == last {
|
||||
return nil
|
||||
}
|
||||
index++
|
||||
}
|
||||
}
|
||||
if r.sizes != nil && index <= len(r.sizes) {
|
||||
return fmt.Errorf("%w: %d blocks listed of the %d the file records", errIntegrity, index-1, len(r.sizes))
|
||||
}
|
||||
if !verify {
|
||||
return nil
|
||||
}
|
||||
if err := r.verifyManifest(hashes, manifest, signer); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, lb := range listed {
|
||||
if !r.fetch(lb.block, lb.hash) {
|
||||
return r.ctx.Err()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// decodeBlockHash returns the base64 SHA-256 hash.
|
||||
func decodeBlockHash(b64 string) ([]byte, error) {
|
||||
hash, err := base64.StdEncoding.DecodeString(b64)
|
||||
if err == nil && len(hash) != sha256.Size {
|
||||
err = fmt.Errorf("%d bytes long", len(hash))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bad hash: %w", err)
|
||||
}
|
||||
return hash, nil
|
||||
}
|
||||
|
||||
// verifyManifest checks the manifest signature of the file's block
|
||||
// hashes against the keys of signer, or the node key if it is empty.
|
||||
func (r *blockReader) verifyManifest(hashes [][]byte, manifest, signer string) error {
|
||||
verifyKR, err := r.c.verifyKeys(r.ctx, signer, r.n.keys.keyRing)
|
||||
if err != nil {
|
||||
return fmt.Errorf("verify manifest: %w", err)
|
||||
}
|
||||
err = verifyManifest(hashes, manifest, verifyKR, r.c.s.serverTime())
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
err = fmt.Errorf("%w: %w", errIntegrity, err)
|
||||
if !r.c.opt.IgnoreSignatureErrors {
|
||||
return fmt.Errorf("%w: %s", err, signatureHint)
|
||||
}
|
||||
fs.Errorf(nil, "protondrive: %q (ID %s): %v", r.n.Name, r.n.ID, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
// fetch queues block b, whose SHA-256 is hash, for the reader and
|
||||
// fetches it in the background. It returns false if the download is
|
||||
// cancelled first.
|
||||
func (r *blockReader) fetch(b api.Block, hash []byte) bool {
|
||||
slot := make(chan blockResult, 1)
|
||||
select {
|
||||
case r.slots <- slot:
|
||||
case <-r.ctx.Done():
|
||||
return false
|
||||
}
|
||||
r.wg.Add(1)
|
||||
go func() {
|
||||
defer r.wg.Done()
|
||||
slot <- r.fetchBlock(b, hash)
|
||||
}()
|
||||
return true
|
||||
}
|
||||
|
||||
// fetchBlock downloads and decrypts block b, whose SHA-256 is hash,
|
||||
// requesting a new URL after a 404 (once) or a hash mismatch.
|
||||
func (r *blockReader) fetchBlock(b api.Block, hash []byte) blockResult {
|
||||
index := b.Index
|
||||
refetched := false
|
||||
for attempt := 1; ; {
|
||||
plain, err := r.fetchOnce(b, hash)
|
||||
var apiErr *api.Error
|
||||
switch {
|
||||
case err == nil:
|
||||
return blockResult{index: index, hash: hash, plain: plain}
|
||||
case r.ctx.Err() != nil:
|
||||
return blockResult{index: index, err: r.ctx.Err()}
|
||||
case errors.As(err, &apiErr) && apiErr.Status == http.StatusNotFound && !refetched:
|
||||
// The URL may have expired.
|
||||
refetched = true
|
||||
case errors.Is(err, errBlockHash) && attempt < blockAttempts:
|
||||
// The bytes may have been damaged on the way.
|
||||
attempt++
|
||||
default:
|
||||
return blockResult{index: index, err: fmt.Errorf("block %d: %w", index, err)}
|
||||
}
|
||||
fs.Debugf(nil, "protondrive: %q (ID %s): block %d: %v: retrying with a new URL", r.n.Name, r.n.ID, index, err)
|
||||
// hash stays the listed one, which the manifest covers.
|
||||
if b, err = r.c.blockURL(r.ctx, r.n, index); err != nil {
|
||||
return blockResult{index: index, err: fmt.Errorf("block %d: %w", index, err)}
|
||||
}
|
||||
if relisted, err := decodeBlockHash(b.Hash); err != nil || !bytes.Equal(relisted, hash) {
|
||||
return blockResult{index: index, err: fmt.Errorf("block %d: %w: listed again with another hash", index, errIntegrity)}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// fetchOnce downloads block b and checks it against hash and its
|
||||
// claimed size, returning it decrypted.
|
||||
func (r *blockReader) fetchOnce(b api.Block, hash []byte) (*pool.RW, error) {
|
||||
enc, sum, err := r.c.downloadBlock(r.ctx, b.BareURL, b.Token)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = enc.Close() }()
|
||||
if !bytes.Equal(sum, hash) {
|
||||
return nil, errBlockHash
|
||||
}
|
||||
dec, err := blockDecrypter(enc, r.sk)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plain := pool.NewRW(pool.Global())
|
||||
size, err := plain.ReadFrom(io.LimitReader(dec, maxBlockSize+1))
|
||||
switch {
|
||||
case err != nil:
|
||||
err = fmt.Errorf("decrypt block: %w", err)
|
||||
case size > maxBlockSize:
|
||||
err = fmt.Errorf("block decrypts to more than %d bytes", maxBlockSize)
|
||||
case r.sizes != nil && size != r.sizes[b.Index-1]:
|
||||
err = fmt.Errorf("%w: block is %d bytes, the file records %d", errIntegrity, size, r.sizes[b.Index-1])
|
||||
}
|
||||
if err != nil {
|
||||
_ = plain.Close()
|
||||
return nil, err
|
||||
}
|
||||
return plain, nil
|
||||
}
|
||||
|
||||
// Read reads the range of the file.
|
||||
func (r *blockReader) Read(p []byte) (int, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
for {
|
||||
switch {
|
||||
case r.closed:
|
||||
return 0, errDownloadClosed
|
||||
case r.err != nil:
|
||||
return 0, r.err
|
||||
case r.remaining == 0:
|
||||
return 0, io.EOF
|
||||
}
|
||||
if r.cur == nil {
|
||||
if err := r.next(); err != nil {
|
||||
r.err = err
|
||||
r.cancel()
|
||||
}
|
||||
continue
|
||||
}
|
||||
if r.remaining > 0 && int64(len(p)) > r.remaining {
|
||||
p = p[:r.remaining]
|
||||
}
|
||||
n, err := r.cur.Read(p)
|
||||
if n > 0 {
|
||||
if r.remaining > 0 {
|
||||
r.remaining -= int64(n)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
if err != nil && err != io.EOF {
|
||||
r.err = err
|
||||
continue
|
||||
}
|
||||
_ = r.cur.Close()
|
||||
r.cur = nil
|
||||
}
|
||||
}
|
||||
|
||||
// next makes the next block holding bytes of the range current,
|
||||
// returning io.EOF at the end.
|
||||
func (r *blockReader) next() error {
|
||||
for {
|
||||
var slot chan blockResult
|
||||
select {
|
||||
case slot = <-r.slots:
|
||||
case <-r.ctx.Done():
|
||||
return r.ctx.Err()
|
||||
}
|
||||
res := <-slot
|
||||
switch {
|
||||
case res.err != nil && r.ctx.Err() != nil:
|
||||
return r.ctx.Err()
|
||||
case res.err != nil:
|
||||
return res.err
|
||||
case res.end:
|
||||
return io.EOF
|
||||
}
|
||||
if r.sizes == nil {
|
||||
if err := r.c.learnt.learn(r.n.RevisionID, res.index, res.hash, res.plain.Size()); err != nil {
|
||||
_ = res.plain.Close()
|
||||
return err
|
||||
}
|
||||
}
|
||||
if size := res.plain.Size(); r.skip > 0 && r.skip >= size {
|
||||
r.skip -= size
|
||||
_ = res.plain.Close()
|
||||
continue
|
||||
}
|
||||
if r.skip > 0 {
|
||||
if _, err := res.plain.Seek(r.skip, io.SeekStart); err != nil {
|
||||
_ = res.plain.Close()
|
||||
return err
|
||||
}
|
||||
r.skip = 0
|
||||
}
|
||||
r.cur = res.plain
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Close stops the download and frees its buffers. It may be called
|
||||
// more than once.
|
||||
func (r *blockReader) Close() error {
|
||||
r.cancel()
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.closed {
|
||||
return nil
|
||||
}
|
||||
r.closed = true
|
||||
r.wg.Wait()
|
||||
if r.cur != nil {
|
||||
_ = r.cur.Close()
|
||||
r.cur = nil
|
||||
}
|
||||
// Every queued slot has its result now.
|
||||
for {
|
||||
select {
|
||||
case slot := <-r.slots:
|
||||
if res := <-slot; res.plain != nil {
|
||||
_ = res.plain.Close()
|
||||
}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// check interfaces
|
||||
var _ io.ReadCloser = (*blockReader)(nil)
|
||||
|
||||
const (
|
||||
// learntRevisions and learntBlocks bound the block sizes a client
|
||||
// learns: of at most this many revisions, and this many blocks in
|
||||
// all, about 2.5 MiB.
|
||||
learntRevisions = 16
|
||||
learntBlocks = 1 << 16
|
||||
)
|
||||
|
||||
// learntBlock is the plaintext size of a block read, with its hash.
|
||||
type learntBlock struct {
|
||||
hash [sha256.Size]byte
|
||||
size int64
|
||||
}
|
||||
|
||||
// blockSizeCache holds the plaintext sizes of the blocks, from the
|
||||
// first on, read of revisions which don't record their block sizes, so
|
||||
// a later ranged read, as each chunk of a multi-thread copy is, can
|
||||
// start at the block holding its offset. The zero value is empty.
|
||||
type blockSizeCache struct {
|
||||
mu sync.Mutex
|
||||
revs map[string][]learntBlock // by revision ID
|
||||
order []string // revision IDs, oldest first
|
||||
blocks int // learnt in all
|
||||
}
|
||||
|
||||
// locate returns where a read of limit bytes (to the end if negative)
|
||||
// from offset of revision id starts, as for blockReader: the first
|
||||
// block, the bytes of it to skip, the last block or -1 if not known,
|
||||
// and the hash the first block had when its size was learnt, or nil if
|
||||
// it wasn't.
|
||||
func (b *blockSizeCache) locate(id string, offset, limit int64) (first int, skip int64, last int, startHash []byte) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
blocks := b.revs[id]
|
||||
first, skip, last = len(blocks)+1, offset, -1
|
||||
for i, lb := range blocks {
|
||||
if skip < lb.size {
|
||||
first = i + 1
|
||||
startHash = slices.Clone(lb.hash[:])
|
||||
break
|
||||
}
|
||||
skip -= lb.size
|
||||
}
|
||||
if first > len(blocks) || limit <= 0 {
|
||||
return first, skip, last, startHash
|
||||
}
|
||||
end := skip + limit - 1
|
||||
for i := first - 1; i < len(blocks); i++ {
|
||||
if end < blocks[i].size {
|
||||
return first, skip, i + 1, startHash
|
||||
}
|
||||
end -= blocks[i].size
|
||||
}
|
||||
return first, skip, last, startHash
|
||||
}
|
||||
|
||||
// learn records that block index of revision id, whose SHA-256 is hash,
|
||||
// is size bytes. A block learnt before with another hash or size is
|
||||
// errIntegrity: the server listed two blocks for one place.
|
||||
func (b *blockSizeCache) learn(id string, index int, hash []byte, size int64) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
blocks, ok := b.revs[id]
|
||||
if index <= len(blocks) {
|
||||
if lb := blocks[index-1]; !bytes.Equal(lb.hash[:], hash) || lb.size != size {
|
||||
return fmt.Errorf("%w: block %d listed with another hash than when it was read before", errIntegrity, index)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if index != len(blocks)+1 || len(hash) != sha256.Size {
|
||||
return nil
|
||||
}
|
||||
if !ok {
|
||||
if b.revs == nil {
|
||||
b.revs = make(map[string][]learntBlock)
|
||||
}
|
||||
for len(b.order) >= learntRevisions {
|
||||
b.forgetOldest()
|
||||
}
|
||||
b.order = append(b.order, id)
|
||||
}
|
||||
for b.blocks >= learntBlocks && len(b.order) > 1 && b.order[0] != id {
|
||||
b.forgetOldest()
|
||||
}
|
||||
if b.blocks >= learntBlocks {
|
||||
return nil
|
||||
}
|
||||
lb := learntBlock{size: size}
|
||||
copy(lb.hash[:], hash)
|
||||
b.revs[id] = append(blocks, lb)
|
||||
b.blocks++
|
||||
return nil
|
||||
}
|
||||
|
||||
// forgetOldest drops the revision learnt first.
|
||||
func (b *blockSizeCache) forgetOldest() {
|
||||
b.blocks -= len(b.revs[b.order[0]])
|
||||
delete(b.revs, b.order[0])
|
||||
b.order = b.order[1:]
|
||||
}
|
||||
@@ -0,0 +1,852 @@
|
||||
package protondrive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"math"
|
||||
"math/rand/v2"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ProtonMail/go-crypto/openpgp/packet"
|
||||
"github.com/rclone/rclone/backend/protondrive/protontest"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
storagePattern = "GET /storage/blocks/{id}"
|
||||
revisionPattern = "GET /drive/v2/volumes/{volumeID}/files/{linkID}/revisions/{revisionID}"
|
||||
|
||||
// bigSize is the size of a file of three blocks, the last partial.
|
||||
bigSize = 9<<20 + 123
|
||||
)
|
||||
|
||||
// testContent returns n bytes of reproducible random content.
|
||||
func testContent(n int) []byte {
|
||||
r := rand.New(rand.NewPCG(uint64(n), 1))
|
||||
b := make([]byte, n)
|
||||
for i := range b {
|
||||
b[i] = byte(r.Uint32())
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// seedFile seeds file name with content in the root of u and returns
|
||||
// its node as c reads it.
|
||||
func seedFile(t *testing.T, c *client, u *protontest.User, name string, content []byte) *node {
|
||||
t.Helper()
|
||||
return getTestNode(t, c, u.SeedFile(u.RootID(), name, content, time.Now()))
|
||||
}
|
||||
|
||||
// getTestNode returns node id as c reads it.
|
||||
func getTestNode(t *testing.T, c *client, id string) *node {
|
||||
t.Helper()
|
||||
n, err := c.getNode(context.Background(), id)
|
||||
require.NoError(t, err)
|
||||
return n
|
||||
}
|
||||
|
||||
// withoutBlockSizes returns a copy of n whose extended attributes
|
||||
// don't record its block sizes, as some old clients wrote them.
|
||||
func withoutBlockSizes(n *node) *node {
|
||||
cp := *n
|
||||
cp.X = nil
|
||||
return &cp
|
||||
}
|
||||
|
||||
// readRange opens n at offset for limit bytes and reads it to the end.
|
||||
func readRange(t *testing.T, c *client, n *node, offset, limit int64) ([]byte, error) {
|
||||
t.Helper()
|
||||
rc, err := c.open(context.Background(), n, offset, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := io.ReadAll(rc)
|
||||
require.NoError(t, rc.Close())
|
||||
return data, err
|
||||
}
|
||||
|
||||
// roundTripFunc is an http.RoundTripper calling itself.
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
// RoundTrip calls f.
|
||||
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
return f(r)
|
||||
}
|
||||
|
||||
// newTestClientTransport returns a client for a new session of u whose
|
||||
// requests go through the transport wrap returns.
|
||||
func newTestClientTransport(t *testing.T, srv *protontest.Server, u *protontest.User, opt clientOptions, wrap func(next http.RoundTripper) http.RoundTripper) *client {
|
||||
t.Helper()
|
||||
base := newAccountSession(srv, u)
|
||||
s := newSession(&http.Client{Transport: wrap(srv.Client().Transport)}, srv.URL(), testAppVersion, base.store)
|
||||
c, err := newClient(context.Background(), s, opt)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(c.close)
|
||||
return c
|
||||
}
|
||||
|
||||
// leakBaseline returns the goroutine count and pool buffers in use with
|
||||
// no idle connections, for requireNoLeaks.
|
||||
func leakBaseline(srv *protontest.Server) (goroutines, buffers int) {
|
||||
srv.Client().CloseIdleConnections()
|
||||
return runtime.NumGoroutine(), pool.Global().InUse()
|
||||
}
|
||||
|
||||
// requireNoLeaks fails the test unless the goroutine count returns to
|
||||
// goroutines within 2s and the pool buffers in use are back to buffers.
|
||||
func requireNoLeaks(t *testing.T, srv *protontest.Server, goroutines, buffers int) {
|
||||
t.Helper()
|
||||
require.Eventually(t, func() bool {
|
||||
srv.Client().CloseIdleConnections()
|
||||
return runtime.NumGoroutine() <= goroutines
|
||||
}, 2*time.Second, 20*time.Millisecond, "goroutines left running")
|
||||
assert.Equal(t, buffers, pool.Global().InUse(), "pool buffers not returned")
|
||||
}
|
||||
|
||||
func TestDownloadSizes(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, _, u := newTestClient(t)
|
||||
for _, size := range []int{0, 1, 4<<20 - 1, 4 << 20, 4<<20 + 1, 8 << 20, bigSize} {
|
||||
t.Run(strconv.Itoa(size), func(t *testing.T) {
|
||||
content := testContent(size)
|
||||
n := seedFile(t, c, u, "f"+strconv.Itoa(size), content)
|
||||
data, err := readRange(t, c, n, 0, -1)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, bytes.Equal(content, data), "content differs")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadRanges(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newTestClient(t)
|
||||
content := testContent(bigSize)
|
||||
claimed := seedFile(t, c, u, "big", content)
|
||||
unrecorded := withoutBlockSizes(claimed)
|
||||
for _, tc := range []struct {
|
||||
offset, limit int64
|
||||
blocks int // blocks fetched with the block sizes claimed
|
||||
}{
|
||||
{0, -1, 3},
|
||||
{0, 1, 1},
|
||||
{0, 0, 0},
|
||||
{4<<20 - 1, 2, 2},
|
||||
{4 << 20, -1, 2},
|
||||
{bigSize - 1, -1, 1},
|
||||
{bigSize, -1, 0},
|
||||
{8 << 20, 100, 1},
|
||||
{bigSize - 10, 100, 1},
|
||||
} {
|
||||
want := content[tc.offset:]
|
||||
if tc.limit >= 0 {
|
||||
want = want[:min(tc.limit, int64(len(want)))]
|
||||
}
|
||||
for _, v := range []struct {
|
||||
name string
|
||||
n *node
|
||||
}{{"claimed", claimed}, {"unrecorded", unrecorded}} {
|
||||
t.Run(v.name+"/"+strconv.FormatInt(tc.offset, 10)+"+"+strconv.FormatInt(tc.limit, 10), func(t *testing.T) {
|
||||
srv.ResetRequests()
|
||||
data, err := readRange(t, c, v.n, tc.offset, tc.limit)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, bytes.Equal(want, data), "got %d bytes, want %d", len(data), len(want))
|
||||
fetched := srv.Requests()[storagePattern]
|
||||
if v.n == claimed {
|
||||
assert.Equal(t, tc.blocks, fetched, "blocks fetched")
|
||||
return
|
||||
}
|
||||
// The block sizes learnt by the first, full, read
|
||||
// place the range; blocks past it may be fetched ahead
|
||||
// before the reader is closed.
|
||||
assert.GreaterOrEqual(t, fetched, tc.blocks, "blocks fetched")
|
||||
assert.LessOrEqual(t, fetched, 3, "blocks fetched")
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadRangesUnrecordedBlockSizes(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, _, u := newTestClient(t)
|
||||
const size = 6<<20 + 5
|
||||
content := testContent(size)
|
||||
// Blocks of 1 MiB, which a reader assuming 4 MiB blocks would
|
||||
// place wrongly.
|
||||
n := getTestNode(t, c, u.SeedFileInBlocks(u.RootID(), "small-blocks", content, 1<<20))
|
||||
require.Nil(t, n.X.BlockSizes)
|
||||
for _, tc := range []struct{ offset, limit int64 }{
|
||||
{0, -1},
|
||||
{0, 3 << 20},
|
||||
{10, 100},
|
||||
{1<<20 - 3, 10},
|
||||
{4<<20 + 10, 100},
|
||||
{4<<20 + 10, -1},
|
||||
{2<<20 + 7, 3 << 20},
|
||||
{size - 1, -1},
|
||||
{size, -1},
|
||||
} {
|
||||
t.Run(strconv.FormatInt(tc.offset, 10)+"+"+strconv.FormatInt(tc.limit, 10), func(t *testing.T) {
|
||||
want := content[tc.offset:]
|
||||
if tc.limit >= 0 {
|
||||
want = want[:min(tc.limit, int64(len(want)))]
|
||||
}
|
||||
data, err := readRange(t, c, n, tc.offset, tc.limit)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, bytes.Equal(want, data), "got %d bytes, want %d", len(data), len(want))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadChunksUnrecordedBlockSizes(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newTestClient(t)
|
||||
const (
|
||||
blocks = 32
|
||||
chunk = 4 << 20
|
||||
blockBytes = 1 << 20
|
||||
)
|
||||
content := testContent(blocks * blockBytes)
|
||||
id := u.SeedFileInBlocks(u.RootID(), "chunks", content, blockBytes)
|
||||
n := getTestNode(t, c, id)
|
||||
require.Nil(t, n.X.BlockSizes)
|
||||
|
||||
// A multi-thread copy opens each chunk as a range of its own.
|
||||
srv.ResetRequests()
|
||||
for offset := int64(0); offset < int64(len(content)); offset += chunk {
|
||||
data, err := readRange(t, c, n, offset, chunk)
|
||||
require.NoError(t, err)
|
||||
require.True(t, bytes.Equal(content[offset:offset+chunk], data), "chunk at %d", offset)
|
||||
}
|
||||
fetched := srv.Requests()[storagePattern]
|
||||
t.Logf("%d blocks fetched for %d chunks of a file of %d blocks", fetched, len(content)/chunk, blocks)
|
||||
// Reading each chunk from the first block would fetch about
|
||||
// blocks^2/(2*blocks per chunk) = 128; each chunk may fetch up to
|
||||
// downloadAhead blocks past its end.
|
||||
assert.LessOrEqual(t, fetched, blocks+len(content)/chunk*downloadAhead, "chunks read from the first block")
|
||||
|
||||
// Once the sizes are known, a range fetches only its own blocks.
|
||||
srv.ResetRequests()
|
||||
data, err := readRange(t, c, n, 10*blockBytes+5, 2*blockBytes)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content[10*blockBytes+5:12*blockBytes+5], data)
|
||||
assert.Equal(t, 3, srv.Requests()[storagePattern], "blocks fetched")
|
||||
|
||||
// A block list which changed since then is an integrity error.
|
||||
u.Corrupt(id, "swap-blocks")
|
||||
_, err = readRange(t, c, n, 5, 10)
|
||||
require.ErrorIs(t, err, errIntegrity)
|
||||
assert.Contains(t, err.Error(), "listed with another hash")
|
||||
}
|
||||
|
||||
func TestDownloadLegacyFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, _, u := newTestClient(t)
|
||||
content := testContent(5<<20 + 7)
|
||||
n := getTestNode(t, c, u.SeedLegacyFile(u.RootID(), "legacy", content))
|
||||
data, err := readRange(t, c, n, 0, -1)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, bytes.Equal(content, data), "content differs")
|
||||
data, err = readRange(t, c, n, 4<<20+3, 10)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content[4<<20+3:4<<20+13], data)
|
||||
}
|
||||
|
||||
func TestDownloadManifestSigners(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newTestClient(t)
|
||||
other := srv.AddUser("other", "pass")
|
||||
content := testContent(1000)
|
||||
for name, id := range map[string]string{
|
||||
"other user": u.SeedFileSignedBy(other, u.RootID(), "other", content, time.Now()),
|
||||
"anonymous": u.SeedAnonymousFile(u.RootID(), "anonymous", content, time.Now()),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
data, err := readRange(t, c, getTestNode(t, c, id), 0, -1)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content, data)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadContentKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
const second = "second@proton.test"
|
||||
u.AddAddress(second)
|
||||
c := newTestClientFor(t, srv, u, clientOptions{})
|
||||
ignoring := newTestClientFor(t, srv, u, clientOptions{IgnoreSignatureErrors: true})
|
||||
before2023 := time.Date(2022, 6, 1, 0, 0, 0, 0, time.UTC)
|
||||
content := testContent(1000)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
signer string
|
||||
created time.Time
|
||||
ok bool
|
||||
}{
|
||||
{"unsigned", "", time.Now(), false},
|
||||
{"unsigned before 2023", "", before2023, false},
|
||||
{"signer address", u.Email(), time.Now(), true},
|
||||
{"other address before 2023", second, before2023, true},
|
||||
{"other address after 2023", second, time.Now(), false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
id := u.SeedFileContentKeySignedBy(tc.signer, tc.created, u.RootID(), tc.name, content)
|
||||
data, err := readRange(t, c, getTestNode(t, c, id), 0, -1)
|
||||
if !tc.ok {
|
||||
require.ErrorIs(t, err, errIntegrity)
|
||||
assert.Contains(t, err.Error(), "ignore_signature_errors")
|
||||
data, err = readRange(t, ignoring, getTestNode(t, ignoring, id), 0, -1)
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content, data)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadNodeKeySignature(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newTestClient(t)
|
||||
content := testContent(1000)
|
||||
id := u.SeedFile(u.RootID(), "f", content, time.Now())
|
||||
u.Corrupt(id, "passphrase-signature")
|
||||
n := getTestNode(t, c, id)
|
||||
|
||||
data, err := readRange(t, c, n, 0, -1)
|
||||
require.ErrorIs(t, err, errIntegrity)
|
||||
assert.Contains(t, err.Error(), "ignore_signature_errors")
|
||||
assert.Empty(t, data)
|
||||
_, err = readRange(t, c, n, 10, 10)
|
||||
require.ErrorIs(t, err, errIntegrity, "a ranged read")
|
||||
|
||||
ignoring := newTestClientFor(t, srv, u, clientOptions{IgnoreSignatureErrors: true})
|
||||
data, err = readRange(t, ignoring, getTestNode(t, ignoring, id), 0, -1)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content, data)
|
||||
}
|
||||
|
||||
func TestDownloadBlockHashMismatch(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newTestClient(t)
|
||||
n := seedFile(t, c, u, "f", testContent(1<<20))
|
||||
u.Corrupt(n.ID, "block-hash")
|
||||
srv.ResetRequests()
|
||||
data, err := readRange(t, c, n, 0, -1)
|
||||
require.ErrorIs(t, err, errBlockHash)
|
||||
assert.Empty(t, data)
|
||||
assert.Equal(t, 3, srv.Requests()[storagePattern], "block attempts")
|
||||
// The listing page, the empty page ending the list and a new URL
|
||||
// for each retry.
|
||||
assert.Equal(t, 4, srv.Requests()[revisionPattern])
|
||||
}
|
||||
|
||||
func TestDownloadSwappedBlocks(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, _, u := newTestClient(t)
|
||||
n := seedFile(t, c, u, "f", testContent(bigSize))
|
||||
u.Corrupt(n.ID, "swap-blocks")
|
||||
_, err := readRange(t, c, n, 0, -1)
|
||||
require.ErrorIs(t, err, errIntegrity)
|
||||
}
|
||||
|
||||
func TestDownloadTruncatedBlocks(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, _, u := newTestClient(t)
|
||||
n := seedFile(t, c, u, "f", testContent(bigSize))
|
||||
u.Corrupt(n.ID, "truncate-blocks")
|
||||
_, err := readRange(t, c, n, 0, -1)
|
||||
require.ErrorIs(t, err, errIntegrity)
|
||||
// Without block sizes, only the manifest shows the loss.
|
||||
_, err = readRange(t, c, withoutBlockSizes(n), 0, -1)
|
||||
require.ErrorIs(t, err, errIntegrity)
|
||||
}
|
||||
|
||||
func TestDownloadBadManifest(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newTestClient(t)
|
||||
content := testContent(bigSize)
|
||||
n := seedFile(t, c, u, "f", content)
|
||||
u.Corrupt(n.ID, "manifest")
|
||||
|
||||
data, err := readRange(t, c, n, 0, -1)
|
||||
require.ErrorIs(t, err, errIntegrity)
|
||||
assert.Contains(t, err.Error(), "ignore_signature_errors")
|
||||
assert.Empty(t, data, "no byte comes before the verdict")
|
||||
|
||||
// A ranged read can't check the manifest.
|
||||
data, err = readRange(t, c, n, 0, 100)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content[:100], data)
|
||||
|
||||
ignoring := newTestClientFor(t, srv, u, clientOptions{IgnoreSignatureErrors: true})
|
||||
data, err = readRange(t, ignoring, getTestNode(t, ignoring, n.ID), 0, -1)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, bytes.Equal(content, data), "content differs")
|
||||
}
|
||||
|
||||
func TestDownloadEmptyFileBadManifest(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, _, u := newTestClient(t)
|
||||
n := seedFile(t, c, u, "empty", nil)
|
||||
u.Corrupt(n.ID, "manifest")
|
||||
_, err := readRange(t, c, n, 0, -1)
|
||||
require.ErrorIs(t, err, errIntegrity)
|
||||
}
|
||||
|
||||
func TestDownloadStorage404Refetch(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newTestClient(t)
|
||||
content := testContent(1 << 20)
|
||||
n := seedFile(t, c, u, "f", content)
|
||||
|
||||
srv.Inject(protontest.Fault{Pattern: "/storage/blocks/{id}", Status: http.StatusNotFound, Times: 1})
|
||||
srv.ResetRequests()
|
||||
data, err := readRange(t, c, n, 0, -1)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content, data)
|
||||
assert.Equal(t, 2, srv.Requests()[storagePattern])
|
||||
// The listing page, the new URL and the empty page ending the list.
|
||||
assert.Equal(t, 3, srv.Requests()[revisionPattern])
|
||||
|
||||
// The URL is requested again only once.
|
||||
srv.Inject(protontest.Fault{Pattern: "/storage/blocks/{id}", Status: http.StatusNotFound})
|
||||
srv.ResetRequests()
|
||||
_, err = readRange(t, c, n, 0, -1)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, 2, srv.Requests()[storagePattern])
|
||||
}
|
||||
|
||||
func TestDownloadRefetchOtherHash(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newTestClient(t)
|
||||
n := seedFile(t, c, u, "f", testContent(1<<20))
|
||||
|
||||
// The block's URL expires, and the server lists it again with
|
||||
// another block, which the checked manifest doesn't cover.
|
||||
srv.Inject(protontest.Fault{Pattern: "/storage/blocks/{id}", Status: http.StatusNotFound, Times: 1, Delay: 300 * time.Millisecond})
|
||||
done := make(chan struct{})
|
||||
var data []byte
|
||||
var err error
|
||||
go func() {
|
||||
defer close(done)
|
||||
data, err = readRange(t, c, n, 0, -1)
|
||||
}()
|
||||
require.Eventually(t, func() bool { return srv.Requests()[storagePattern] == 1 }, 5*time.Second, 5*time.Millisecond)
|
||||
u.Corrupt(n.ID, "foreign-block")
|
||||
<-done
|
||||
require.ErrorIs(t, err, errIntegrity)
|
||||
assert.Contains(t, err.Error(), "listed again with another hash")
|
||||
assert.Empty(t, data)
|
||||
}
|
||||
|
||||
func TestDownloadRetries5xx(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newTestClient(t)
|
||||
content := testContent(1 << 20)
|
||||
n := seedFile(t, c, u, "f", content)
|
||||
srv.Inject(protontest.Fault{Pattern: "/storage/blocks/{id}", Status: http.StatusServiceUnavailable, Times: 2})
|
||||
srv.Inject(protontest.Fault{Pattern: "/drive/v2/volumes/{volumeID}/files/{linkID}/revisions/{revisionID}", Status: http.StatusBadGateway, Times: 1})
|
||||
srv.ResetRequests()
|
||||
data, err := readRange(t, c, n, 0, -1)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content, data)
|
||||
assert.Equal(t, 3, srv.Requests()[storagePattern])
|
||||
}
|
||||
|
||||
func TestDownloadStorageRetryAfter(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newTestClient(t)
|
||||
content := testContent(1000)
|
||||
n := seedFile(t, c, u, "f", content)
|
||||
srv.Inject(protontest.Fault{Pattern: "/storage/blocks/{id}", Status: http.StatusTooManyRequests, RetryAfter: 1, Times: 1})
|
||||
start := time.Now()
|
||||
data, err := readRange(t, c, n, 0, -1)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content, data)
|
||||
assert.GreaterOrEqual(t, time.Since(start), time.Second)
|
||||
}
|
||||
|
||||
func TestDownloadStorageHeaders(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
var (
|
||||
mu sync.Mutex
|
||||
headers []http.Header
|
||||
)
|
||||
c := newTestClientTransport(t, srv, u, clientOptions{}, func(next http.RoundTripper) http.RoundTripper {
|
||||
return roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if strings.HasPrefix(r.URL.Path, "/storage/") {
|
||||
mu.Lock()
|
||||
headers = append(headers, r.Header.Clone())
|
||||
mu.Unlock()
|
||||
}
|
||||
return next.RoundTrip(r)
|
||||
})
|
||||
})
|
||||
n := seedFile(t, c, u, "f", testContent(4<<20+1))
|
||||
_, err := readRange(t, c, n, 0, -1)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, headers, 2)
|
||||
for _, h := range headers {
|
||||
assert.Empty(t, h.Get("Authorization"))
|
||||
assert.Empty(t, h.Get("x-pm-uid"))
|
||||
assert.NotEmpty(t, h.Get("pm-storage-token"))
|
||||
assert.Equal(t, testAppVersion, h.Get("x-pm-appversion"))
|
||||
}
|
||||
|
||||
// Block URLs on another host, as the real storage host is, get the
|
||||
// same headers. The other host passes requests on to the fake.
|
||||
var other []http.Header
|
||||
storage := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
other = append(other, r.Header.Clone())
|
||||
mu.Unlock()
|
||||
req, err := http.NewRequestWithContext(r.Context(), "GET", srv.URL()+r.URL.Path, nil)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
for _, k := range []string{"pm-storage-token", "x-pm-appversion"} {
|
||||
req.Header.Set(k, r.Header.Get(k))
|
||||
}
|
||||
resp, err := srv.Client().Do(req)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
_, _ = io.Copy(w, resp.Body)
|
||||
}))
|
||||
t.Cleanup(storage.Close)
|
||||
require.NotEqual(t, srv.URL(), storage.URL)
|
||||
moved := newTestClientTransport(t, srv, u, clientOptions{}, func(next http.RoundTripper) http.RoundTripper {
|
||||
return roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
resp, err := next.RoundTrip(r)
|
||||
if err != nil || !strings.Contains(r.URL.Path, "/revisions/") {
|
||||
return resp, err
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body = bytes.ReplaceAll(body, []byte(srv.URL()+"/storage/"), []byte(storage.URL+"/storage/"))
|
||||
resp.Body = io.NopCloser(bytes.NewReader(body))
|
||||
resp.ContentLength = int64(len(body))
|
||||
resp.Header.Del("Content-Length")
|
||||
return resp, nil
|
||||
})
|
||||
})
|
||||
_, err = readRange(t, moved, getTestNode(t, moved, n.ID), 0, -1)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, other, 2)
|
||||
for _, h := range other {
|
||||
assert.Empty(t, h.Get("Authorization"))
|
||||
assert.Empty(t, h.Get("x-pm-uid"))
|
||||
assert.NotEmpty(t, h.Get("pm-storage-token"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadBlockTooLarge(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
var fetches atomic.Int32
|
||||
c := newTestClientTransport(t, srv, u, clientOptions{}, func(next http.RoundTripper) http.RoundTripper {
|
||||
return roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if !strings.HasPrefix(r.URL.Path, "/storage/") {
|
||||
return next.RoundTrip(r)
|
||||
}
|
||||
fetches.Add(1)
|
||||
// An endless body of unknown length.
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: http.Header{},
|
||||
Body: io.NopCloser(zeroReader{}),
|
||||
ContentLength: -1,
|
||||
Request: r,
|
||||
}, nil
|
||||
})
|
||||
})
|
||||
n := seedFile(t, c, u, "f", testContent(1000))
|
||||
_, err := readRange(t, c, n, 0, -1)
|
||||
require.ErrorContains(t, err, "larger than")
|
||||
assert.NotErrorIs(t, err, errBlockHash)
|
||||
assert.Equal(t, int32(1), fetches.Load(), "an oversized block is not fetched again")
|
||||
}
|
||||
|
||||
// zeroReader reads zeros forever.
|
||||
type zeroReader struct{}
|
||||
|
||||
// Read fills p with zeros.
|
||||
func (zeroReader) Read(p []byte) (int, error) {
|
||||
clear(p)
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// endlessBlockList returns a transport wrapper whose revision pages are
|
||||
// all full of copies of the file's first block, renumbered, counting
|
||||
// the pages in pages.
|
||||
func endlessBlockList(pages *atomic.Int32) func(next http.RoundTripper) http.RoundTripper {
|
||||
var (
|
||||
mu sync.Mutex
|
||||
first map[string]any
|
||||
)
|
||||
return func(next http.RoundTripper) http.RoundTripper {
|
||||
return roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
resp, err := next.RoundTrip(r)
|
||||
if err != nil || !strings.Contains(r.URL.Path, "/revisions/") {
|
||||
return resp, err
|
||||
}
|
||||
pages.Add(1)
|
||||
var body map[string]any
|
||||
err = json.NewDecoder(resp.Body).Decode(&body)
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rev := body["Revision"].(map[string]any)
|
||||
mu.Lock()
|
||||
if blocks := rev["Blocks"].([]any); first == nil && len(blocks) > 0 {
|
||||
first = blocks[0].(map[string]any)
|
||||
}
|
||||
from, _ := strconv.Atoi(r.URL.Query().Get("FromBlockIndex"))
|
||||
size, _ := strconv.Atoi(r.URL.Query().Get("PageSize"))
|
||||
var blocks []any
|
||||
for i := range size {
|
||||
b := make(map[string]any, len(first))
|
||||
for k, v := range first {
|
||||
b[k] = v
|
||||
}
|
||||
b["Index"] = from + i
|
||||
blocks = append(blocks, b)
|
||||
}
|
||||
mu.Unlock()
|
||||
rev["Blocks"] = blocks
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp.Body = io.NopCloser(bytes.NewReader(data))
|
||||
resp.ContentLength = int64(len(data))
|
||||
resp.Header.Del("Content-Length")
|
||||
return resp, nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadEndlessBlockList(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
var pages atomic.Int32
|
||||
c := newTestClientTransport(t, srv, u, clientOptions{EnableCaching: true}, endlessBlockList(&pages))
|
||||
n := seedFile(t, c, u, "f", testContent(4<<20+1))
|
||||
for name, n := range map[string]*node{"claimed": n, "unrecorded": withoutBlockSizes(n)} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
pages.Store(0)
|
||||
data, err := readRange(t, c, n, 0, -1)
|
||||
require.ErrorIs(t, err, errIntegrity)
|
||||
assert.LessOrEqual(t, len(data), 5*(4<<20))
|
||||
assert.LessOrEqual(t, pages.Load(), int32(2))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadBlockCap(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
var pages atomic.Int32
|
||||
c := newTestClientTransport(t, srv, u, clientOptions{EnableCaching: true}, endlessBlockList(&pages))
|
||||
c.maxFileBlocks = 45
|
||||
n := seedFile(t, c, u, "f", testContent(4<<20+1))
|
||||
|
||||
// The encrypted size, which nothing signs, claims room for many
|
||||
// more blocks than the cap.
|
||||
huge := withoutBlockSizes(n)
|
||||
huge.EncryptedSize = 1 << 50
|
||||
data, err := readRange(t, c, huge, 0, -1)
|
||||
require.ErrorIs(t, err, errIntegrity)
|
||||
assert.Contains(t, err.Error(), "more than 45 blocks")
|
||||
assert.Empty(t, data)
|
||||
assert.LessOrEqual(t, pages.Load(), int32(3), "listed past the cap")
|
||||
|
||||
// Recorded block sizes beyond the cap fail before any request.
|
||||
claims := *n
|
||||
x := *n.X
|
||||
x.BlockSizes = make([]int64, 46)
|
||||
claims.X = &x
|
||||
pages.Store(0)
|
||||
_, err = readRange(t, c, &claims, 0, -1)
|
||||
require.ErrorIs(t, err, errIntegrity)
|
||||
assert.Zero(t, pages.Load())
|
||||
}
|
||||
|
||||
func TestDownloadCloseEarly(t *testing.T) {
|
||||
c, srv, u := newTestClient(t)
|
||||
n := seedFile(t, c, u, "f", testContent(8<<20))
|
||||
goroutines, buffers := leakBaseline(srv)
|
||||
|
||||
rc, err := c.open(context.Background(), n, 0, -1)
|
||||
require.NoError(t, err)
|
||||
buf := make([]byte, 10)
|
||||
_, err = io.ReadFull(rc, buf)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
require.NoError(t, rc.Close())
|
||||
_, err = rc.Read(buf)
|
||||
require.Error(t, err)
|
||||
requireNoLeaks(t, srv, goroutines, buffers)
|
||||
}
|
||||
|
||||
func TestDownloadContextCancel(t *testing.T) {
|
||||
c, srv, u := newTestClient(t)
|
||||
n := seedFile(t, c, u, "f", testContent(8<<20))
|
||||
// One of the blocks takes far longer than the test.
|
||||
srv.Inject(protontest.Fault{Pattern: "/storage/blocks/{id}", Skip: 1, Times: 1, Delay: time.Minute})
|
||||
goroutines, buffers := leakBaseline(srv)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
rc, err := c.open(ctx, n, 0, -1)
|
||||
require.NoError(t, err)
|
||||
time.AfterFunc(200*time.Millisecond, cancel)
|
||||
start := time.Now()
|
||||
data, err := io.ReadAll(rc)
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
assert.Less(t, len(data), 8<<20)
|
||||
assert.Less(t, time.Since(start), 10*time.Second)
|
||||
require.NoError(t, rc.Close())
|
||||
requireNoLeaks(t, srv, goroutines, buffers)
|
||||
}
|
||||
|
||||
func TestDownloadHostileBlockSizes(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, _, u := newTestClient(t)
|
||||
small := testContent(1000)
|
||||
big := testContent(4<<20 + 1)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
content []byte
|
||||
sizes []int64
|
||||
}{
|
||||
{"overflowing", small, []int64{math.MaxInt64, 1}},
|
||||
{"above the cap", small, []int64{maxBlockSize + 1}},
|
||||
{"sum too small", big, []int64{1000}},
|
||||
{"too few blocks", big, []int64{4 << 20}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
n := getTestNode(t, c, u.SeedFileClaimingBlockSizes(u.RootID(), tc.name, tc.content, tc.sizes))
|
||||
require.Equal(t, tc.sizes, n.X.BlockSizes)
|
||||
_, err := readRange(t, c, n, 0, -1)
|
||||
require.ErrorIs(t, err, errIntegrity)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// frozenFileDir holds files the old library (go-proton-api v1.0.4 and
|
||||
// Proton-API-Bridge v1.0.5) wrote, as the server stored them, one
|
||||
// directory per format: "legacy" (v3 PKESK content key, SEIPDv1 blocks)
|
||||
// and "v6" (v6 PKESK content key, SEIPDv2 blocks). The old library only
|
||||
// writes v6 content as a revision of a file another client created, so
|
||||
// in "v6" the node key and content key packet with its signature are
|
||||
// SDK-style stand-ins made with plain gopenpgp.
|
||||
const frozenFileDir = "testdata/oldlib/file"
|
||||
|
||||
// frozenFile is a file a client holding Keys stored, with what it
|
||||
// encrypted. In a directory, file.json holds it but Stored, which is
|
||||
// in link.json, revision.json and block-1.bin, block-2.bin...
|
||||
type frozenFile struct {
|
||||
Comment string
|
||||
Email string // address which signed everything, the owner's primary address
|
||||
Keys protontest.FixedKeys
|
||||
Name string
|
||||
ModTime string // as the writer recorded it in the extended attributes
|
||||
Content []byte
|
||||
Stored protontest.StoredFile `json:"-"`
|
||||
}
|
||||
|
||||
// frozenFileParts returns the names of the files of f in a directory
|
||||
// and the values they hold.
|
||||
func frozenFileParts(f *frozenFile) map[string]any {
|
||||
return map[string]any{"file.json": f, "link.json": &f.Stored.Link, "revision.json": &f.Stored.Revision}
|
||||
}
|
||||
|
||||
// blockFileName returns the name of the file holding block i, from 0.
|
||||
func blockFileName(i int) string {
|
||||
return "block-" + strconv.Itoa(i+1) + ".bin"
|
||||
}
|
||||
|
||||
// loadFrozenFile returns the file in dir.
|
||||
func loadFrozenFile(t testing.TB, dir string) frozenFile {
|
||||
t.Helper()
|
||||
var f frozenFile
|
||||
for name, v := range frozenFileParts(&f) {
|
||||
data, err := os.ReadFile(filepath.Join(dir, name))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, json.Unmarshal(data, v), name)
|
||||
}
|
||||
for i := range f.Stored.Revision.Blocks {
|
||||
data, err := os.ReadFile(filepath.Join(dir, blockFileName(i)))
|
||||
require.NoError(t, err)
|
||||
f.Stored.Revision.Blocks[i].Data = data
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// checkFrozenFile stores f for a user holding its keys and checks that
|
||||
// the client reads its metadata, verifying every signature, and
|
||||
// downloads its content.
|
||||
func checkFrozenFile(t *testing.T, f frozenFile) {
|
||||
t.Helper()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUserWithKeys(f.Email, "pass", f.Keys)
|
||||
c := newTestClientFor(t, srv, u, clientOptions{})
|
||||
n := getTestNode(t, c, u.SeedStoredFile(u.RootID(), f.Stored))
|
||||
assert.Equal(t, f.Name, n.Name)
|
||||
require.NotNil(t, n.X, "extended attributes")
|
||||
modTime, err := time.Parse("2006-01-02T15:04:05-0700", f.ModTime)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, modTime.Equal(n.modTime()), "modification time %v, want %v", n.modTime(), modTime)
|
||||
assert.Equal(t, int64(len(f.Content)), n.size(true))
|
||||
assert.Equal(t, sha1Of(f.Content), n.sha1())
|
||||
data, err := readRange(t, c, n, 0, -1)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, bytes.Equal(f.Content, data), "content differs: %d bytes, want %d", len(data), len(f.Content))
|
||||
assert.Empty(t, c.reported, "problems reported")
|
||||
}
|
||||
|
||||
func TestFrozenFileFixture(t *testing.T) {
|
||||
for format, want := range map[string]struct{ pkesk, seipd int }{"legacy": {3, 1}, "v6": {6, 2}} {
|
||||
t.Run(format, func(t *testing.T) {
|
||||
f := loadFrozenFile(t, filepath.Join(frozenFileDir, format))
|
||||
// The fixture is in the format its directory names.
|
||||
keyPacket, err := base64.StdEncoding.DecodeString(f.Stored.Link.ContentKeyPacket)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, want.pkesk, readPackets(t, keyPacket, 1)[0].(*packet.EncryptedKey).Version, "content key packet")
|
||||
require.NotEmpty(t, f.Stored.Revision.Blocks)
|
||||
for i, b := range f.Stored.Revision.Blocks {
|
||||
assert.Equal(t, want.seipd, readPackets(t, b.Data, 1)[0].(*packet.SymmetricallyEncrypted).Version, "block %d", i+1)
|
||||
}
|
||||
checkFrozenFile(t, f)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package protondrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
)
|
||||
|
||||
// ConfigBrowserSignIn runs the browser sign-in of rclone config for
|
||||
// remote name on m, handing the sign-in page to approve instead of
|
||||
// printing it.
|
||||
func ConfigBrowserSignIn(ctx context.Context, name string, m configmap.Mapper, approve func(signInURL string)) error {
|
||||
out, err := configure(ctx, name, m, fs.ConfigIn{State: "browser"}, signInOptions{
|
||||
showURL: approve,
|
||||
pollInterval: 10 * time.Millisecond,
|
||||
pollTimeout: 10 * time.Second,
|
||||
})
|
||||
if err == nil && out != nil {
|
||||
err = fmt.Errorf("config stopped in state %q", out.State)
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package protondrive_test
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/backend/protondrive"
|
||||
"github.com/rclone/rclone/backend/protondrive/protontest"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeEnvPrefix starts the environment variables configuring the
|
||||
// TestProtonDriveFake: remote.
|
||||
const fakeEnvPrefix = "RCLONE_CONFIG_TESTPROTONDRIVEFAKE_"
|
||||
|
||||
// setupFakeRemote starts a fake Proton server with one user and
|
||||
// configures the TestProtonDriveFake: remote, through environment
|
||||
// variables, with a saved session of that user.
|
||||
func setupFakeRemote(t *testing.T) (remoteName string, srv *protontest.Server, user *protontest.User) {
|
||||
srv = protontest.New(t)
|
||||
user = srv.AddUser("user", "pass")
|
||||
sess := srv.NewSession(user)
|
||||
t.Setenv(fakeEnvPrefix+"TYPE", "protondrive")
|
||||
t.Setenv(fakeEnvPrefix+"ENDPOINT", srv.URL())
|
||||
t.Setenv(fakeEnvPrefix+"CLIENT_UID", sess.UID)
|
||||
t.Setenv(fakeEnvPrefix+"CLIENT_ACCESS_TOKEN", sess.AccessToken)
|
||||
t.Setenv(fakeEnvPrefix+"CLIENT_REFRESH_TOKEN", sess.RefreshToken)
|
||||
// The config holds the salted key password base64 encoded.
|
||||
t.Setenv(fakeEnvPrefix+"CLIENT_SALTED_KEY_PASS", base64.StdEncoding.EncodeToString([]byte(user.KeyPassword())))
|
||||
t.Setenv(fakeEnvPrefix+"DRAFT_CLIENT_UID", random.String(32))
|
||||
return "TestProtonDriveFake:", srv, user
|
||||
}
|
||||
|
||||
// TestSetupFakeRemote checks the configured session is accepted by the
|
||||
// fake server.
|
||||
func TestSetupFakeRemote(t *testing.T) {
|
||||
remoteName, srv, user := setupFakeRemote(t)
|
||||
assert.Equal(t, "TestProtonDriveFake:", remoteName)
|
||||
assert.Equal(t, "protondrive", os.Getenv(fakeEnvPrefix+"TYPE"))
|
||||
assert.Equal(t, srv.URL(), os.Getenv(fakeEnvPrefix+"ENDPOINT"))
|
||||
assert.NotEmpty(t, os.Getenv(fakeEnvPrefix+"DRAFT_CLIENT_UID"))
|
||||
keyPass, err := base64.StdEncoding.DecodeString(os.Getenv(fakeEnvPrefix + "CLIENT_SALTED_KEY_PASS"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, user.KeyPassword(), string(keyPass))
|
||||
require.NotEmpty(t, os.Getenv(fakeEnvPrefix+"CLIENT_REFRESH_TOKEN"))
|
||||
|
||||
req, err := http.NewRequest("GET", os.Getenv(fakeEnvPrefix+"ENDPOINT")+"/core/v4/users", nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("x-pm-appversion", "external-drive-rclone@1.0.0-stable")
|
||||
req.Header.Set("x-pm-uid", os.Getenv(fakeEnvPrefix+"CLIENT_UID"))
|
||||
req.Header.Set("Authorization", "Bearer "+os.Getenv(fakeEnvPrefix+"CLIENT_ACCESS_TOKEN"))
|
||||
resp, err := srv.Client().Do(req)
|
||||
require.NoError(t, err)
|
||||
_ = resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
}
|
||||
|
||||
// TestIntegrationFake runs integration tests against the fake server.
|
||||
func TestIntegrationFake(t *testing.T) {
|
||||
if *fstest.RemoteName != "" {
|
||||
t.Skip("skipping as -remote is set")
|
||||
}
|
||||
remoteName, _, _ := setupFakeRemote(t)
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: remoteName,
|
||||
NilObject: (*protondrive.Object)(nil),
|
||||
QuickTestOK: true,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
package protondrive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode/utf16"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/ProtonMail/gopenpgp/v3/crypto"
|
||||
"github.com/rclone/rclone/backend/protondrive/api"
|
||||
"github.com/rclone/rclone/backend/protondrive/protontest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Fuzz targets for everything which parses bytes the server controls.
|
||||
// Each is seeded from the old library's fixtures in testdata/oldlib and
|
||||
// from payloads the fake server makes, and checks invariants of what
|
||||
// is accepted, not only that nothing panics.
|
||||
|
||||
// oldlibFixture is the part of testdata/oldlib/file.json and
|
||||
// folder.json the fuzz targets use.
|
||||
type oldlibFixture struct {
|
||||
NodeKey string
|
||||
NodePassphrasePlain string
|
||||
NamePlain string
|
||||
ChildName string
|
||||
XAttr string
|
||||
}
|
||||
|
||||
// loadOldlibFixture returns testdata/oldlib/name.
|
||||
func loadOldlibFixture(t testing.TB, name string) oldlibFixture {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join("testdata/oldlib", name))
|
||||
require.NoError(t, err)
|
||||
var fx oldlibFixture
|
||||
require.NoError(t, json.Unmarshal(data, &fx))
|
||||
return fx
|
||||
}
|
||||
|
||||
// fuzzLinks returns a client for a user of a fake server holding the
|
||||
// keys of the old library's v6 file fixture, the root folder's keys,
|
||||
// and the JSON of a link of each kind the client reads: the fixture
|
||||
// itself, and nodes the fake makes as SDK clients would.
|
||||
func fuzzLinks(f *testing.F) (*client, *nodeKeys, [][]byte) {
|
||||
frozen := loadFrozenFile(f, filepath.Join(frozenFileDir, "v6"))
|
||||
srv := protontest.New(f)
|
||||
u := srv.AddUserWithKeys(frozen.Email, "pass", frozen.Keys)
|
||||
c := newTestClientFor(f, srv, u, clientOptions{})
|
||||
root := u.RootID()
|
||||
ids := []string{
|
||||
u.SeedStoredFile(root, frozen.Stored),
|
||||
u.SeedFolder(root, "folder"),
|
||||
u.SeedFile(root, "file.txt", []byte("content"), time.Now()),
|
||||
u.SeedFile(root, "empty", nil, time.Now()),
|
||||
u.SeedLegacyFile(root, "legacy", []byte("legacy")),
|
||||
u.SeedAnonymousFile(root, "anonymous", []byte("anonymous"), time.Now()),
|
||||
u.SeedAnonymousFolder(root, "anonymous folder"),
|
||||
u.SeedFileClaimingBlockSizes(root, "claims", []byte("claims"), []int64{1, 2, 3}),
|
||||
u.SeedDoc(root, "doc"),
|
||||
}
|
||||
ctx := context.Background()
|
||||
links, err := c.fetchLinks(ctx, ids)
|
||||
require.NoError(f, err)
|
||||
require.Len(f, links, len(ids))
|
||||
var seeds [][]byte
|
||||
for _, d := range links {
|
||||
data, err := json.Marshal(d)
|
||||
require.NoError(f, err)
|
||||
seeds = append(seeds, data)
|
||||
}
|
||||
keys, err := c.folderKeysOf(ctx, root)
|
||||
require.NoError(f, err)
|
||||
return c, keys, seeds
|
||||
}
|
||||
|
||||
func FuzzValidateName(f *testing.F) {
|
||||
for _, name := range []string{
|
||||
loadOldlibFixture(f, "file.json").NamePlain,
|
||||
loadOldlibFixture(f, "folder.json").NamePlain,
|
||||
loadOldlibFixture(f, "folder.json").ChildName,
|
||||
"file.txt", ".", "..", "...", "", "a/b", "/", "a\x00b", " ", "é", "é",
|
||||
strings.Repeat("a", maxNameLength), strings.Repeat("a", maxNameLength+1),
|
||||
strings.Repeat("\U0001F600", maxNameLength/2), strings.Repeat("\U0001F600", maxNameLength/2+1),
|
||||
"\xff\xfe",
|
||||
} {
|
||||
f.Add(name)
|
||||
}
|
||||
f.Fuzz(func(t *testing.T, name string) {
|
||||
if validateName(name) != nil {
|
||||
return
|
||||
}
|
||||
assert.NotEmpty(t, name)
|
||||
assert.True(t, utf8.ValidString(name), "%q", name)
|
||||
assert.NotEqual(t, ".", name)
|
||||
assert.NotEqual(t, "..", name)
|
||||
assert.NotContains(t, name, "/")
|
||||
assert.NotContains(t, name, "\x00")
|
||||
assert.LessOrEqual(t, len(utf16.Encode([]rune(name))), maxNameLength)
|
||||
})
|
||||
}
|
||||
|
||||
func FuzzDecodeLink(f *testing.F) {
|
||||
c, rootKeys, seeds := fuzzLinks(f)
|
||||
for _, seed := range seeds {
|
||||
f.Add(seed)
|
||||
// Hostile values in links which otherwise decrypt.
|
||||
var d api.LinkDetails
|
||||
require.NoError(f, json.Unmarshal(seed, &d))
|
||||
for _, change := range []func(d *api.LinkDetails){
|
||||
func(d *api.LinkDetails) { d.Link.LinkID = "" },
|
||||
func(d *api.LinkDetails) { d.Link.Type = 3 - d.Link.Type },
|
||||
func(d *api.LinkDetails) {
|
||||
if d.File != nil && d.File.ActiveRevision != nil {
|
||||
d.File.ActiveRevision.EncryptedSize = -5
|
||||
}
|
||||
},
|
||||
} {
|
||||
changed := d
|
||||
changed.File = cloneFile(d.File)
|
||||
change(&changed)
|
||||
data, err := json.Marshal(changed)
|
||||
require.NoError(f, err)
|
||||
f.Add(data)
|
||||
}
|
||||
}
|
||||
// Shapes the client can't decrypt.
|
||||
for _, format := range []string{"legacy", "v6"} {
|
||||
data, err := os.ReadFile(filepath.Join(frozenFileDir, format, "link.json"))
|
||||
require.NoError(f, err)
|
||||
f.Add(data)
|
||||
}
|
||||
f.Add([]byte(`{}`))
|
||||
f.Add([]byte(`{"Link":{"LinkID":"x","Type":1,"ParentLinkID":"p"},"Folder":{}}`))
|
||||
ctx := context.Background()
|
||||
f.Fuzz(func(t *testing.T, data []byte) {
|
||||
var d api.LinkDetails
|
||||
if json.Unmarshal(data, &d) != nil {
|
||||
return
|
||||
}
|
||||
n, unreadable, err := c.decryptNode(ctx, &d, rootKeys)
|
||||
if err != nil || unreadable != nil {
|
||||
assert.Nil(t, n)
|
||||
return
|
||||
}
|
||||
require.NotNil(t, n)
|
||||
assert.NotEmpty(t, n.ID)
|
||||
assert.Equal(t, d.Link.LinkID, n.ID)
|
||||
assert.True(t, utf8.ValidString(n.Name), "%q", n.Name)
|
||||
if d.Link.ParentLinkID != "" {
|
||||
assert.NoError(t, validateName(n.Name))
|
||||
}
|
||||
assert.True(t, n.IsFolder == (d.Link.Type == api.LinkTypeFolder))
|
||||
for _, original := range []bool{true, false} {
|
||||
assert.GreaterOrEqual(t, n.size(original), int64(0), "original=%v", original)
|
||||
}
|
||||
if n.X != nil {
|
||||
checkXAttrs(t, n.X)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// cloneFile returns a copy of file, sharing nothing changeable with it.
|
||||
func cloneFile(file *api.File) *api.File {
|
||||
if file == nil {
|
||||
return nil
|
||||
}
|
||||
out := *file
|
||||
if file.ActiveRevision != nil {
|
||||
rev := *file.ActiveRevision
|
||||
out.ActiveRevision = &rev
|
||||
}
|
||||
return &out
|
||||
}
|
||||
|
||||
// checkXAttrs checks the invariants of parsed extended attributes.
|
||||
//
|
||||
// That the block sizes add up to the size isn't one: as in the SDK
|
||||
// neither is checked against the other, and a read checks each block
|
||||
// against its recorded size, so a file whose attributes disagree fails
|
||||
// to read rather than reading wrong.
|
||||
func checkXAttrs(t *testing.T, x *xattrs) {
|
||||
t.Helper()
|
||||
assert.GreaterOrEqual(t, x.Size, int64(0))
|
||||
if x.NoSize {
|
||||
assert.Zero(t, x.Size)
|
||||
}
|
||||
if x.BlockSizes != nil {
|
||||
assert.NotEmpty(t, x.BlockSizes)
|
||||
}
|
||||
for i, size := range x.BlockSizes {
|
||||
assert.Positive(t, size, "block %d", i+1)
|
||||
}
|
||||
if !x.ModTime.IsZero() {
|
||||
assert.Equal(t, time.UTC, x.ModTime.Location())
|
||||
// RFC 3339 years, moved by a zone offset of under a day.
|
||||
assert.True(t, x.ModTime.Year() >= -1 && x.ModTime.Year() <= 10000, "%v", x.ModTime)
|
||||
}
|
||||
assert.NotContains(t, x.Extra, "Common")
|
||||
}
|
||||
|
||||
func FuzzCheckKeyS2K(f *testing.F) {
|
||||
f.Add(unarmor(f, loadOldlibFixture(f, "file.json").NodeKey))
|
||||
now := time.Now()
|
||||
parent, signer := newTestKeyRing(f, now), newTestKeyRing(f, now)
|
||||
_, _, _, keys := testNodeKeys(f, parent, signer, true, now)
|
||||
for _, memExp := range []byte{3, 17, 18, 31} {
|
||||
f.Add(unarmor(f, argon2Locked(f, keys, 1, 1, memExp)))
|
||||
}
|
||||
f.Fuzz(func(t *testing.T, data []byte) {
|
||||
// Whatever the bytes, the check never runs a KDF and never
|
||||
// panics; a key it lets through still parses or fails to.
|
||||
if checkKeyS2K(data) == nil {
|
||||
_, _ = crypto.NewKey(data)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func FuzzDecryptXAttrs(f *testing.F) {
|
||||
fx := loadOldlibFixture(f, "file.json")
|
||||
key, err := unlockKey(fx.NodeKey, []byte(fx.NodePassphrasePlain))
|
||||
require.NoError(f, err)
|
||||
nodeKR, err := crypto.NewKeyRing(key)
|
||||
require.NoError(f, err)
|
||||
now := time.Now()
|
||||
|
||||
// The old library's attributes, encrypted and in plain text.
|
||||
res, _, _, err := decryptMessage(fx.XAttr, nodeKR, nil, now, maxXAttrsSize)
|
||||
require.NoError(f, err)
|
||||
f.Add(fx.XAttr, res.Bytes())
|
||||
// Attributes as the fake writes them for SDK clients.
|
||||
c, _, seeds := fuzzLinks(f)
|
||||
for _, seed := range seeds {
|
||||
var d api.LinkDetails
|
||||
require.NoError(f, json.Unmarshal(seed, &d))
|
||||
if d.File == nil || d.File.ActiveRevision == nil || d.File.ActiveRevision.XAttr == "" {
|
||||
continue
|
||||
}
|
||||
n, err := c.getNode(context.Background(), d.Link.LinkID)
|
||||
require.NoError(f, err)
|
||||
res, _, _, err := decryptMessage(d.File.ActiveRevision.XAttr, n.keys.keyRing, nil, now, maxXAttrsSize)
|
||||
require.NoError(f, err)
|
||||
f.Add("", res.Bytes())
|
||||
}
|
||||
for _, plain := range []string{
|
||||
`{"Common":{"ModificationTime":"2021-09-16T07:40:54+0000","Size":-1,"BlockSizes":[4194304,-1],"Digests":{"SHA1":"x"}}}`,
|
||||
`{"Common":{"Size":1152921504606846976,"BlockSizes":[1152921504606846976,1]}}`,
|
||||
`{"Common":{"ModificationTime":"99999-01-01T00:00:00Z"}}`,
|
||||
`{"Common":{"ModificationTime":"0000-01-01T00:00:00+23:59"}}`,
|
||||
`{"Common":{"BlockSizes":[]},"Media":{"Width":1}}`,
|
||||
`{"Common":{"BlockSizes":[` + strings.Repeat("4194304,", 10000) + `1]}}`,
|
||||
strings.Repeat(`{"Common":`, 20000) + strings.Repeat("}", 20000),
|
||||
`{"Common":` + strings.Repeat("[", 100000) + strings.Repeat("]", 100000) + `}`,
|
||||
} {
|
||||
f.Add("", []byte(plain))
|
||||
}
|
||||
|
||||
f.Fuzz(func(t *testing.T, enc string, plain []byte) {
|
||||
if x, _, err := decryptXAttrs(enc, nodeKR, nodeKR, now); err == nil {
|
||||
checkXAttrs(t, x)
|
||||
}
|
||||
if len(plain) > maxXAttrsSize/2 {
|
||||
return
|
||||
}
|
||||
enc, err := encryptArmored(plain, nil, nodeKR, nodeKR, true, now)
|
||||
require.NoError(t, err)
|
||||
x, verifyErr, err := decryptXAttrs(enc, nodeKR, nodeKR, now)
|
||||
if !utf8.Valid(plain) {
|
||||
assert.Error(t, err)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, verifyErr)
|
||||
checkXAttrs(t, x)
|
||||
})
|
||||
}
|
||||
|
||||
func FuzzDecodeAPIError(f *testing.F) {
|
||||
for _, seed := range []struct {
|
||||
status int
|
||||
body string
|
||||
retryAfter string
|
||||
}{
|
||||
{422, `{"Code":2500,"Error":"A file or folder with that name already exists","Details":{"ConflictLinkID":"abc","ConflictRevisionID":"def"}}`, ""},
|
||||
{422, `{"Code":2501,"Error":"File or folder not found"}`, ""},
|
||||
{429, `{"Code":85131,"Error":"Too many recent API requests"}`, "17"},
|
||||
{429, ``, "Wed, 21 Oct 2015 07:28:00 GMT"},
|
||||
{429, ``, "99999999999999999999999"},
|
||||
{401, `{"Code":401,"Error":"Invalid access token"}`, ""},
|
||||
{400, `{"Code":2064,"Error":"protontest: invalid x-pm-appversion \"x\""}`, ""},
|
||||
{500, `<html>Internal Server Error</html>`, ""},
|
||||
{200, `{"Code":"1000"}`, ""},
|
||||
{0, `{"Error":null,"Details":[1,2,3]}`, "-5"},
|
||||
} {
|
||||
f.Add(seed.status, []byte(seed.body), seed.retryAfter)
|
||||
}
|
||||
f.Fuzz(func(t *testing.T, status int, body []byte, retryAfterValue string) {
|
||||
resp := &http.Response{
|
||||
StatusCode: status,
|
||||
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
|
||||
Header: http.Header{"Retry-After": {retryAfterValue}},
|
||||
Body: io.NopCloser(bytes.NewReader(body)),
|
||||
}
|
||||
err := errorHandler(resp)
|
||||
var apiErr *api.Error
|
||||
require.ErrorAs(t, err, &apiErr)
|
||||
assert.Equal(t, status, apiErr.Status)
|
||||
assert.NotEmpty(t, apiErr.Message)
|
||||
assert.NotEmpty(t, err.Error())
|
||||
assert.Equal(t, apiErr.Code, apiErrorCode(err))
|
||||
|
||||
wait := retryAfter(resp)
|
||||
assert.Positive(t, wait)
|
||||
assert.LessOrEqual(t, wait, maxRetryAfter)
|
||||
})
|
||||
}
|
||||
|
||||
func FuzzDecryptForkPayload(f *testing.F) {
|
||||
key := bytes.Repeat([]byte{7}, 32)
|
||||
for _, seed := range []struct{ plain, aad string }{
|
||||
{`{"type":"default","keyPassword":"the-key-password"}`, forkPayloadAAD},
|
||||
{`{"keyPassword":""}`, forkPayloadAAD},
|
||||
{`{"keyPassword":null}`, forkPayloadAAD},
|
||||
{`{"type":"default"}`, forkPayloadAAD},
|
||||
{`not json`, forkPayloadAAD},
|
||||
{`{"keyPassword":"wrong additional data"}`, "other"},
|
||||
} {
|
||||
f.Add(sealForkPayload(f, key, seed.plain, seed.aad))
|
||||
}
|
||||
f.Add("")
|
||||
f.Add("not base64!")
|
||||
f.Add(base64.StdEncoding.EncodeToString(make([]byte, 12)))
|
||||
f.Fuzz(func(t *testing.T, payload string) {
|
||||
keyPassword, err := decryptForkPayload(payload, key)
|
||||
if err != nil {
|
||||
assert.Empty(t, keyPassword)
|
||||
return
|
||||
}
|
||||
// Success only for a payload sealed with key: open it here
|
||||
// independently.
|
||||
blob, err := base64.StdEncoding.DecodeString(payload)
|
||||
require.NoError(t, err)
|
||||
block, err := aes.NewCipher(key)
|
||||
require.NoError(t, err)
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
require.NoError(t, err)
|
||||
require.GreaterOrEqual(t, len(blob), gcm.NonceSize())
|
||||
plain, err := gcm.Open(nil, blob[:gcm.NonceSize()], blob[gcm.NonceSize():], []byte(forkPayloadAAD))
|
||||
require.NoError(t, err, "accepted a payload which doesn't authenticate")
|
||||
var p struct{ KeyPassword *string }
|
||||
require.NoError(t, json.Unmarshal(plain, &p))
|
||||
require.NotNil(t, p.KeyPassword)
|
||||
assert.Equal(t, *p.KeyPassword, keyPassword)
|
||||
})
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
+895
-880
File diff suppressed because it is too large.
Load diff
@@ -1,97 +1,678 @@
|
||||
package protondrive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"io"
|
||||
"maps"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/go-proton-api"
|
||||
"github.com/pquerna/otp/totp"
|
||||
"github.com/rclone/rclone/backend/protondrive/api"
|
||||
"github.com/rclone/rclone/backend/protondrive/protontest"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var protonDriveAppVersionPattern = regexp.MustCompile(`(?i)^external-drive(-[a-z_]+)+@[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?-((stable|beta|RC|alpha)(([.-]?\d+)*)?)?([.-]?dev)?(\+.*)?$`)
|
||||
func TestTranslateError(t *testing.T) {
|
||||
apiErr := func(code int) error {
|
||||
return fmt.Errorf("call: %w", &api.Error{Status: 422, Code: code, Message: "proton says no"})
|
||||
}
|
||||
notFound := fmt.Errorf("get node: %w", errNotFound)
|
||||
exists := fmt.Errorf("create: %w", errExists)
|
||||
other := errors.New("other")
|
||||
|
||||
func TestProtonDriveAppVersionFromRcloneVersion(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
rcloneVersion string
|
||||
want string
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
err error
|
||||
op errorOp
|
||||
want error // compared with ==, if set
|
||||
check func(t *testing.T, got error)
|
||||
}{
|
||||
{
|
||||
name: "release",
|
||||
rcloneVersion: "v1.73.5",
|
||||
want: "external-drive-rclone@1.73.5-stable",
|
||||
},
|
||||
{
|
||||
name: "dev build",
|
||||
rcloneVersion: "v1.74.0-DEV",
|
||||
want: "external-drive-rclone@1.74.0-dev",
|
||||
},
|
||||
{
|
||||
name: "beta build with extra metadata",
|
||||
rcloneVersion: "v1.74.0-beta.9519.990f33f2a.fix-protondrive-sdk-2026",
|
||||
want: "external-drive-rclone@1.74.0-beta.9519+990f33f2a.fix-protondrive-sdk-2026",
|
||||
},
|
||||
{
|
||||
name: "beta build with unsanitized branch name",
|
||||
rcloneVersion: "v1.74.0-beta.9519.990f33f2a.fix/protondrive-sdk-2026",
|
||||
want: "external-drive-rclone@1.74.0-beta.9519+990f33f2a.fix-protondrive-sdk-2026",
|
||||
},
|
||||
{
|
||||
name: "invalid version falls back to stable",
|
||||
rcloneVersion: "not-a-version",
|
||||
want: "external-drive-rclone@1.0.0-stable",
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
got := protonDriveAppVersionFromRcloneVersion(testCase.rcloneVersion)
|
||||
|
||||
if got != testCase.want {
|
||||
t.Fatalf("unexpected app version: got %q, want %q", got, testCase.want)
|
||||
}
|
||||
if !protonDriveAppVersionPattern.MatchString(got) {
|
||||
t.Fatalf("app version %q does not match Proton pattern", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldRetry(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cancelledCtx, cancel := context.WithCancel(ctx)
|
||||
cancel()
|
||||
|
||||
apiErr := func(status int, code proton.Code) error {
|
||||
return &proton.APIError{Status: status, Code: code, Message: "test"}
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
ctx context.Context
|
||||
err error
|
||||
wantRetry bool
|
||||
}{
|
||||
{"nil error", ctx, nil, false},
|
||||
{"cancelled context", cancelledCtx, errors.New("some error"), false},
|
||||
{"permanent validation error Code=200501 Status=422 (not retried)", ctx, apiErr(422, 200501), false},
|
||||
{"transient storage block error Code=200501 Status=500 (retried)", ctx, apiErr(500, 200501), true},
|
||||
{"server error Status=500", ctx, apiErr(500, 0), true},
|
||||
{"server error Status=502", ctx, apiErr(502, 0), true},
|
||||
{"server error Status=504", ctx, apiErr(504, 0), true},
|
||||
{"server error Status=503 (handled by SDK, not retried here)", ctx, apiErr(503, 0), false},
|
||||
{"rate limit Status=429 (handled by SDK, not retried here)", ctx, apiErr(429, 0), false},
|
||||
{"client error Status=400", ctx, apiErr(400, 0), false},
|
||||
{"client error Status=404", ctx, apiErr(404, 0), false},
|
||||
{"wrapped API error retried via errors.As", ctx, fmt.Errorf("wrapped: %w", apiErr(500, 0)), true},
|
||||
{"non-API error falls back to fserrors.ShouldRetry", ctx, errors.New("plain error"), false},
|
||||
{name: "nil", err: nil, op: opObject, want: nil},
|
||||
{name: "not found object", err: notFound, op: opObject, want: fs.ErrorObjectNotFound},
|
||||
{name: "not found dir", err: notFound, op: opDir, want: fs.ErrorDirNotFound},
|
||||
{name: "code 2501 object", err: apiErr(api.CodeNotExists), op: opObject, want: fs.ErrorObjectNotFound},
|
||||
{name: "code 2501 dir", err: apiErr(api.CodeNotExists), op: opDir, want: fs.ErrorDirNotFound},
|
||||
{name: "exists mkdir", err: exists, op: opMkdir, want: nil},
|
||||
{name: "exists move", err: exists, op: opMove, want: fs.ErrorCantMove},
|
||||
{name: "exists copy", err: exists, op: opCopy, want: fs.ErrorCantCopy},
|
||||
{name: "exists dirmove", err: exists, op: opDirMove, want: fs.ErrorDirExists},
|
||||
{name: "exists object", err: exists, op: opObject, check: func(t *testing.T, got error) {
|
||||
assert.ErrorIs(t, got, errExists)
|
||||
}},
|
||||
{name: "other unchanged", err: other, op: opObject, want: other},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
gotRetry, _ := shouldRetry(tc.ctx, tc.err)
|
||||
assert.Equal(t, tc.wantRetry, gotRetry)
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got := translateError(test.err, test.op)
|
||||
if test.check != nil {
|
||||
test.check(t, got)
|
||||
return
|
||||
}
|
||||
assert.Equal(t, test.want, got)
|
||||
})
|
||||
}
|
||||
|
||||
for _, code := range []int{
|
||||
api.CodeInsufficientQuota, api.CodeInsufficientSpace, api.CodeMaxFileSizeForFreeUser,
|
||||
api.CodeMaxPublicEditModeForFreeUser, api.CodeTooManyChildren, api.CodeNestingTooDeep,
|
||||
} {
|
||||
t.Run(fmt.Sprintf("code %d no retry", code), func(t *testing.T) {
|
||||
got := translateError(apiErr(code), opObject)
|
||||
assert.True(t, fserrors.IsNoRetryError(got))
|
||||
assert.False(t, fserrors.IsFatalError(got))
|
||||
assert.Contains(t, got.Error(), "proton says no")
|
||||
})
|
||||
}
|
||||
for _, code := range []int{api.CodeAppVersionMissing, api.CodeAppVersionBad} {
|
||||
t.Run(fmt.Sprintf("code %d fatal", code), func(t *testing.T) {
|
||||
got := translateError(apiErr(code), opObject)
|
||||
assert.True(t, fserrors.IsFatalError(got))
|
||||
assert.Contains(t, got.Error(), "upgrade rclone")
|
||||
assert.Contains(t, got.Error(), "proton says no")
|
||||
})
|
||||
}
|
||||
t.Run("code 2000 no retry", func(t *testing.T) {
|
||||
got := translateError(apiErr(api.CodeInvalidRequirements), opMove)
|
||||
assert.True(t, fserrors.IsNoRetryError(got))
|
||||
assert.False(t, fserrors.IsFatalError(got))
|
||||
assert.NotContains(t, got.Error(), "upgrade rclone")
|
||||
assert.Contains(t, got.Error(), "proton says no")
|
||||
})
|
||||
t.Run("code 2028 not fatal", func(t *testing.T) {
|
||||
got := translateError(apiErr(api.CodeFeatureBlocked), opObject)
|
||||
assert.False(t, fserrors.IsFatalError(got))
|
||||
assert.False(t, fserrors.IsNoRetryError(got))
|
||||
assert.NotContains(t, got.Error(), "upgrade rclone")
|
||||
assert.Contains(t, got.Error(), "proton says no")
|
||||
})
|
||||
for _, err := range []error{errIntegrity, errUploadMismatch, errUploadSuperseded} {
|
||||
t.Run(err.Error()+" no retry", func(t *testing.T) {
|
||||
got := translateError(fmt.Errorf("op: %w", err), opObject)
|
||||
assert.True(t, fserrors.IsNoRetryError(got))
|
||||
assert.ErrorIs(t, got, err)
|
||||
})
|
||||
}
|
||||
for _, err := range []error{errUploadOutcomeUnknown, errUploadUnconfirmed} {
|
||||
t.Run(err.Error()+" no low level retry", func(t *testing.T) {
|
||||
got := translateError(fmt.Errorf("op: %w", err), opObject)
|
||||
assert.True(t, fserrors.IsNoLowLevelRetryError(got))
|
||||
assert.False(t, fserrors.IsNoRetryError(got))
|
||||
assert.ErrorIs(t, got, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// sessionConfig returns the config keys of a v1.76 remote holding a new
|
||||
// saved session of u, pointed at srv.
|
||||
func sessionConfig(srv *protontest.Server, u *protontest.User) configmap.Simple {
|
||||
sess := srv.NewSession(u)
|
||||
return configmap.Simple{
|
||||
"type": "protondrive",
|
||||
"username": "",
|
||||
"password": "",
|
||||
"mailbox_password": "",
|
||||
"2fa": "",
|
||||
"otp_secret_key": "",
|
||||
"client_uid": sess.UID,
|
||||
"client_access_token": sess.AccessToken,
|
||||
"client_refresh_token": sess.RefreshToken,
|
||||
"client_salted_key_pass": base64.StdEncoding.EncodeToString([]byte(u.KeyPassword())),
|
||||
"encoding": "Slash,LeftSpace,RightSpace,InvalidUtf8,Dot",
|
||||
"original_file_size": "true",
|
||||
"app_version": "",
|
||||
"replace_existing_draft": "false",
|
||||
"enable_caching": "true",
|
||||
// Not a v1.76 key, but the only way to reach the fake.
|
||||
"endpoint": srv.URL(),
|
||||
}
|
||||
}
|
||||
|
||||
// newTestFs returns an Fs for the config m, shut down when the test ends.
|
||||
func newTestFs(t *testing.T, m configmap.Mapper) *Fs {
|
||||
t.Helper()
|
||||
f, err := NewFs(context.Background(), "test", "", m)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = f.(*Fs).Shutdown(context.Background()) })
|
||||
return f.(*Fs)
|
||||
}
|
||||
|
||||
// authRequests returns the auth requests made to srv.
|
||||
func authRequests(srv *protontest.Server) []string {
|
||||
var out []string
|
||||
for route := range srv.Requests() {
|
||||
if strings.Contains(route, "/auth") {
|
||||
out = append(out, route)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestOptionsBackwardsCompatible(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
m := sessionConfig(srv, u)
|
||||
want := m["client_access_token"]
|
||||
f := newTestFs(t, m)
|
||||
|
||||
_, err := f.List(context.Background(), "")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, authRequests(srv), "a saved session must be used without signing in")
|
||||
assert.Equal(t, want, m["client_access_token"])
|
||||
assert.Len(t, m["draft_client_uid"], 32, "draft_client_uid is generated and saved")
|
||||
assert.True(t, f.opt.ReportOriginalSize)
|
||||
assert.True(t, f.opt.EnableCaching)
|
||||
}
|
||||
|
||||
// unsavedMap is a config which saves nothing, as one from environment
|
||||
// variables or a connection string.
|
||||
type unsavedMap struct{ configmap.Simple }
|
||||
|
||||
// Set does nothing.
|
||||
func (unsavedMap) Set(key, value string) {}
|
||||
|
||||
func TestDraftClientUIDStableWhenUnsaved(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
uid := func(name string) string {
|
||||
m := sessionConfig(srv, u)
|
||||
f, err := NewFs(ctx, name, "", unsavedMap{m})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = f.(*Fs).Shutdown(ctx) })
|
||||
got := f.(*Fs).c.opt.DraftClientUID
|
||||
assert.Len(t, got, 32)
|
||||
return got
|
||||
}
|
||||
first := uid("env")
|
||||
assert.Equal(t, first, uid("env"), "a later run of the same remote has another draft client ID")
|
||||
assert.NotEqual(t, first, uid("other"), "another remote of the account has the same draft client ID")
|
||||
}
|
||||
|
||||
func TestNewFsPasswordFallback(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
m := sessionConfig(srv, u)
|
||||
m["username"] = "user"
|
||||
m["password"] = obscure.MustObscure("pass")
|
||||
oldUID := m["client_uid"]
|
||||
srv.RevokeSession(oldUID)
|
||||
|
||||
f := newTestFs(t, m)
|
||||
_, err := f.List(context.Background(), "")
|
||||
require.NoError(t, err)
|
||||
assert.NotEqual(t, oldUID, m["client_uid"])
|
||||
assert.NotEmpty(t, m["client_access_token"])
|
||||
assert.NotEmpty(t, m["client_refresh_token"])
|
||||
keyPass, err := base64.StdEncoding.DecodeString(m["client_salted_key_pass"])
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, u.KeyPassword(), string(keyPass))
|
||||
assert.Equal(t, 1, srv.Requests()["POST /core/v4/auth"])
|
||||
}
|
||||
|
||||
func TestNewFsPasswordFallbackTOTP(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
secret := u.EnableTOTP()
|
||||
m := sessionConfig(srv, u)
|
||||
for _, k := range []string{"client_uid", "client_access_token", "client_refresh_token", "client_salted_key_pass"} {
|
||||
m[k] = ""
|
||||
}
|
||||
m["username"] = "user"
|
||||
m["password"] = obscure.MustObscure("pass")
|
||||
m["otp_secret_key"] = obscure.MustObscure(secret)
|
||||
|
||||
newTestFs(t, m)
|
||||
assert.NotEmpty(t, m["client_uid"])
|
||||
assert.Equal(t, 1, srv.Requests()["POST /core/v4/auth/2fa"])
|
||||
}
|
||||
|
||||
func TestNewFsNoCredentials(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
|
||||
t.Run("none saved", func(t *testing.T) {
|
||||
m := configmap.Simple{"endpoint": srv.URL()}
|
||||
_, err := NewFs(context.Background(), "myremote", "", m)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), `rclone config reconnect myremote:`)
|
||||
})
|
||||
t.Run("revoked", func(t *testing.T) {
|
||||
m := sessionConfig(srv, u)
|
||||
srv.RevokeSession(m["client_uid"])
|
||||
_, err := NewFs(context.Background(), "myremote", "", m)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), `rclone config reconnect myremote:`)
|
||||
})
|
||||
t.Run("undecodable key password", func(t *testing.T) {
|
||||
m := sessionConfig(srv, u)
|
||||
m["client_salted_key_pass"] = "not base64!"
|
||||
_, err := NewFs(context.Background(), "myremote", "", m)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), `rclone config reconnect myremote:`)
|
||||
})
|
||||
}
|
||||
|
||||
func TestTwoRemotesIsolated(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
srv := protontest.New(t)
|
||||
u1 := srv.AddUser("user1", "pass1")
|
||||
u2 := srv.AddUser("user2", "pass2")
|
||||
m1, m2 := sessionConfig(srv, u1), sessionConfig(srv, u2)
|
||||
uid1, uid2 := m1["client_uid"], m2["client_uid"]
|
||||
f1, f2 := newTestFs(t, m1), newTestFs(t, m2)
|
||||
require.NoError(t, f1.Mkdir(ctx, "one"))
|
||||
require.NoError(t, f2.Mkdir(ctx, "two"))
|
||||
|
||||
// Make both sessions refresh, saving new tokens.
|
||||
access1, access2 := m1["client_access_token"], m2["client_access_token"]
|
||||
srv.ResetRequests()
|
||||
srv.ExpireAccessToken(uid1)
|
||||
srv.ExpireAccessToken(uid2)
|
||||
f1.DirCacheFlush()
|
||||
f2.DirCacheFlush()
|
||||
entries1, err := f1.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
entries2, err := f2.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Len(t, entries1, 1)
|
||||
assert.Equal(t, "one", entries1[0].Remote())
|
||||
require.Len(t, entries2, 1)
|
||||
assert.Equal(t, "two", entries2[0].Remote())
|
||||
assert.Equal(t, 2, srv.Requests()["POST /auth/v4/refresh"])
|
||||
assert.NotEqual(t, access1, m1["client_access_token"], "remote 1 saved its refreshed token")
|
||||
assert.NotEqual(t, access2, m2["client_access_token"], "remote 2 saved its refreshed token")
|
||||
assert.Equal(t, uid1, m1["client_uid"])
|
||||
assert.Equal(t, uid2, m2["client_uid"])
|
||||
for _, k := range []string{"client_uid", "client_access_token", "client_refresh_token", "client_salted_key_pass", "draft_client_uid"} {
|
||||
assert.NotEqual(t, m1[k], m2[k], k)
|
||||
}
|
||||
}
|
||||
|
||||
// configAnswers drives configure through its states, answering each
|
||||
// question by its state from answers, and returns the questions asked.
|
||||
func configAnswers(t *testing.T, m configmap.Mapper, signIn signInOptions, answers map[string]string) []*fs.ConfigOut {
|
||||
t.Helper()
|
||||
var asked []*fs.ConfigOut
|
||||
in := fs.ConfigIn{}
|
||||
for range 10 {
|
||||
out, err := configure(context.Background(), "test", m, in, signIn)
|
||||
require.NoError(t, err)
|
||||
if out == nil {
|
||||
return asked
|
||||
}
|
||||
require.Empty(t, out.Error)
|
||||
in = fs.ConfigIn{State: out.State}
|
||||
if out.Option != nil {
|
||||
asked = append(asked, out)
|
||||
answer, ok := answers[out.State]
|
||||
require.True(t, ok, "unexpected question in state %q: %s", out.State, out.Option.Help)
|
||||
in.Result = answer
|
||||
}
|
||||
}
|
||||
t.Fatal("config didn't finish")
|
||||
return nil
|
||||
}
|
||||
|
||||
// requireSession checks m holds a working session of u on srv.
|
||||
func requireSession(t *testing.T, srv *protontest.Server, m configmap.Simple, u *protontest.User) {
|
||||
t.Helper()
|
||||
keyPass, err := base64.StdEncoding.DecodeString(m["client_salted_key_pass"])
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, u.KeyPassword(), string(keyPass))
|
||||
srv.ResetRequests()
|
||||
newTestFs(t, m)
|
||||
assert.Empty(t, authRequests(srv))
|
||||
}
|
||||
|
||||
func TestConfigBrowser(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
m := configmap.Simple{"endpoint": srv.URL()}
|
||||
var shown string
|
||||
signIn := signInOptions{
|
||||
pollInterval: 10 * time.Millisecond,
|
||||
pollTimeout: 10 * time.Second,
|
||||
showURL: func(signInURL string) {
|
||||
shown = signInURL
|
||||
require.NoError(t, srv.ApproveFork(signInURL, u))
|
||||
},
|
||||
}
|
||||
|
||||
asked := configAnswers(t, m, signIn, map[string]string{"choose": "true"})
|
||||
require.Len(t, asked, 1)
|
||||
assert.Contains(t, asked[0].Option.Help, "rclone is a third-party application not officially supported by Proton.")
|
||||
assert.Equal(t, true, asked[0].Option.Default)
|
||||
assert.True(t, strings.HasPrefix(shown, forkSignInURL), shown)
|
||||
requireSession(t, srv, m, u)
|
||||
}
|
||||
|
||||
func TestConfigPassword(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
|
||||
t.Run("no 2fa", func(t *testing.T) {
|
||||
u := srv.AddUser("user", "pass")
|
||||
m := configmap.Simple{"endpoint": srv.URL(), "username": "user", "password": obscure.MustObscure("pass")}
|
||||
asked := configAnswers(t, m, signInOptions{}, map[string]string{"choose": "false"})
|
||||
require.Len(t, asked, 1)
|
||||
assert.Equal(t, false, asked[0].Option.Default, "password sign-in is the default when a password is set")
|
||||
requireSession(t, srv, m, u)
|
||||
})
|
||||
t.Run("2fa asked", func(t *testing.T) {
|
||||
u := srv.AddUser("user2fa", "pass")
|
||||
secret := u.EnableTOTP()
|
||||
m := configmap.Simple{"endpoint": srv.URL(), "username": "user2fa", "password": obscure.MustObscure("pass")}
|
||||
code, err := totp.GenerateCode(secret, time.Now())
|
||||
require.NoError(t, err)
|
||||
asked := configAnswers(t, m, signInOptions{}, map[string]string{"choose": "false", "password_2fa": code})
|
||||
require.Len(t, asked, 2)
|
||||
requireSession(t, srv, m, u)
|
||||
assert.Empty(t, m["2fa"], "the 2FA code is not saved")
|
||||
})
|
||||
t.Run("captcha", func(t *testing.T) {
|
||||
u := srv.AddUser("usercaptcha", "pass")
|
||||
srv.RequireCaptcha(u)
|
||||
m := configmap.Simple{"endpoint": srv.URL(), "username": "usercaptcha", "password": obscure.MustObscure("pass")}
|
||||
out, err := configure(context.Background(), "test", m, fs.ConfigIn{State: "password"}, signInOptions{})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out)
|
||||
assert.Contains(t, out.Error, "web browser")
|
||||
assert.Equal(t, "method", out.State, "goes back to the choice of sign-in")
|
||||
assert.Empty(t, m["client_uid"])
|
||||
})
|
||||
t.Run("no password", func(t *testing.T) {
|
||||
m := configmap.Simple{"endpoint": srv.URL()}
|
||||
out, err := configure(context.Background(), "test", m, fs.ConfigIn{State: "password"}, signInOptions{})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out)
|
||||
assert.NotEmpty(t, out.Error)
|
||||
})
|
||||
}
|
||||
|
||||
func TestConfigStore(t *testing.T) {
|
||||
creds := credentials{UID: "uid", AccessToken: "access", RefreshToken: "refresh", KeyPassword: "key\x00pass"}
|
||||
|
||||
t.Run("round trip", func(t *testing.T) {
|
||||
m := configmap.Simple{}
|
||||
s := &configStore{m: m}
|
||||
s.Save(creds)
|
||||
assert.Equal(t, base64.StdEncoding.EncodeToString([]byte(creds.KeyPassword)), m[clientSaltedKeyPassKey])
|
||||
assert.Equal(t, creds, s.Load())
|
||||
// Another user of the config saves newer tokens.
|
||||
m[clientRefreshTokenKey] = "newer"
|
||||
assert.Equal(t, "newer", s.Load().RefreshToken)
|
||||
})
|
||||
t.Run("overridden", func(t *testing.T) {
|
||||
// Values from the environment or the command line take
|
||||
// precedence over the config file the store saves to.
|
||||
env := configmap.Simple{clientUIDKey: "old uid", clientAccessTokenKey: "old access", clientRefreshTokenKey: "old refresh"}
|
||||
file := configmap.Simple{}
|
||||
m := configmap.New().AddGetter(env, configmap.PriorityNormal).AddGetter(file, configmap.PriorityConfig).AddSetter(file)
|
||||
s := &configStore{m: m}
|
||||
assert.Equal(t, "old refresh", s.Load().RefreshToken)
|
||||
s.Save(creds)
|
||||
assert.Equal(t, creds, s.Load(), "saved credentials must not be shadowed by spent ones")
|
||||
})
|
||||
}
|
||||
|
||||
// putFile uploads content to f at remote and returns the object.
|
||||
func putFile(t *testing.T, f fs.Fs, remote string, content []byte) fs.Object {
|
||||
t.Helper()
|
||||
src := object.NewStaticObjectInfo(remote, time.Now(), int64(len(content)), true, nil, nil)
|
||||
o, err := f.Put(context.Background(), bytes.NewReader(content), src)
|
||||
require.NoError(t, err)
|
||||
return o
|
||||
}
|
||||
|
||||
// readObject returns the content of o read with options.
|
||||
func readObject(t *testing.T, o fs.Object, options ...fs.OpenOption) ([]byte, error) {
|
||||
t.Helper()
|
||||
rc, err := o.Open(context.Background(), options...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { require.NoError(t, rc.Close()) }()
|
||||
return io.ReadAll(rc)
|
||||
}
|
||||
|
||||
func TestRmdirRefusesHiddenChildren(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
f := newTestFs(t, sessionConfig(srv, u))
|
||||
|
||||
require.NoError(t, f.Mkdir(ctx, "docs"))
|
||||
docsID, err := f.dirCache.FindDir(ctx, "docs", false)
|
||||
require.NoError(t, err)
|
||||
u.SeedDoc(docsID, "notes")
|
||||
assert.Equal(t, fs.ErrorDirectoryNotEmpty, f.Rmdir(ctx, "docs"))
|
||||
|
||||
// Filled by another client after an empty listing was cached.
|
||||
require.NoError(t, f.Mkdir(ctx, "later"))
|
||||
entries, err := f.List(ctx, "later")
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, entries)
|
||||
laterID, err := f.dirCache.FindDir(ctx, "later", false)
|
||||
require.NoError(t, err)
|
||||
u.SeedFile(laterID, "new.txt", []byte("x"), time.Now())
|
||||
assert.Equal(t, fs.ErrorDirectoryNotEmpty, f.Rmdir(ctx, "later"))
|
||||
|
||||
require.NoError(t, f.Mkdir(ctx, "empty"))
|
||||
require.NoError(t, f.Rmdir(ctx, "empty"))
|
||||
paths := map[string]bool{}
|
||||
for _, d := range u.Dump() {
|
||||
paths[d.Path] = d.State == api.LinkStateActive
|
||||
}
|
||||
assert.True(t, paths["docs/notes"])
|
||||
assert.True(t, paths["later/new.txt"])
|
||||
}
|
||||
|
||||
func TestConfigKeepsSession(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
ri, err := fs.Find("protondrive")
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("default", func(t *testing.T) {
|
||||
m := sessionConfig(srv, u)
|
||||
want := maps.Clone(m)
|
||||
srv.ResetRequests()
|
||||
asked := configAnswers(t, m, signInOptions{}, map[string]string{"reauth": "false"})
|
||||
require.Len(t, asked, 1)
|
||||
assert.Equal(t, false, asked[0].Option.Default)
|
||||
assert.Equal(t, "config_refresh_token", asked[0].Option.Name)
|
||||
assert.Empty(t, srv.Requests())
|
||||
assert.Equal(t, want, m)
|
||||
})
|
||||
t.Run("config update", func(t *testing.T) {
|
||||
// As "rclone config update remote enable_caching=false" runs it.
|
||||
m := sessionConfig(srv, u)
|
||||
srv.ResetRequests()
|
||||
out, err := fs.BackendConfig(context.Background(), "test", m, ri, configmap.Simple{"config_refresh_token": "false"}, fs.ConfigIn{})
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, out)
|
||||
assert.Empty(t, srv.Requests())
|
||||
})
|
||||
}
|
||||
|
||||
func TestConfigSignInAgain(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
m := sessionConfig(srv, u)
|
||||
m["username"] = "user"
|
||||
m["password"] = obscure.MustObscure("pass")
|
||||
old := maps.Clone(m)
|
||||
|
||||
configAnswers(t, m, signInOptions{}, map[string]string{"reauth": "true", "choose": "false"})
|
||||
assert.NotEqual(t, old["client_uid"], m["client_uid"])
|
||||
assert.Equal(t, 1, srv.Requests()["DELETE /core/v4/auth"], "the old session is signed out")
|
||||
requireSession(t, srv, m, u)
|
||||
|
||||
// The old session no longer works.
|
||||
old["username"], old["password"] = "", ""
|
||||
_, err := NewFs(context.Background(), "test", "", old)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestShutdownKeepsFsUsable(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
f := newTestFs(t, sessionConfig(srv, u))
|
||||
o := putFile(t, f, "dir/a.txt", []byte("hello"))
|
||||
|
||||
require.NoError(t, f.Shutdown(ctx))
|
||||
require.NoError(t, f.Shutdown(ctx))
|
||||
got, err := readObject(t, o)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "hello", string(got))
|
||||
putFile(t, f, "dir/b.txt", []byte("world"))
|
||||
entries, err := f.List(ctx, "dir")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, entries, 2)
|
||||
o, err = f.NewObject(ctx, "dir/b.txt")
|
||||
require.NoError(t, err)
|
||||
got, err = readObject(t, o)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "world", string(got))
|
||||
}
|
||||
|
||||
func TestDisconnect(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
m := sessionConfig(srv, u)
|
||||
f := newTestFs(t, m)
|
||||
o := putFile(t, f, "a.txt", []byte("hello"))
|
||||
|
||||
require.NoError(t, f.Disconnect(ctx))
|
||||
assert.Empty(t, m["client_access_token"])
|
||||
assert.Empty(t, m["client_refresh_token"])
|
||||
|
||||
_, err := f.List(ctx, "")
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), `rclone config reconnect test:`)
|
||||
_, err = readObject(t, o)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), `rclone config reconnect test:`)
|
||||
_, err = f.Put(ctx, bytes.NewReader([]byte("x")), object.NewStaticObjectInfo("b.txt", time.Now(), 1, true, nil, nil))
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), `rclone config reconnect test:`)
|
||||
}
|
||||
|
||||
func TestSessionExpiredNamesRemote(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
m := sessionConfig(srv, u)
|
||||
f := newTestFs(t, m)
|
||||
f.c.s.adoptWait = 0
|
||||
srv.RevokeSession(m["client_uid"])
|
||||
|
||||
_, err := f.List(ctx, "")
|
||||
require.ErrorIs(t, err, errSessionExpired)
|
||||
assert.Contains(t, err.Error(), `rclone config reconnect test:`)
|
||||
assert.NotContains(t, err.Error(), "remote:")
|
||||
}
|
||||
|
||||
func TestTrashedBehindTheFsBack(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
f := newTestFs(t, sessionConfig(srv, u))
|
||||
other := newTestFs(t, sessionConfig(srv, u))
|
||||
|
||||
o := putFile(t, f, "a.txt", []byte("hello"))
|
||||
require.NoError(t, f.Mkdir(ctx, "dir"))
|
||||
_, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
trash := func(remote string) {
|
||||
otherObj, err := other.NewObject(ctx, remote)
|
||||
if err == nil {
|
||||
require.NoError(t, otherObj.Remove(ctx))
|
||||
return
|
||||
}
|
||||
require.NoError(t, other.Purge(ctx, remote))
|
||||
}
|
||||
trash("a.txt")
|
||||
trash("dir")
|
||||
|
||||
src := object.NewStaticObjectInfo("a.txt", time.Now(), 1, true, nil, nil)
|
||||
assert.Equal(t, fs.ErrorObjectNotFound, o.Update(ctx, bytes.NewReader([]byte("x")), src))
|
||||
_, err = f.Move(ctx, o, "b.txt")
|
||||
assert.Equal(t, fs.ErrorObjectNotFound, err)
|
||||
_, err = f.Copy(ctx, o, "c.txt")
|
||||
assert.Equal(t, fs.ErrorObjectNotFound, err)
|
||||
assert.Equal(t, fs.ErrorDirNotFound, f.DirMove(ctx, f, "dir", "dir2"))
|
||||
for _, d := range u.Dump() {
|
||||
assert.NotContains(t, []string{"b.txt", "c.txt", "dir2"}, d.Path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenRange(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
content := []byte("0123456789abcdefghijklmnopqrstuvwxyz")
|
||||
|
||||
// A file without extended attributes, whose plaintext size only
|
||||
// its content tells.
|
||||
seeded := u.SeedFile(u.RootID(), "tmp", content, time.Now())
|
||||
stored := u.StoredFile(seeded)
|
||||
stored.Revision.XAttr = ""
|
||||
m := sessionConfig(srv, u)
|
||||
cleanup := newTestFs(t, m)
|
||||
o, err := cleanup.NewObject(context.Background(), "tmp")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, o.Remove(context.Background()))
|
||||
u.SeedStoredFile(u.RootID(), stored)
|
||||
|
||||
for _, original := range []bool{true, false} {
|
||||
t.Run(fmt.Sprint("original_file_size=", original), func(t *testing.T) {
|
||||
m := sessionConfig(srv, u)
|
||||
m["original_file_size"] = fmt.Sprint(original)
|
||||
f := newTestFs(t, m)
|
||||
put := putFile(t, f, fmt.Sprintf("put-%v.txt", original), content)
|
||||
bare, err := f.NewObject(context.Background(), "tmp")
|
||||
require.NoError(t, err)
|
||||
for _, o := range []fs.Object{put, bare} {
|
||||
for _, test := range []struct {
|
||||
opt fs.OpenOption
|
||||
want string
|
||||
}{
|
||||
{&fs.RangeOption{Start: -1, End: 5}, "vwxyz"},
|
||||
{&fs.RangeOption{Start: 30, End: -1}, "uvwxyz"},
|
||||
{&fs.RangeOption{Start: 30, End: 1000}, "uvwxyz"},
|
||||
{&fs.RangeOption{Start: 2, End: 4}, "234"},
|
||||
{&fs.SeekOption{Offset: 33}, "xyz"},
|
||||
{&fs.RangeOption{Start: -1, End: 1000}, string(content)},
|
||||
} {
|
||||
got, err := readObject(t, o, test.opt)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, test.want, string(got), "%s %v", o.Remote(), test.opt)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestPutNestedReadBackLag puts a file into new nested directories
|
||||
// while the server takes a moment to show new nodes.
|
||||
func TestPutNestedReadBackLag(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
f := newTestFs(t, sessionConfig(srv, u))
|
||||
srv.SetReadLag(2, protontest.LagMissing)
|
||||
o := putFile(t, f, "A1/B1/C1/one", []byte("aaa"))
|
||||
assert.Equal(t, int64(3), o.Size())
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package protondrive_test
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/backend/protondrive"
|
||||
@@ -9,6 +10,13 @@ import (
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
// A golden capture runs with the real config, where this test would
|
||||
// write to and delete from the live remote if -run let it through.
|
||||
for _, name := range []string{"capture", "capture-remote"} {
|
||||
if f := flag.Lookup(name); f != nil && f.Value.String() != "" {
|
||||
t.Skipf("-%s is set: not running integration tests during a golden capture", name)
|
||||
}
|
||||
}
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestProtonDrive:",
|
||||
NilObject: (*protondrive.Object)(nil),
|
||||
|
||||
@@ -0,0 +1,340 @@
|
||||
package protontest
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/ProtonMail/go-srp"
|
||||
"github.com/ProtonMail/gopenpgp/v3/crypto"
|
||||
"github.com/rclone/rclone/backend/protondrive/api"
|
||||
)
|
||||
|
||||
// userKey is a user key with the password salt which locks it.
|
||||
type userKey struct {
|
||||
id string
|
||||
salt string // base64
|
||||
keyPassword string // derived from the mailbox password and salt
|
||||
armored string // private key locked with keyPassword
|
||||
key *crypto.Key
|
||||
}
|
||||
|
||||
// address is an email address of a user.
|
||||
type address struct {
|
||||
id string
|
||||
email string
|
||||
keys []*addressKey // the primary key is first
|
||||
}
|
||||
|
||||
// addressKey is an address key, locked with a random passphrase which
|
||||
// is stored encrypted to and signed by the primary user key.
|
||||
type addressKey struct {
|
||||
id string
|
||||
passphrase []byte
|
||||
armored string // private key locked with passphrase
|
||||
token string // armored passphrase encrypted to the user key
|
||||
signature string // armored detached signature of passphrase by the user key
|
||||
key *crypto.Key
|
||||
}
|
||||
|
||||
// mustCrypto panics with a message naming what failed if err is not nil.
|
||||
//
|
||||
// Key operations on keys the fake generated itself fail only on a bug.
|
||||
func mustCrypto(what string, err error) {
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("protontest: %s: %v", what, err))
|
||||
}
|
||||
}
|
||||
|
||||
// generateKey returns a new unlocked Curve25519 key for email, which
|
||||
// is fast to generate unlike RSA.
|
||||
func generateKey(name, email string) *crypto.Key {
|
||||
key, err := crypto.PGP().KeyGeneration().AddUserId(name, email).GenerationTime(signTime()).New().GenerateKey()
|
||||
mustCrypto("generate key", err)
|
||||
return key
|
||||
}
|
||||
|
||||
// lockKey returns key armored and locked with passphrase.
|
||||
func lockKey(key *crypto.Key, passphrase []byte) string {
|
||||
locked, err := crypto.PGP().LockKey(key, passphrase)
|
||||
mustCrypto("lock key", err)
|
||||
armored, err := locked.Armor()
|
||||
mustCrypto("armor key", err)
|
||||
return armored
|
||||
}
|
||||
|
||||
// newUserKey returns a new user key for u locked with a key password
|
||||
// derived from u's mailbox password and a new salt.
|
||||
func newUserKey(u *User) *userKey {
|
||||
rawSalt := randomBytes(16)
|
||||
hashed, err := srp.MailboxPassword([]byte(u.MailboxPassword), rawSalt)
|
||||
mustCrypto("hash mailbox password", err)
|
||||
k := &userKey{
|
||||
id: randomID(),
|
||||
salt: base64.StdEncoding.EncodeToString(rawSalt),
|
||||
keyPassword: string(hashed[len(hashed)-31:]),
|
||||
key: generateKey(u.Username, u.Username),
|
||||
}
|
||||
k.armored = lockKey(k.key, []byte(k.keyPassword))
|
||||
return k
|
||||
}
|
||||
|
||||
// newAddress returns a new address with one address key whose
|
||||
// passphrase is encrypted to userKey.
|
||||
func newAddress(email string, userKey *userKey) *address {
|
||||
passphrase := []byte(hex.EncodeToString(randomBytes(32)))
|
||||
key := generateKey(email, email)
|
||||
k := &addressKey{
|
||||
id: randomID(),
|
||||
passphrase: passphrase,
|
||||
armored: lockKey(key, passphrase),
|
||||
key: key,
|
||||
}
|
||||
k.sealPassphrase(userKey)
|
||||
return &address{id: randomID(), email: email, keys: []*addressKey{k}}
|
||||
}
|
||||
|
||||
// sealPassphrase sets the token and signature of k to its passphrase
|
||||
// encrypted to and signed by userKey, as Proton stores it.
|
||||
func (k *addressKey) sealPassphrase(userKey *userKey) {
|
||||
eh, err := crypto.PGP().Encryption().Recipient(userKey.key).New()
|
||||
mustCrypto("address key token", err)
|
||||
msg, err := eh.Encrypt(k.passphrase)
|
||||
mustCrypto("address key token", err)
|
||||
token, err := msg.ArmorBytes()
|
||||
mustCrypto("address key token", err)
|
||||
sh, err := crypto.PGP().Sign().SigningKey(userKey.key).Detached().SignTime(signTime()).New()
|
||||
mustCrypto("address key signature", err)
|
||||
sig, err := sh.Sign(k.passphrase, crypto.Armor)
|
||||
mustCrypto("address key signature", err)
|
||||
k.token, k.signature = string(token), string(sig)
|
||||
}
|
||||
|
||||
// KeyPassword returns the password which unlocks u's primary user key:
|
||||
// the last 31 characters of the bcrypt hash of the mailbox password
|
||||
// and the key salt, as a client derives it.
|
||||
func (u *User) KeyPassword() string {
|
||||
u.srv.mu.Lock()
|
||||
defer u.srv.mu.Unlock()
|
||||
return u.keyPassword
|
||||
}
|
||||
|
||||
// Email returns u's primary address.
|
||||
func (u *User) Email() string {
|
||||
u.srv.mu.Lock()
|
||||
defer u.srv.mu.Unlock()
|
||||
return u.addresses[0].email
|
||||
}
|
||||
|
||||
// AddAddress adds an address with a new address key to u.
|
||||
func (u *User) AddAddress(email string) {
|
||||
u.srv.mu.Lock()
|
||||
primary := u.keys[len(u.keys)-1]
|
||||
u.srv.mu.Unlock()
|
||||
addr := newAddress(email, primary)
|
||||
u.srv.mu.Lock()
|
||||
defer u.srv.mu.Unlock()
|
||||
u.addresses = append(u.addresses, addr)
|
||||
}
|
||||
|
||||
// AddUserKey gives u a new primary user key with a new salt, so
|
||||
// KeySalt and KeyPassword change.
|
||||
//
|
||||
// The old keys are listed before the new one and stay locked with
|
||||
// their old key password, as after a key password change on an account
|
||||
// with legacy keys. The address key passphrases are re-encrypted to the
|
||||
// new key.
|
||||
func (u *User) AddUserKey() {
|
||||
key := newUserKey(u)
|
||||
u.srv.mu.Lock()
|
||||
defer u.srv.mu.Unlock()
|
||||
u.keys = append(u.keys, key)
|
||||
u.KeySalt, u.keyPassword = key.salt, key.keyPassword
|
||||
for _, addr := range u.addresses {
|
||||
for _, k := range addr.keys {
|
||||
k.sealPassphrase(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// CorruptAddressKeySignature replaces the Signature of every key of u's
|
||||
// address email, as a malicious server would: with a valid signature
|
||||
// by an unrelated key, or with nothing if empty is true. AddUserKey
|
||||
// restores the signatures.
|
||||
func (s *Server) CorruptAddressKeySignature(u *User, email string, empty bool) {
|
||||
impostor := generateKey("impostor", "impostor@proton.test")
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
addr := u.addressByEmailLocked(email)
|
||||
if addr == nil {
|
||||
panic("protontest: no address " + email)
|
||||
}
|
||||
for _, k := range addr.keys {
|
||||
if empty {
|
||||
k.signature = ""
|
||||
continue
|
||||
}
|
||||
// Sign the real passphrase, so only the signer is wrong.
|
||||
sh, err := crypto.PGP().Sign().SigningKey(impostor).Detached().SignTime(signTime()).New()
|
||||
mustCrypto("impostor signature", err)
|
||||
sig, err := sh.Sign(k.passphrase, crypto.Armor)
|
||||
mustCrypto("impostor signature", err)
|
||||
k.signature = string(sig)
|
||||
}
|
||||
}
|
||||
|
||||
// EncryptToAddress returns plaintext as an armored message encrypted to
|
||||
// the primary key of u's address email.
|
||||
func (u *User) EncryptToAddress(email string, plaintext []byte) string {
|
||||
u.srv.mu.Lock()
|
||||
addr := u.addressByEmailLocked(email)
|
||||
u.srv.mu.Unlock()
|
||||
if addr == nil {
|
||||
panic("protontest: no address " + email)
|
||||
}
|
||||
eh, err := crypto.PGP().Encryption().Recipient(addr.keys[0].key).New()
|
||||
mustCrypto("encrypt to address", err)
|
||||
msg, err := eh.Encrypt(plaintext)
|
||||
mustCrypto("encrypt to address", err)
|
||||
armored, err := msg.ArmorBytes()
|
||||
mustCrypto("encrypt to address", err)
|
||||
return string(armored)
|
||||
}
|
||||
|
||||
// addressByEmailLocked returns u's address email or nil. srv.mu must
|
||||
// be held.
|
||||
func (u *User) addressByEmailLocked(email string) *address {
|
||||
for _, addr := range u.addresses {
|
||||
if addr.email == email {
|
||||
return addr
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// boolInt returns 1 for true and 0 for false, as the API encodes flags.
|
||||
func boolInt(b bool) int {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// registerAccount registers the user, address and public key routes.
|
||||
func (s *Server) registerAccount() {
|
||||
s.handleAuth("GET", "/core/v4/users", s.handleUser)
|
||||
s.handleAuth("GET", "/core/v4/addresses", s.handleAddresses)
|
||||
s.handleAuth("GET", "/core/v4/keys/all", s.handlePublicKeys)
|
||||
}
|
||||
|
||||
// userWithQuota is api.User with the quota fields of Proton's User
|
||||
// schema (coreTypes.ts).
|
||||
type userWithQuota struct {
|
||||
api.User
|
||||
UsedSpace int64
|
||||
MaxSpace int64
|
||||
ProductUsedSpace productUsedSpace
|
||||
}
|
||||
|
||||
// productUsedSpace is the UserUsage schema: used space per product.
|
||||
type productUsedSpace struct {
|
||||
Calendar int64
|
||||
Contact int64
|
||||
Drive int64
|
||||
Mail int64
|
||||
Pass int64
|
||||
Lumo int64
|
||||
}
|
||||
|
||||
// handleUser returns the user, its keys and its quota, all used space
|
||||
// being Drive's.
|
||||
func (s *Server) handleUser(w http.ResponseWriter, r *http.Request, u *User) {
|
||||
s.mu.Lock()
|
||||
user := userWithQuota{
|
||||
User: api.User{ID: u.ID, Name: u.Username, Email: u.addresses[0].email},
|
||||
MaxSpace: DefaultMaxSpace,
|
||||
}
|
||||
for i, key := range u.keys {
|
||||
user.Keys = append(user.Keys, api.UserKey{
|
||||
ID: key.id,
|
||||
Version: 3,
|
||||
PrivateKey: key.armored,
|
||||
Primary: boolInt(i == len(u.keys)-1),
|
||||
Active: 1,
|
||||
})
|
||||
}
|
||||
if u.drive != nil {
|
||||
user.UsedSpace = u.drive.usedSpace()
|
||||
user.ProductUsedSpace.Drive = user.UsedSpace
|
||||
}
|
||||
s.mu.Unlock()
|
||||
writeJSON(w, http.StatusOK, &struct {
|
||||
Code int
|
||||
User userWithQuota
|
||||
}{Code: api.CodeOK, User: user})
|
||||
}
|
||||
|
||||
// handleAddresses returns the user's addresses with their keys.
|
||||
func (s *Server) handleAddresses(w http.ResponseWriter, r *http.Request, u *User) {
|
||||
s.mu.Lock()
|
||||
resp := &api.AddressesResponse{Code: api.CodeOK}
|
||||
for i, addr := range u.addresses {
|
||||
out := api.Address{ID: addr.id, Email: addr.email, Status: 1, Type: 1, Priority: i + 1}
|
||||
for j, key := range addr.keys {
|
||||
out.Keys = append(out.Keys, api.AddressKey{
|
||||
ID: key.id,
|
||||
Version: 3,
|
||||
PrivateKey: key.armored,
|
||||
Token: key.token,
|
||||
Signature: key.signature,
|
||||
Primary: boolInt(j == 0),
|
||||
Active: 1,
|
||||
Flags: 3,
|
||||
})
|
||||
}
|
||||
resp.Addresses = append(resp.Addresses, out)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
// handlePublicKeys returns the public keys of the address in the Email
|
||||
// query parameter, or HTTP 422 code 33102 if no user has it.
|
||||
func (s *Server) handlePublicKeys(w http.ResponseWriter, r *http.Request, _ *User) {
|
||||
email := r.URL.Query().Get("Email")
|
||||
if email == "" {
|
||||
writeError(w, http.StatusBadRequest, api.CodeInvalidValue, "protontest: no Email")
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
var addr *address
|
||||
if owner := s.keysAs[email]; owner != nil {
|
||||
addr = owner.addresses[0]
|
||||
}
|
||||
for _, u := range s.users {
|
||||
if addr != nil {
|
||||
break
|
||||
}
|
||||
addr = u.addressByEmailLocked(email)
|
||||
}
|
||||
var keys []*addressKey
|
||||
if addr != nil {
|
||||
keys = addr.keys
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if addr == nil {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeAddressMissing, "protontest: address does not exist")
|
||||
return
|
||||
}
|
||||
resp := &api.PublicKeysResponse{Code: api.CodeOK}
|
||||
for i, key := range keys {
|
||||
pub, err := key.key.GetArmoredPublicKey()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, 0, "protontest: public key: "+err.Error())
|
||||
return
|
||||
}
|
||||
resp.Address.Keys = append(resp.Address.Keys, api.PublicKey{PublicKey: pub, Primary: boolInt(i == 0), Flags: 3})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
@@ -0,0 +1,555 @@
|
||||
package protontest
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/ProtonMail/go-srp"
|
||||
"github.com/pquerna/otp/totp"
|
||||
"github.com/rclone/rclone/backend/protondrive/api"
|
||||
)
|
||||
|
||||
// signedModulus is a 2048-bit SRP modulus signed by Proton's modulus
|
||||
// key, from go-srp's tests.
|
||||
const signedModulus = `-----BEGIN PGP SIGNED MESSAGE-----
|
||||
Hash: SHA256
|
||||
|
||||
W2z5HBi8RvsfYzZTS7qBaUxxPhsfHJFZpu3Kd6s1JafNrCCH9rfvPLrfuqocxWPgWDH2R8neK7PkNvjxto9TStuY5z7jAzWRvFWN9cQhAKkdWgy0JY6ywVn22+HFpF4cYesHrqFIKUPDMSSIlWjBVmEJZ/MusD44ZT29xcPrOqeZvwtCffKtGAIjLYPZIEbZKnDM1Dm3q2K/xS5h+xdhjnndhsrkwm9U9oyA2wxzSXFL+pdfj2fOdRwuR5nW0J2NFrq3kJjkRmpO/Genq1UW+TEknIWAb6VzJJJA244K/H8cnSx2+nSNZO3bbo6Ys228ruV9A8m6DhxmS+bihN3ttQ==
|
||||
-----BEGIN PGP SIGNATURE-----
|
||||
Version: ProtonMail
|
||||
Comment: https://protonmail.com
|
||||
|
||||
wl4EARYIABAFAlwB1j0JEDUFhcTpUY8mAAD8CgEAnsFnF4cF0uSHKkXa1GIa
|
||||
GO86yMV4zDZEZcDSJo0fgr8A/AlupGN9EdHlsrZLmTA1vhIx+rOgxdEff28N
|
||||
kvNM7qIK
|
||||
=q6vu
|
||||
-----END PGP SIGNATURE-----`
|
||||
|
||||
// srpBits is the bit length of the SRP modulus.
|
||||
const srpBits = 2048
|
||||
|
||||
// forkClientID is the client ID rclone's sign-in URL must carry.
|
||||
const forkClientID = "external-drive"
|
||||
|
||||
// User is an account on the fake server.
|
||||
type User struct {
|
||||
ID string
|
||||
Username string
|
||||
Password string // login password, checked with SRP
|
||||
MailboxPassword string // derives the key password
|
||||
KeySalt string // base64 salt of the primary user key's password
|
||||
|
||||
srv *Server
|
||||
srpSalt []byte
|
||||
|
||||
// Guarded by srv.mu once the user is added.
|
||||
keyPassword string
|
||||
keys []*userKey // the primary key is last
|
||||
addresses []*address // the primary address is first
|
||||
drive *drive // nil until the user has a volume
|
||||
|
||||
verifierOnce sync.Once
|
||||
verifier []byte
|
||||
verifierErr error
|
||||
|
||||
// Guarded by srv.mu.
|
||||
totpSecret string
|
||||
captcha bool
|
||||
corruptProof bool
|
||||
}
|
||||
|
||||
// srpVerifier returns the SRP verifier of u's login password, computing
|
||||
// it on first use since it costs a bcrypt hash.
|
||||
func (u *User) srpVerifier() ([]byte, error) {
|
||||
u.verifierOnce.Do(func() {
|
||||
var auth *srp.Auth
|
||||
auth, u.verifierErr = srp.NewAuthForVerifier([]byte(u.Password), signedModulus, u.srpSalt)
|
||||
if u.verifierErr == nil {
|
||||
u.verifier, u.verifierErr = auth.GenerateVerifier(srpBits)
|
||||
}
|
||||
})
|
||||
return u.verifier, u.verifierErr
|
||||
}
|
||||
|
||||
// EnableTOTP turns on TOTP two-factor authentication for u and returns
|
||||
// the base32 secret to generate codes from.
|
||||
func (u *User) EnableTOTP() (secret string) {
|
||||
key, err := totp.Generate(totp.GenerateOpts{Issuer: "protontest", AccountName: u.Username})
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("protontest: generate TOTP secret: %v", err))
|
||||
}
|
||||
u.srv.mu.Lock()
|
||||
defer u.srv.mu.Unlock()
|
||||
u.totpSecret = key.Secret()
|
||||
return u.totpSecret
|
||||
}
|
||||
|
||||
// srpLogin is an SRP challenge issued by POST /core/v4/auth/info.
|
||||
type srpLogin struct {
|
||||
user *User
|
||||
server *srp.Server
|
||||
}
|
||||
|
||||
// fork is a session fork requested by GET /auth/v4/sessions/forks.
|
||||
type fork struct {
|
||||
userCode string
|
||||
key []byte // encryption key from the sign-in URL, once approved
|
||||
approved *session // child session, once approved
|
||||
}
|
||||
|
||||
// Session holds the tokens of a signed in session.
|
||||
type Session struct {
|
||||
UID string
|
||||
AccessToken string
|
||||
RefreshToken string
|
||||
}
|
||||
|
||||
// session is the server side state of a Session.
|
||||
type session struct {
|
||||
Session
|
||||
user *User
|
||||
expired bool // access token no longer accepted
|
||||
locked bool // waiting for the second factor
|
||||
}
|
||||
|
||||
// randomBytes returns n random bytes.
|
||||
func randomBytes(n int) []byte {
|
||||
b := make([]byte, n)
|
||||
_, _ = rand.Read(b)
|
||||
return b
|
||||
}
|
||||
|
||||
// randomID returns a random hex string.
|
||||
func randomID() string {
|
||||
return hex.EncodeToString(randomBytes(16))
|
||||
}
|
||||
|
||||
// registerAuth registers the session routes.
|
||||
func (s *Server) registerAuth() {
|
||||
s.handle("POST", "/auth/v4/refresh", s.handleRefresh)
|
||||
s.handle("POST", "/core/v4/auth/info", s.handleAuthInfo)
|
||||
s.handle("POST", "/core/v4/auth", s.handleLogin)
|
||||
s.handleSession("POST", "/core/v4/auth/2fa", s.handle2FA)
|
||||
s.handleSession("DELETE", "/core/v4/auth", s.handleLogout)
|
||||
s.handleAuth("GET", "/core/v4/keys/salts", s.handleKeySalts)
|
||||
s.handle("GET", "/auth/v4/sessions/forks", s.handleForkInit)
|
||||
s.handle("GET", "/auth/v4/sessions/forks/{selector}", s.handleForkStatus)
|
||||
s.handleAuth("GET", "/tests/auth", func(w http.ResponseWriter, r *http.Request, u *User) {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"Code": api.CodeOK, "UserID": u.ID})
|
||||
})
|
||||
s.handleAuth("POST", "/tests/echo", func(w http.ResponseWriter, r *http.Request, u *User) {
|
||||
var body struct{ Echo string }
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
writeError(w, http.StatusBadRequest, api.CodeInvalidValue, "protontest: "+err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"Code": api.CodeOK, "Echo": body.Echo})
|
||||
})
|
||||
}
|
||||
|
||||
// AddUser creates an account whose mailbox password is its password.
|
||||
func (s *Server) AddUser(username, password string) *User {
|
||||
return s.AddTwoPasswordUser(username, password, password)
|
||||
}
|
||||
|
||||
// AddTwoPasswordUser creates an account which signs in with password
|
||||
// and unlocks its keys with mailboxPassword.
|
||||
//
|
||||
// The account has a user key and one address, username itself if it
|
||||
// contains "@" or else username@proton.test, with an address key. Its
|
||||
// drive has a volume, a main share and an empty root folder.
|
||||
func (s *Server) AddTwoPasswordUser(username, password, mailboxPassword string) *User {
|
||||
u := newUser(s, username, password, mailboxPassword)
|
||||
u.drive = s.newDrive(u.addresses[0])
|
||||
s.addUser(u)
|
||||
return u
|
||||
}
|
||||
|
||||
// AddUserWithoutDrive creates an account like AddUser but without a
|
||||
// volume, as a brand-new Proton account is: its my-files route answers
|
||||
// HTTP 422 code 2501 until a client creates the volume.
|
||||
func (s *Server) AddUserWithoutDrive(username, password string) *User {
|
||||
u := newUser(s, username, password, password)
|
||||
s.addUser(u)
|
||||
return u
|
||||
}
|
||||
|
||||
// newUser returns an account with a user key and one address, not yet
|
||||
// added to s.
|
||||
func newUser(s *Server, username, password, mailboxPassword string) *User {
|
||||
u := &User{
|
||||
ID: randomID(),
|
||||
Username: username,
|
||||
Password: password,
|
||||
MailboxPassword: mailboxPassword,
|
||||
srv: s,
|
||||
srpSalt: randomBytes(10),
|
||||
}
|
||||
key := newUserKey(u)
|
||||
u.keys = []*userKey{key}
|
||||
u.KeySalt, u.keyPassword = key.salt, key.keyPassword
|
||||
email := username
|
||||
if !strings.Contains(email, "@") {
|
||||
email += "@proton.test"
|
||||
}
|
||||
u.addresses = []*address{newAddress(email, key)}
|
||||
return u
|
||||
}
|
||||
|
||||
// addUser adds u to s.
|
||||
func (s *Server) addUser(u *User) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.users[u.Username] = u
|
||||
}
|
||||
|
||||
// RequireCaptcha makes password sign-in for u fail with HTTP 422 code
|
||||
// 9001, as when Proton asks for human verification.
|
||||
func (s *Server) RequireCaptcha(u *User) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
u.captcha = true
|
||||
}
|
||||
|
||||
// CorruptServerProof makes password sign-in for u return a server
|
||||
// proof with one byte changed, as an impostor server would.
|
||||
func (s *Server) CorruptServerProof(u *User) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
u.corruptProof = true
|
||||
}
|
||||
|
||||
// NewSession signs u in and returns the new session's tokens.
|
||||
func (s *Server) NewSession(u *User) Session {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.newSessionLocked(u, false).Session
|
||||
}
|
||||
|
||||
// newSessionLocked creates a session for u. s.mu must be held.
|
||||
func (s *Server) newSessionLocked(u *User, locked bool) *session {
|
||||
sess := &session{
|
||||
Session: Session{UID: randomID(), AccessToken: s.newSecretLocked(SessionToken), RefreshToken: s.newSecretLocked(SessionToken)},
|
||||
user: u,
|
||||
locked: locked,
|
||||
}
|
||||
s.sessions[sess.UID] = sess
|
||||
return sess
|
||||
}
|
||||
|
||||
// ApproveFork completes the session fork whose sign-in URL is
|
||||
// signInURL as if u had signed in on Proton's page: the next poll of
|
||||
// the fork returns a new session for u and u's key password encrypted
|
||||
// with the key from the URL.
|
||||
func (s *Server) ApproveFork(signInURL string, u *User) error {
|
||||
parsed, err := url.Parse(signInURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("protontest: parse sign-in URL: %w", err)
|
||||
}
|
||||
payload, ok := strings.CutPrefix(parsed.Fragment, "payload=")
|
||||
parts := strings.Split(payload, ":")
|
||||
if !ok || len(parts) != 4 || parts[0] != "0" || parts[3] != forkClientID {
|
||||
return fmt.Errorf("protontest: bad sign-in URL payload %q", payload)
|
||||
}
|
||||
key, err := base64.StdEncoding.DecodeString(parts[2])
|
||||
if err != nil || len(key) != 32 {
|
||||
return fmt.Errorf("protontest: bad key in sign-in URL payload %q", payload)
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, f := range s.forks {
|
||||
if f.userCode == parts[1] {
|
||||
f.key = key
|
||||
f.approved = s.newSessionLocked(u, false)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("protontest: no fork with user code %q", parts[1])
|
||||
}
|
||||
|
||||
// ExpireAccessToken makes the access token of session uid return 401
|
||||
// until the session is refreshed.
|
||||
func (s *Server) ExpireAccessToken(uid string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if sess := s.sessions[uid]; sess != nil {
|
||||
sess.expired = true
|
||||
}
|
||||
}
|
||||
|
||||
// RevokeSession ends session uid, so neither its access token nor its
|
||||
// refresh token is accepted.
|
||||
func (s *Server) RevokeSession(uid string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
delete(s.sessions, uid)
|
||||
}
|
||||
|
||||
// handleAuth registers h for an authenticated route. Requests without
|
||||
// a valid x-pm-uid and bearer access token get HTTP 401, and requests
|
||||
// from a session waiting for its second factor get HTTP 403.
|
||||
func (s *Server) handleAuth(method, pattern string, h func(w http.ResponseWriter, r *http.Request, u *User)) {
|
||||
s.handleSession(method, pattern, func(w http.ResponseWriter, r *http.Request, sess *session) {
|
||||
s.mu.Lock()
|
||||
locked := sess.locked
|
||||
s.mu.Unlock()
|
||||
if locked {
|
||||
writeError(w, http.StatusForbidden, 0, "protontest: second factor required")
|
||||
return
|
||||
}
|
||||
h(w, r, sess.user)
|
||||
})
|
||||
}
|
||||
|
||||
// handleSession registers h for a route which needs a session, which
|
||||
// may be waiting for its second factor. Requests without a valid
|
||||
// x-pm-uid and bearer access token get HTTP 401.
|
||||
func (s *Server) handleSession(method, pattern string, h func(w http.ResponseWriter, r *http.Request, sess *session)) {
|
||||
s.handle(method, pattern, func(w http.ResponseWriter, r *http.Request) {
|
||||
token, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
|
||||
s.mu.Lock()
|
||||
sess := s.sessions[r.Header.Get("x-pm-uid")]
|
||||
valid := ok && sess != nil && !sess.expired && sess.AccessToken == token
|
||||
s.mu.Unlock()
|
||||
if !valid {
|
||||
writeError(w, http.StatusUnauthorized, 401, "protontest: invalid access token")
|
||||
return
|
||||
}
|
||||
h(w, r, sess)
|
||||
})
|
||||
}
|
||||
|
||||
// handleRefresh rotates the tokens of the session named by x-pm-uid.
|
||||
// Refresh tokens are single use.
|
||||
func (s *Server) handleRefresh(w http.ResponseWriter, r *http.Request) {
|
||||
var req api.AuthRefreshRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.ResponseType != "token" || req.GrantType != "refresh_token" {
|
||||
writeError(w, http.StatusBadRequest, api.CodeInvalidValue, "protontest: bad refresh request")
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
sess := s.sessions[r.Header.Get("x-pm-uid")]
|
||||
if sess == nil || req.RefreshToken == "" || sess.RefreshToken != req.RefreshToken {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeRefreshTokenInvalid, "protontest: invalid refresh token")
|
||||
return
|
||||
}
|
||||
sess.AccessToken = s.newSecretLocked(SessionToken)
|
||||
sess.RefreshToken = s.newSecretLocked(SessionToken)
|
||||
sess.expired = false
|
||||
writeJSON(w, http.StatusOK, &api.AuthRefreshResponse{
|
||||
Code: api.CodeOK,
|
||||
UID: sess.UID,
|
||||
AccessToken: sess.AccessToken,
|
||||
RefreshToken: sess.RefreshToken,
|
||||
})
|
||||
}
|
||||
|
||||
// handleAuthInfo issues an SRP challenge for the user named in the
|
||||
// request.
|
||||
func (s *Server) handleAuthInfo(w http.ResponseWriter, r *http.Request) {
|
||||
var req api.AuthInfoRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Intent != "Proton" {
|
||||
writeError(w, http.StatusBadRequest, api.CodeInvalidValue, "protontest: bad auth info request")
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
u := s.users[req.Username]
|
||||
s.mu.Unlock()
|
||||
if u == nil {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodePasswordWrong, "protontest: unknown user")
|
||||
return
|
||||
}
|
||||
verifier, err := u.srpVerifier()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, 0, "protontest: SRP verifier: "+err.Error())
|
||||
return
|
||||
}
|
||||
server, err := srp.NewServerFromSigned(signedModulus, verifier, srpBits)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, 0, "protontest: SRP server: "+err.Error())
|
||||
return
|
||||
}
|
||||
challenge, err := server.GenerateChallenge()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, 0, "protontest: SRP challenge: "+err.Error())
|
||||
return
|
||||
}
|
||||
srpSession := randomID()
|
||||
s.mu.Lock()
|
||||
s.srp[srpSession] = &srpLogin{user: u, server: server}
|
||||
s.mu.Unlock()
|
||||
writeJSON(w, http.StatusOK, &api.AuthInfoResponse{
|
||||
Code: api.CodeOK,
|
||||
Version: 4,
|
||||
Modulus: signedModulus,
|
||||
ServerEphemeral: base64.StdEncoding.EncodeToString(challenge),
|
||||
Salt: base64.StdEncoding.EncodeToString(u.srpSalt),
|
||||
SRPSession: srpSession,
|
||||
Username: u.Username,
|
||||
})
|
||||
}
|
||||
|
||||
// handleLogin checks the SRP proof for a challenge and creates a
|
||||
// session, locked until the second factor if the user has TOTP.
|
||||
// Challenges are single use.
|
||||
func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) {
|
||||
var req api.AuthRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, api.CodeInvalidValue, "protontest: bad auth request")
|
||||
return
|
||||
}
|
||||
ephemeral, err1 := base64.StdEncoding.DecodeString(req.ClientEphemeral)
|
||||
proof, err2 := base64.StdEncoding.DecodeString(req.ClientProof)
|
||||
if err := errors.Join(err1, err2); err != nil {
|
||||
writeError(w, http.StatusBadRequest, api.CodeInvalidValue, "protontest: bad auth request: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
login := s.srp[req.SRPSession]
|
||||
delete(s.srp, req.SRPSession)
|
||||
if login == nil || login.user.Username != req.Username {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidValue, "protontest: unknown SRP session")
|
||||
return
|
||||
}
|
||||
u := login.user
|
||||
if u.captcha {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeHumanVerification, "protontest: human verification required")
|
||||
return
|
||||
}
|
||||
serverProof, err := login.server.VerifyProofs(ephemeral, proof)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodePasswordWrong, "protontest: incorrect login credentials")
|
||||
return
|
||||
}
|
||||
if u.corruptProof {
|
||||
serverProof[0] ^= 1
|
||||
}
|
||||
resp := &api.AuthResponse{
|
||||
Code: api.CodeOK,
|
||||
UserID: u.ID,
|
||||
TokenType: "Bearer",
|
||||
ServerProof: base64.StdEncoding.EncodeToString(serverProof),
|
||||
Scopes: []string{"full"},
|
||||
PasswordMode: api.PasswordModeOne,
|
||||
}
|
||||
if u.MailboxPassword != u.Password {
|
||||
resp.PasswordMode = api.PasswordModeTwo
|
||||
}
|
||||
if u.totpSecret != "" {
|
||||
resp.TwoFA.Enabled = api.TwoFATOTP
|
||||
resp.Scopes = []string{"twofactor"}
|
||||
}
|
||||
sess := s.newSessionLocked(u, u.totpSecret != "")
|
||||
resp.UID, resp.AccessToken, resp.RefreshToken = sess.UID, sess.AccessToken, sess.RefreshToken
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
// handle2FA unlocks a session with a TOTP code.
|
||||
func (s *Server) handle2FA(w http.ResponseWriter, r *http.Request, sess *session) {
|
||||
var req api.Auth2FARequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, api.CodeInvalidValue, "protontest: bad 2FA request")
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
secret := sess.user.totpSecret
|
||||
if secret == "" || !totp.Validate(req.TwoFactorCode, secret) {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidValue, "protontest: incorrect 2FA code")
|
||||
return
|
||||
}
|
||||
sess.locked = false
|
||||
writeJSON(w, http.StatusOK, map[string]any{"Code": api.CodeOK, "Scopes": []string{"full"}})
|
||||
}
|
||||
|
||||
// handleLogout ends the session.
|
||||
func (s *Server) handleLogout(w http.ResponseWriter, r *http.Request, sess *session) {
|
||||
s.mu.Lock()
|
||||
delete(s.sessions, sess.UID)
|
||||
s.mu.Unlock()
|
||||
writeJSON(w, http.StatusOK, map[string]any{"Code": api.CodeOK})
|
||||
}
|
||||
|
||||
// handleKeySalts returns the password salt of each user key.
|
||||
func (s *Server) handleKeySalts(w http.ResponseWriter, r *http.Request, u *User) {
|
||||
resp := &api.KeySaltsResponse{Code: api.CodeOK}
|
||||
s.mu.Lock()
|
||||
for _, key := range u.keys {
|
||||
resp.KeySalts = append(resp.KeySalts, api.KeySalt{ID: key.id, KeySalt: key.salt})
|
||||
}
|
||||
s.mu.Unlock()
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
// handleForkInit starts a session fork for a client to sign in with.
|
||||
func (s *Server) handleForkInit(w http.ResponseWriter, r *http.Request) {
|
||||
f := &fork{userCode: strings.ToUpper(hex.EncodeToString(randomBytes(4)))}
|
||||
s.mu.Lock()
|
||||
selector := s.newSecretLocked(ForkSelector)
|
||||
s.forks[selector] = f
|
||||
s.mu.Unlock()
|
||||
writeJSON(w, http.StatusOK, &api.ForkInitResponse{Code: api.CodeOK, Selector: selector, UserCode: f.userCode})
|
||||
}
|
||||
|
||||
// handleForkStatus returns the child session of an approved fork, once,
|
||||
// or HTTP 422 while the fork is waiting for approval.
|
||||
func (s *Server) handleForkStatus(w http.ResponseWriter, r *http.Request) {
|
||||
selector := r.PathValue("selector")
|
||||
s.mu.Lock()
|
||||
f := s.forks[selector]
|
||||
if f == nil || f.approved == nil {
|
||||
s.mu.Unlock()
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidValue, "protontest: fork not ready")
|
||||
return
|
||||
}
|
||||
delete(s.forks, selector)
|
||||
sess := f.approved.Session
|
||||
user := f.approved.user
|
||||
s.mu.Unlock()
|
||||
|
||||
payload, err := sealForkPayload(f.key, user.KeyPassword())
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, 0, "protontest: fork payload: "+err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, &api.ForkStatusResponse{
|
||||
Code: api.CodeOK,
|
||||
Payload: payload,
|
||||
UID: sess.UID,
|
||||
AccessToken: sess.AccessToken,
|
||||
RefreshToken: sess.RefreshToken,
|
||||
})
|
||||
}
|
||||
|
||||
// sealForkPayload encrypts keyPassword for a fork as Proton's sign-in
|
||||
// page does: base64 of nonce, AES-256-GCM ciphertext and tag, with
|
||||
// additional data "fork".
|
||||
func sealForkPayload(key []byte, keyPassword string) (string, error) {
|
||||
plain, err := json.Marshal(map[string]string{"keyPassword": keyPassword})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
block, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
nonce := randomBytes(gcm.NonceSize())
|
||||
return base64.StdEncoding.EncodeToString(gcm.Seal(nonce, nonce, plain, []byte("fork"))), nil
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package protontest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ProtonMail/go-srp"
|
||||
"github.com/pquerna/otp/totp"
|
||||
"github.com/rclone/rclone/backend/protondrive/api"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// refresh calls POST /auth/v4/refresh for uid with refreshToken.
|
||||
func refresh(t *testing.T, s *Server, uid, refreshToken string) (*http.Response, []byte) {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(&api.AuthRefreshRequest{ResponseType: "token", GrantType: "refresh_token", RefreshToken: refreshToken})
|
||||
require.NoError(t, err)
|
||||
req, err := http.NewRequest("POST", s.URL()+"/auth/v4/refresh", bytes.NewReader(body))
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("x-pm-appversion", testAppVersion)
|
||||
req.Header.Set("x-pm-uid", uid)
|
||||
resp, err := s.Client().Do(req)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
out, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
return resp, out
|
||||
}
|
||||
|
||||
func authed(t *testing.T, s *Server, sess Session) int {
|
||||
t.Helper()
|
||||
resp, _ := do(t, s, "GET", "/tests/auth", map[string]string{
|
||||
"x-pm-appversion": testAppVersion,
|
||||
"x-pm-uid": sess.UID,
|
||||
"Authorization": "Bearer " + sess.AccessToken,
|
||||
})
|
||||
return resp.StatusCode
|
||||
}
|
||||
|
||||
func TestAuth(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
assert.Equal(t, "pass", u.MailboxPassword)
|
||||
sess := s.NewSession(u)
|
||||
|
||||
assert.Equal(t, http.StatusOK, authed(t, s, sess))
|
||||
assert.Equal(t, http.StatusUnauthorized, authed(t, s, Session{UID: sess.UID, AccessToken: "wrong"}))
|
||||
assert.Equal(t, http.StatusUnauthorized, authed(t, s, Session{UID: "unknown", AccessToken: sess.AccessToken}))
|
||||
|
||||
s.ExpireAccessToken(sess.UID)
|
||||
assert.Equal(t, http.StatusUnauthorized, authed(t, s, sess))
|
||||
|
||||
resp, body := refresh(t, s, sess.UID, sess.RefreshToken)
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode, "body: %s", body)
|
||||
var out api.AuthRefreshResponse
|
||||
require.NoError(t, json.Unmarshal(body, &out))
|
||||
assert.Equal(t, sess.UID, out.UID)
|
||||
next := Session{UID: out.UID, AccessToken: out.AccessToken, RefreshToken: out.RefreshToken}
|
||||
assert.Equal(t, http.StatusOK, authed(t, s, next))
|
||||
assert.Equal(t, http.StatusUnauthorized, authed(t, s, sess), "old access token")
|
||||
|
||||
// Refresh tokens are single use.
|
||||
resp, body = refresh(t, s, sess.UID, sess.RefreshToken)
|
||||
assert.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode)
|
||||
assert.Equal(t, api.CodeRefreshTokenInvalid, decodeError(t, body).Code)
|
||||
|
||||
s.RevokeSession(sess.UID)
|
||||
assert.Equal(t, http.StatusUnauthorized, authed(t, s, next))
|
||||
resp, body = refresh(t, s, next.UID, next.RefreshToken)
|
||||
assert.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode)
|
||||
assert.Equal(t, api.CodeRefreshTokenInvalid, decodeError(t, body).Code)
|
||||
}
|
||||
|
||||
// postJSON sends in as JSON to path with the given extra headers and
|
||||
// decodes the response into out, returning the HTTP status.
|
||||
func postJSON(t *testing.T, s *Server, path string, header map[string]string, in, out any) int {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(in)
|
||||
require.NoError(t, err)
|
||||
req, err := http.NewRequest("POST", s.URL()+path, bytes.NewReader(body))
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("x-pm-appversion", testAppVersion)
|
||||
for k, v := range header {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
resp, err := s.Client().Do(req)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
require.NoError(t, json.NewDecoder(resp.Body).Decode(out))
|
||||
return resp.StatusCode
|
||||
}
|
||||
|
||||
func TestLoginTOTPLocksSession(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
secret := u.EnableTOTP()
|
||||
|
||||
var info api.AuthInfoResponse
|
||||
require.Equal(t, http.StatusOK, postJSON(t, s, "/core/v4/auth/info", nil, &api.AuthInfoRequest{Intent: "Proton", Username: "user"}, &info))
|
||||
auth, err := srp.NewAuth(info.Version, "user", []byte("pass"), info.Salt, info.Modulus, info.ServerEphemeral)
|
||||
require.NoError(t, err)
|
||||
proofs, err := auth.GenerateProofs(srpBits)
|
||||
require.NoError(t, err)
|
||||
var out api.AuthResponse
|
||||
require.Equal(t, http.StatusOK, postJSON(t, s, "/core/v4/auth", nil, &api.AuthRequest{
|
||||
Username: "user",
|
||||
ClientEphemeral: base64.StdEncoding.EncodeToString(proofs.ClientEphemeral),
|
||||
ClientProof: base64.StdEncoding.EncodeToString(proofs.ClientProof),
|
||||
SRPSession: info.SRPSession,
|
||||
}, &out))
|
||||
assert.Equal(t, base64.StdEncoding.EncodeToString(proofs.ExpectedServerProof), out.ServerProof)
|
||||
assert.Equal(t, api.TwoFATOTP, out.TwoFA.Enabled)
|
||||
sess := Session{UID: out.UID, AccessToken: out.AccessToken, RefreshToken: out.RefreshToken}
|
||||
assert.Equal(t, http.StatusForbidden, authed(t, s, sess))
|
||||
|
||||
headers := map[string]string{"x-pm-uid": sess.UID, "Authorization": "Bearer " + sess.AccessToken}
|
||||
var res api.Error
|
||||
assert.Equal(t, http.StatusUnprocessableEntity, postJSON(t, s, "/core/v4/auth/2fa", headers, &api.Auth2FARequest{TwoFactorCode: "000000x"}, &res))
|
||||
assert.Equal(t, http.StatusForbidden, authed(t, s, sess))
|
||||
|
||||
code, err := totp.GenerateCode(secret, time.Now())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, http.StatusOK, postJSON(t, s, "/core/v4/auth/2fa", headers, &api.Auth2FARequest{TwoFactorCode: code}, &res))
|
||||
assert.Equal(t, http.StatusOK, authed(t, s, sess))
|
||||
|
||||
// SRP challenges are single use.
|
||||
assert.Equal(t, http.StatusUnprocessableEntity, postJSON(t, s, "/core/v4/auth", nil, &api.AuthRequest{
|
||||
Username: "user",
|
||||
ClientEphemeral: base64.StdEncoding.EncodeToString(proofs.ClientEphemeral),
|
||||
ClientProof: base64.StdEncoding.EncodeToString(proofs.ClientProof),
|
||||
SRPSession: info.SRPSession,
|
||||
}, &res))
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
// Command protonfake serves a fake Proton Drive API for integration
|
||||
// tests.
|
||||
//
|
||||
// It prints the rclone config of a signed in user with a Drive volume
|
||||
// as key=value lines on stdout, then serves until interrupted.
|
||||
//
|
||||
// go run ./backend/protondrive/protontest/cmd/protonfake -addr 127.0.0.1:0
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/rclone/rclone/backend/protondrive/protontest"
|
||||
)
|
||||
|
||||
func main() {
|
||||
addr := flag.String("addr", "127.0.0.1:0", "address to listen on")
|
||||
flag.Parse()
|
||||
stop := make(chan struct{})
|
||||
sig := make(chan os.Signal, 1)
|
||||
signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
|
||||
go func() {
|
||||
<-sig
|
||||
close(stop)
|
||||
}()
|
||||
if err := run(*addr, os.Stdout, stop); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "protonfake:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// run serves a fake server on addr until stop is closed, after writing
|
||||
// its endpoint and a fresh session's config to out.
|
||||
func run(addr string, out io.Writer, stop <-chan struct{}) error {
|
||||
srv, err := protontest.NewStandalone(addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer srv.Close()
|
||||
user := srv.AddUser("protonfake", "protonfake")
|
||||
sess := srv.NewSession(user)
|
||||
// The config holds the salted key password base64 encoded.
|
||||
keyPass := base64.StdEncoding.EncodeToString([]byte(user.KeyPassword()))
|
||||
_, err = fmt.Fprintf(out, "endpoint=%s\nclient_uid=%s\nclient_access_token=%s\nclient_refresh_token=%s\nclient_salted_key_pass=%s\ntype=protondrive\n",
|
||||
srv.URL(), sess.UID, sess.AccessToken, sess.RefreshToken, keyPass)
|
||||
if err != nil {
|
||||
return fmt.Errorf("write config: %w", err)
|
||||
}
|
||||
<-stop
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRun(t *testing.T) {
|
||||
pr, pw := io.Pipe()
|
||||
stop := make(chan struct{})
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
err := run("127.0.0.1:0", pw, stop)
|
||||
_ = pw.Close()
|
||||
done <- err
|
||||
}()
|
||||
|
||||
wantKeys := []string{"endpoint", "client_uid", "client_access_token", "client_refresh_token", "client_salted_key_pass", "type"}
|
||||
values := make(map[string]string)
|
||||
lines := bufio.NewScanner(pr)
|
||||
for _, key := range wantKeys {
|
||||
require.True(t, lines.Scan(), "missing line for %s", key)
|
||||
k, v, ok := strings.Cut(lines.Text(), "=")
|
||||
require.True(t, ok, "line %q is not key=value", lines.Text())
|
||||
require.Equal(t, key, k)
|
||||
require.NotEmpty(t, v, key)
|
||||
values[k] = v
|
||||
}
|
||||
assert.Equal(t, "protondrive", values["type"])
|
||||
assert.False(t, strings.HasSuffix(values["endpoint"], ":0"), "endpoint %q must carry the bound port", values["endpoint"])
|
||||
keyPass, err := base64.StdEncoding.DecodeString(values["client_salted_key_pass"])
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, keyPass, 31, "a key password is the last 31 characters of a bcrypt hash")
|
||||
|
||||
req, err := http.NewRequest("GET", values["endpoint"]+"/core/v4/users", nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("x-pm-appversion", "external-drive-rclone@1.0.0-stable")
|
||||
req.Header.Set("x-pm-uid", values["client_uid"])
|
||||
req.Header.Set("Authorization", "Bearer "+values["client_access_token"])
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
require.NoError(t, err)
|
||||
_ = resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
|
||||
close(stop)
|
||||
extra := lines.Scan()
|
||||
assert.False(t, extra, "unexpected output %q", lines.Text())
|
||||
require.NoError(t, <-done)
|
||||
_, err = http.Get(values["endpoint"] + "/tests/ping")
|
||||
assert.Error(t, err, "server still answering after stop")
|
||||
}
|
||||
|
||||
func TestRunBadAddress(t *testing.T) {
|
||||
var out strings.Builder
|
||||
err := run("not an address", &out, make(chan struct{}))
|
||||
assert.Error(t, err)
|
||||
assert.Empty(t, out.String())
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
package protontest
|
||||
|
||||
// The Drive cryptography of the fake, written from Proton's Drive SDK
|
||||
// (client/js/src/crypto/driveCrypto.ts and the nodes, shares and upload
|
||||
// crypto services) independently of the protondrive backend, so a bug
|
||||
// in one is unlikely to be mirrored in the other.
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/ProtonMail/go-crypto/openpgp/packet"
|
||||
"github.com/ProtonMail/gopenpgp/v3/constants"
|
||||
"github.com/ProtonMail/gopenpgp/v3/crypto"
|
||||
"github.com/ProtonMail/gopenpgp/v3/profile"
|
||||
)
|
||||
|
||||
// driveKeyName is the user ID of every node and share key, as the SDK
|
||||
// generates them.
|
||||
const driveKeyName = "Drive key"
|
||||
|
||||
// aeadPGP returns a handle whose keys advertise SEIPDv2 and whose
|
||||
// session-key encryption uses AES-256 GCM, as the SDK's file content
|
||||
// does (openpgp.js aeadProtect with Proton's GCM preference).
|
||||
//
|
||||
// Everything that is not file content goes through encryptArmored,
|
||||
// which never produces SEIPDv2 whatever the recipient advertises.
|
||||
func aeadPGP() *crypto.PGPHandle {
|
||||
p := profile.Default()
|
||||
p.AeadEncryption = &packet.AEADConfig{DefaultMode: packet.AEADModeGCM}
|
||||
return crypto.PGPWithProfile(p)
|
||||
}
|
||||
|
||||
// signTime returns when the fake generates keys and makes signatures: a
|
||||
// day before the local clock, so that what it writes is valid at the
|
||||
// server clock whatever skew within a day a test sets.
|
||||
func signTime() int64 {
|
||||
return time.Now().Add(-24 * time.Hour).Unix()
|
||||
}
|
||||
|
||||
// signTimeOf returns when the fake signs with signer: signTime, or
|
||||
// signer's creation time if later, as for a key a test supplied which
|
||||
// was generated less than a day ago.
|
||||
func signTimeOf(signer *crypto.Key) int64 {
|
||||
return max(signTime(), signer.GetEntity().PrimaryKey.CreationTime.Unix())
|
||||
}
|
||||
|
||||
// newPassphrase returns base64 of 32 random bytes, the SDK's
|
||||
// generatePassphrase.
|
||||
func newPassphrase() []byte {
|
||||
return []byte(base64.StdEncoding.EncodeToString(randomBytes(32)))
|
||||
}
|
||||
|
||||
// lockedKey is a generated key with its passphrase and armored locked
|
||||
// form.
|
||||
type lockedKey struct {
|
||||
key *crypto.Key
|
||||
passphrase []byte
|
||||
armored string
|
||||
}
|
||||
|
||||
// newDriveKey generates a node or share key locked with a new
|
||||
// passphrase. aead makes the key advertise SEIPDv2, as file node keys
|
||||
// do; folder and share keys do not.
|
||||
func newDriveKey(aead bool) lockedKey {
|
||||
pgp := crypto.PGP()
|
||||
if aead {
|
||||
pgp = aeadPGP()
|
||||
}
|
||||
key, err := pgp.KeyGeneration().AddUserId(driveKeyName, "").GenerationTime(signTime()).New().GenerateKey()
|
||||
mustCrypto("generate drive key", err)
|
||||
passphrase := newPassphrase()
|
||||
return lockedKey{key: key, passphrase: passphrase, armored: lockKey(key, passphrase)}
|
||||
}
|
||||
|
||||
// sealedPassphrase is a node or share passphrase as the server stores it.
|
||||
type sealedPassphrase struct {
|
||||
armored string // encrypted to the parent key
|
||||
signature string // armored detached signature of the plaintext
|
||||
}
|
||||
|
||||
// sealPassphrase encrypts passphrase to parent without AEAD and signs it
|
||||
// with signer in a detached signature, the SDK's encryptPassphrase.
|
||||
func sealPassphrase(passphrase []byte, parent, signer *crypto.Key) sealedPassphrase {
|
||||
return sealedPassphrase{
|
||||
armored: encryptArmored(passphrase, parent, nil, false),
|
||||
signature: signArmored(passphrase, signer),
|
||||
}
|
||||
}
|
||||
|
||||
// encryptArmored encrypts data to key without AEAD, signed inline by
|
||||
// signer if not nil, compressed if compress is set.
|
||||
//
|
||||
// Encrypting to recipients picks SEIPDv2 whenever every recipient key
|
||||
// advertises it, as file node keys do, so the session key is generated
|
||||
// and wrapped separately: the SDK's ignoreSEIPDv2FeatureFlag.
|
||||
func encryptArmored(data []byte, key, signer *crypto.Key, compress bool) string {
|
||||
sk, err := crypto.GenerateSessionKeyAlgo(constants.AES256)
|
||||
mustCrypto("session key", err)
|
||||
kh, err := crypto.PGP().Encryption().Recipient(key).New()
|
||||
mustCrypto("encrypt session key", err)
|
||||
keyPacket, err := kh.EncryptSessionKey(sk)
|
||||
mustCrypto("encrypt session key", err)
|
||||
b := crypto.PGP().Encryption().SessionKey(sk)
|
||||
if signer != nil {
|
||||
b = b.SigningKey(signer).SignTime(signTimeOf(signer))
|
||||
}
|
||||
if compress {
|
||||
b = b.Compress()
|
||||
}
|
||||
eh, err := b.New()
|
||||
mustCrypto("encrypt", err)
|
||||
msg, err := eh.Encrypt(data)
|
||||
mustCrypto("encrypt", err)
|
||||
armored, err := crypto.NewPGPMessage(append(keyPacket, msg.DataPacket...)).Armor()
|
||||
mustCrypto("armor message", err)
|
||||
return armored
|
||||
}
|
||||
|
||||
// signArmored returns an armored detached signature of data by signer.
|
||||
func signArmored(data []byte, signer *crypto.Key) string {
|
||||
sh, err := crypto.PGP().Sign().SigningKey(signer).Detached().SignTime(signTimeOf(signer)).New()
|
||||
mustCrypto("sign", err)
|
||||
sig, err := sh.Sign(data, crypto.Armor)
|
||||
mustCrypto("sign", err)
|
||||
return string(sig)
|
||||
}
|
||||
|
||||
// signBinary returns a binary detached signature of data by signer.
|
||||
func signBinary(data []byte, signer *crypto.Key) []byte {
|
||||
sh, err := crypto.PGP().Sign().SigningKey(signer).Detached().SignTime(signTimeOf(signer)).New()
|
||||
mustCrypto("sign", err)
|
||||
sig, err := sh.Sign(data, crypto.Bytes)
|
||||
mustCrypto("sign", err)
|
||||
return sig
|
||||
}
|
||||
|
||||
// newHashKey returns a new folder hash key and its armored form,
|
||||
// encrypted to and signed by nodeKey, the SDK's generateHashKey.
|
||||
func newHashKey(nodeKey *crypto.Key) (hashKey []byte, armored string) {
|
||||
hashKey = newPassphrase()
|
||||
return hashKey, encryptArmored(hashKey, nodeKey, nodeKey, false)
|
||||
}
|
||||
|
||||
// lookupHash returns the hex HMAC-SHA256 of name keyed with the parent
|
||||
// folder's hash key, the SDK's generateLookupHash.
|
||||
func lookupHash(parentHashKey []byte, name string) string {
|
||||
mac := hmac.New(sha256.New, parentHashKey)
|
||||
mac.Write([]byte(name))
|
||||
return hex.EncodeToString(mac.Sum(nil))
|
||||
}
|
||||
|
||||
// contentKey is a file's content session key with its key packet.
|
||||
type contentKey struct {
|
||||
sk *crypto.SessionKey
|
||||
packet string // base64 PKESK to the node key
|
||||
signature string // armored detached signature of the session key by the node key
|
||||
aead bool
|
||||
}
|
||||
|
||||
// newContentKey generates a content session key for nodeKey, the SDK's
|
||||
// generateContentKey. With aead the key packet is a v6 PKESK and blocks
|
||||
// are SEIPDv2; without, the legacy v3 PKESK and SEIPDv1.
|
||||
func newContentKey(nodeKey *crypto.Key, aead bool) contentKey {
|
||||
token := randomBytes(32)
|
||||
sk := crypto.NewSessionKeyFromTokenWithAead(token, constants.AES256, aead)
|
||||
eh, err := crypto.PGP().Encryption().Recipient(nodeKey).New()
|
||||
mustCrypto("content key packet", err)
|
||||
keyPacket, err := eh.EncryptSessionKey(sk)
|
||||
mustCrypto("content key packet", err)
|
||||
return contentKey{
|
||||
sk: sk,
|
||||
packet: base64.StdEncoding.EncodeToString(keyPacket),
|
||||
signature: signArmored(sk.Key, nodeKey),
|
||||
aead: aead,
|
||||
}
|
||||
}
|
||||
|
||||
// pgp returns the handle for encrypting with ck.
|
||||
func (ck contentKey) pgp() *crypto.PGPHandle {
|
||||
if ck.aead {
|
||||
return aeadPGP()
|
||||
}
|
||||
return crypto.PGP()
|
||||
}
|
||||
|
||||
// encryptBlock encrypts one block of plaintext with the content key and
|
||||
// returns the data packet and the armored block signature: a detached
|
||||
// signature by signer encrypted to nodeKey with the content key, the
|
||||
// SDK's encryptBlock.
|
||||
func (ck contentKey) encryptBlock(plain []byte, nodeKey, signer *crypto.Key) (data []byte, encSignature string) {
|
||||
eh, err := ck.pgp().Encryption().SessionKey(ck.sk).New()
|
||||
mustCrypto("encrypt block", err)
|
||||
msg, err := eh.Encrypt(plain)
|
||||
mustCrypto("encrypt block", err)
|
||||
|
||||
sigHandle, err := ck.pgp().Encryption().Recipient(nodeKey).SessionKey(ck.sk).New()
|
||||
mustCrypto("encrypt block signature", err)
|
||||
sigMsg, err := sigHandle.Encrypt(signBinary(plain, signer))
|
||||
mustCrypto("encrypt block signature", err)
|
||||
encSignature, err = sigMsg.Armor()
|
||||
mustCrypto("armor block signature", err)
|
||||
return msg.DataPacket, encSignature
|
||||
}
|
||||
|
||||
// keyRing returns a key ring of keys, which must not be empty.
|
||||
func keyRing(keys ...*crypto.Key) *crypto.KeyRing {
|
||||
kr, err := crypto.NewKeyRing(nil)
|
||||
mustCrypto("key ring", err)
|
||||
for _, k := range keys {
|
||||
mustCrypto("key ring", kr.AddKey(k))
|
||||
}
|
||||
return kr
|
||||
}
|
||||
|
||||
// errSignature wraps every signature verification failure.
|
||||
var errSignature = errors.New("protontest: bad signature")
|
||||
|
||||
// decryptArmored decrypts an armored message with key. If verify is not
|
||||
// nil the embedded signature must be valid for one of its keys at time
|
||||
// at, or now if at is zero.
|
||||
func decryptArmored(armored string, key *crypto.Key, verify *crypto.KeyRing, at time.Time) ([]byte, error) {
|
||||
b := crypto.PGP().Decryption().DecryptionKey(key)
|
||||
if verify != nil {
|
||||
b = b.VerificationKeys(verify)
|
||||
if !at.IsZero() {
|
||||
b = b.VerifyTime(at.Unix())
|
||||
}
|
||||
}
|
||||
dh, err := b.New()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, err := dh.Decrypt([]byte(armored), crypto.Armor)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if verify != nil {
|
||||
if err := res.SignatureError(); err != nil {
|
||||
return res.Bytes(), fmt.Errorf("%w: %v", errSignature, err)
|
||||
}
|
||||
}
|
||||
return res.Bytes(), nil
|
||||
}
|
||||
|
||||
// verifyDetached checks a detached signature, armored or binary per
|
||||
// encoding, of data against verify at time at, or now if at is zero.
|
||||
func verifyDetached(data, sig []byte, encoding int8, verify *crypto.KeyRing, at time.Time) error {
|
||||
b := crypto.PGP().Verify().VerificationKeys(verify)
|
||||
if !at.IsZero() {
|
||||
b = b.VerifyTime(at.Unix())
|
||||
}
|
||||
vh, err := b.New()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res, err := vh.VerifyDetached(data, sig, encoding)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %v", errSignature, err)
|
||||
}
|
||||
if err := res.SignatureError(); err != nil {
|
||||
return fmt.Errorf("%w: %v", errSignature, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unlockFunc unlocks an armored private key with a passphrase.
|
||||
type unlockFunc func(armoredKey string, passphrase []byte) (*crypto.Key, error)
|
||||
|
||||
// unlockDriveKey decrypts a node or share key: the passphrase with
|
||||
// parent, its detached signature against verify, then the key itself
|
||||
// with unlock. A bad signature is returned alongside the unlocked key.
|
||||
// Signatures are checked at time at, or now if it is zero.
|
||||
func unlockDriveKey(armoredKey string, sealed sealedPassphrase, parent *crypto.Key, verify *crypto.KeyRing, unlock unlockFunc, at time.Time) (*crypto.Key, error) {
|
||||
passphrase, err := decryptArmored(sealed.armored, parent, nil, at)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decrypt passphrase: %w", err)
|
||||
}
|
||||
key, err := unlock(armoredKey, passphrase)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unlock key: %w", err)
|
||||
}
|
||||
if err := verifyDetached(passphrase, []byte(sealed.signature), crypto.Armor, verify, at); err != nil {
|
||||
return key, fmt.Errorf("passphrase: %w", err)
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// decryptContentKey decrypts a base64 content key packet with nodeKey
|
||||
// and checks its signature against nodeKey and verify at time at.
|
||||
func decryptContentKey(packet64, signature string, nodeKey *crypto.Key, verify *crypto.KeyRing, at time.Time) (*crypto.SessionKey, error) {
|
||||
keyPacket, err := base64.StdEncoding.DecodeString(packet64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("content key packet: %w", err)
|
||||
}
|
||||
dh, err := crypto.PGP().Decryption().DecryptionKey(nodeKey).New()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sk, err := dh.DecryptSessionKey(keyPacket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("content key packet: %w", err)
|
||||
}
|
||||
kr := keyRing(nodeKey)
|
||||
for _, k := range verify.GetKeys() {
|
||||
mustCrypto("key ring", kr.AddKey(k))
|
||||
}
|
||||
if err := verifyDetached(sk.Key, []byte(signature), crypto.Armor, kr, at); err != nil {
|
||||
return sk, fmt.Errorf("content key packet: %w", err)
|
||||
}
|
||||
return sk, nil
|
||||
}
|
||||
|
||||
// decryptBlock decrypts a block's data packet with sk and checks its
|
||||
// encrypted signature, decrypted with nodeKey, against verify at time
|
||||
// at.
|
||||
func decryptBlock(data []byte, encSignature string, sk *crypto.SessionKey, nodeKey *crypto.Key, verify *crypto.KeyRing, at time.Time) ([]byte, error) {
|
||||
dh, err := crypto.PGP().Decryption().SessionKey(sk).New()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, err := dh.Decrypt(data, crypto.Bytes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decrypt block: %w", err)
|
||||
}
|
||||
sig, err := decryptArmored(encSignature, nodeKey, nil, at)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decrypt block signature: %w", err)
|
||||
}
|
||||
if err := verifyDetached(res.Bytes(), sig, crypto.Bytes, verify, at); err != nil {
|
||||
return res.Bytes(), fmt.Errorf("block: %w", err)
|
||||
}
|
||||
return res.Bytes(), nil
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,716 @@
|
||||
package protontest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ProtonMail/go-crypto/openpgp/armor"
|
||||
"github.com/ProtonMail/go-crypto/openpgp/packet"
|
||||
"github.com/ProtonMail/gopenpgp/v3/crypto"
|
||||
"github.com/rclone/rclone/backend/protondrive/api"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// sessionHeader returns the headers of an authenticated request.
|
||||
func sessionHeader(sess Session) map[string]string {
|
||||
return map[string]string{
|
||||
"x-pm-appversion": testAppVersion,
|
||||
"x-pm-uid": sess.UID,
|
||||
"Authorization": "Bearer " + sess.AccessToken,
|
||||
}
|
||||
}
|
||||
|
||||
// getJSON calls GET path as sess, decodes the response into out if it
|
||||
// is not nil and returns the HTTP status and body.
|
||||
func getJSON(t *testing.T, s *Server, sess Session, path string, out any) (int, []byte) {
|
||||
t.Helper()
|
||||
resp, body := do(t, s, "GET", path, sessionHeader(sess))
|
||||
if out != nil && resp.StatusCode == http.StatusOK {
|
||||
require.NoError(t, json.Unmarshal(body, out), "body: %s", body)
|
||||
}
|
||||
return resp.StatusCode, body
|
||||
}
|
||||
|
||||
// patterned returns content of n bytes which differs per block.
|
||||
func patterned(n int) []byte {
|
||||
b := make([]byte, n)
|
||||
for i := range b {
|
||||
b[i] = byte(i*7 + i/blockSize)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func sha1Hex(b []byte) string {
|
||||
sum := sha1.Sum(b)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func TestSeedAndDump(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
modTime := time.Date(2024, 5, 6, 7, 8, 9, 123e6, time.FixedZone("x", 3600))
|
||||
|
||||
docs := u.SeedFolder(u.RootID(), "Documents")
|
||||
sub := u.SeedFolder(docs, "Sub")
|
||||
empty := u.SeedFile(u.RootID(), "empty.txt", nil, modTime)
|
||||
small := u.SeedFile(docs, "small.bin", []byte("hello"), modTime)
|
||||
exact := u.SeedFile(sub, "exact.bin", patterned(blockSize), modTime)
|
||||
big := u.SeedFile(sub, "big.bin", patterned(2*blockSize+123), modTime)
|
||||
before := time.Now().Truncate(time.Second)
|
||||
legacy := u.SeedLegacyFile(docs, "legacy.txt", []byte("old format"))
|
||||
doc := u.SeedDoc(u.RootID(), "Notes")
|
||||
|
||||
type want struct {
|
||||
id, path string
|
||||
folder bool
|
||||
content []byte
|
||||
hasTime bool
|
||||
}
|
||||
wants := []want{
|
||||
{docs, "Documents", true, nil, false},
|
||||
{sub, "Documents/Sub", true, nil, false},
|
||||
{big, "Documents/Sub/big.bin", false, patterned(2*blockSize + 123), true},
|
||||
{exact, "Documents/Sub/exact.bin", false, patterned(blockSize), true},
|
||||
{legacy, "Documents/legacy.txt", false, []byte("old format"), true},
|
||||
{small, "Documents/small.bin", false, []byte("hello"), true},
|
||||
{doc, "Notes", false, nil, false},
|
||||
{empty, "empty.txt", false, nil, true},
|
||||
}
|
||||
dump := u.Dump()
|
||||
require.Len(t, dump, len(wants))
|
||||
for i, w := range wants {
|
||||
n := dump[i]
|
||||
assert.NoError(t, n.Err, w.path)
|
||||
assert.Equal(t, w.id, n.ID, w.path)
|
||||
assert.Equal(t, w.path, n.Path)
|
||||
assert.Equal(t, w.folder, n.IsFolder, w.path)
|
||||
assert.Equal(t, linkStateActive, n.State, w.path)
|
||||
assert.True(t, bytes.Equal(w.content, n.Content), "%s: content", w.path)
|
||||
switch {
|
||||
case !w.hasTime:
|
||||
assert.True(t, n.ModTime.IsZero(), w.path)
|
||||
assert.Empty(t, n.SHA1, w.path)
|
||||
case w.id == legacy:
|
||||
assert.False(t, n.ModTime.Before(before), w.path)
|
||||
assert.False(t, n.ModTime.After(time.Now()), w.path)
|
||||
assert.Equal(t, sha1Hex(w.content), n.SHA1, w.path)
|
||||
default:
|
||||
assert.True(t, modTime.Equal(n.ModTime), "%s: modtime %v", w.path, n.ModTime)
|
||||
assert.Equal(t, sha1Hex(w.content), n.SHA1, w.path)
|
||||
}
|
||||
}
|
||||
assert.Empty(t, u.Drafts())
|
||||
}
|
||||
|
||||
// packetVersion returns the tag and version byte of the first packet
|
||||
// of a binary OpenPGP message.
|
||||
func packetVersion(t *testing.T, msg []byte) (tag packet.Packet, version byte) {
|
||||
t.Helper()
|
||||
p, err := packet.NewReader(bytes.NewReader(msg)).Next()
|
||||
require.NoError(t, err)
|
||||
switch p := p.(type) {
|
||||
case *packet.EncryptedKey:
|
||||
return p, byte(p.Version)
|
||||
case *packet.SymmetricallyEncrypted:
|
||||
return p, byte(p.Version)
|
||||
}
|
||||
t.Fatalf("unexpected packet %T", p)
|
||||
return nil, 0
|
||||
}
|
||||
|
||||
// dataPacketVersion returns the SEIPD version of an armored message.
|
||||
func dataPacketVersion(t *testing.T, armored string) byte {
|
||||
t.Helper()
|
||||
block, err := armor.Decode(strings.NewReader(armored))
|
||||
require.NoError(t, err)
|
||||
packets := packet.NewReader(block.Body)
|
||||
for {
|
||||
p, err := packets.Next()
|
||||
require.NoError(t, err)
|
||||
if p, ok := p.(*packet.SymmetricallyEncrypted); ok {
|
||||
return byte(p.Version)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeedFormats(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
current := u.SeedFile(u.RootID(), "current", []byte("new"), time.Now())
|
||||
legacy := u.SeedLegacyFile(u.RootID(), "legacy", []byte("old"))
|
||||
folder := u.SeedFolder(u.RootID(), "folder")
|
||||
v := u.view()
|
||||
|
||||
for _, tc := range []struct {
|
||||
id string
|
||||
pkesk byte
|
||||
seipd byte
|
||||
aeadKey bool
|
||||
mediaType string
|
||||
blockCount int
|
||||
}{
|
||||
{current, 6, 2, true, "application/octet-stream", 1},
|
||||
{legacy, 3, 1, false, "application/octet-stream", 1},
|
||||
} {
|
||||
l := v.d.links[tc.id]
|
||||
keyPacket, err := base64.StdEncoding.DecodeString(l.contentKeyPacket)
|
||||
require.NoError(t, err)
|
||||
_, version := packetVersion(t, keyPacket)
|
||||
assert.Equal(t, tc.pkesk, version, "content key packet version")
|
||||
rev := l.activeRevision()
|
||||
require.Len(t, rev.blocks, tc.blockCount)
|
||||
_, version = packetVersion(t, rev.blocks[0].data)
|
||||
assert.Equal(t, tc.seipd, version, "block SEIPD version")
|
||||
assert.Equal(t, tc.seipd, dataPacketVersion(t, rev.blocks[0].encSignature), "block signature SEIPD version")
|
||||
assert.Equal(t, tc.mediaType, l.mediaType)
|
||||
|
||||
key, err := crypto.NewKeyFromArmored(l.nodeKey)
|
||||
require.NoError(t, err)
|
||||
sig, err := key.GetEntity().PrimarySelfSignature(time.Now(), nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.aeadKey, sig.SEIPDv2, "node key advertises SEIPDv2")
|
||||
|
||||
// Everything but content avoids AEAD whatever the node key
|
||||
// advertises.
|
||||
assert.Equal(t, byte(1), dataPacketVersion(t, l.name), "name")
|
||||
assert.Equal(t, byte(1), dataPacketVersion(t, l.nodePass.armored), "passphrase")
|
||||
assert.Equal(t, byte(1), dataPacketVersion(t, rev.xattr), "extended attributes")
|
||||
}
|
||||
|
||||
l := v.d.links[folder]
|
||||
key, err := crypto.NewKeyFromArmored(l.nodeKey)
|
||||
require.NoError(t, err)
|
||||
sig, err := key.GetEntity().PrimarySelfSignature(time.Now(), nil)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, sig.SEIPDv2, "folder key advertises SEIPDv2")
|
||||
assert.Equal(t, byte(1), dataPacketVersion(t, l.hashKey), "hash key")
|
||||
}
|
||||
|
||||
func TestCorrupt(t *testing.T) {
|
||||
for _, what := range []string{"name", "name-signature", "passphrase-signature", "block-hash", "manifest", "swap-blocks", "truncate-blocks", "block-rollback", "foreign-block"} {
|
||||
t.Run(what, func(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
victim := u.SeedFile(u.RootID(), "victim", patterned(blockSize+10), time.Now())
|
||||
if what == "block-rollback" {
|
||||
u.SeedRevision(victim, patterned(blockSize+20), time.Now())
|
||||
}
|
||||
bystander := u.SeedFile(u.RootID(), "bystander", []byte("fine"), time.Now())
|
||||
u.Corrupt(victim, what)
|
||||
for _, n := range u.Dump() {
|
||||
switch n.ID {
|
||||
case victim:
|
||||
assert.Error(t, n.Err)
|
||||
case bystander:
|
||||
assert.NoError(t, n.Err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootstrapRoutes(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
sess := s.NewSession(u)
|
||||
|
||||
var myFiles myFilesResponse
|
||||
status, body := getJSON(t, s, sess, "/drive/v2/shares/my-files", &myFiles)
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
assert.Equal(t, u.VolumeID(), myFiles.Volume.VolumeID)
|
||||
assert.Equal(t, u.ShareID(), myFiles.Share.ShareID)
|
||||
assert.Equal(t, u.RootID(), myFiles.Link.Link.LinkID)
|
||||
assert.Equal(t, u.Email(), myFiles.Share.CreatorEmail)
|
||||
assert.Nil(t, myFiles.Link.Link.ParentLinkID)
|
||||
|
||||
var volume getVolumeResponse
|
||||
status, body = getJSON(t, s, sess, "/drive/volumes/"+u.VolumeID(), &volume)
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
assert.Equal(t, u.ShareID(), volume.Volume.Share.ShareID)
|
||||
assert.Equal(t, u.RootID(), volume.Volume.Share.LinkID)
|
||||
|
||||
var share shareResponse
|
||||
status, body = getJSON(t, s, sess, "/drive/shares/"+u.ShareID(), &share)
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
assert.Equal(t, u.RootID(), share.LinkID)
|
||||
assert.Equal(t, myFiles.Share.Key, share.Key)
|
||||
require.NotNil(t, share.AddressID)
|
||||
assert.Equal(t, myFiles.Share.AddressID, *share.AddressID)
|
||||
|
||||
// Another user's IDs are not found.
|
||||
other := s.AddUser("other", "pass")
|
||||
status, body = getJSON(t, s, sess, "/drive/volumes/"+other.VolumeID(), nil)
|
||||
assert.Equal(t, http.StatusUnprocessableEntity, status)
|
||||
assert.Equal(t, api.CodeNotExists, decodeError(t, body).Code)
|
||||
status, _ = getJSON(t, s, sess, "/drive/shares/"+other.ShareID(), nil)
|
||||
assert.Equal(t, http.StatusUnprocessableEntity, status)
|
||||
}
|
||||
|
||||
// keysOf returns the keys of a JSON object.
|
||||
func keysOf(t *testing.T, v any) []string {
|
||||
t.Helper()
|
||||
m, ok := v.(map[string]any)
|
||||
require.True(t, ok, "not an object: %v", v)
|
||||
var keys []string
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// TestWireFieldNames checks the link metadata response carries the
|
||||
// fields the SDK's nodes/apiService.ts reads, by name.
|
||||
func TestWireFieldNames(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
sess := s.NewSession(u)
|
||||
folder := u.SeedFolder(u.RootID(), "folder")
|
||||
file := u.SeedFile(u.RootID(), "file", []byte("x"), time.Now())
|
||||
|
||||
var resp struct{ Links []map[string]any }
|
||||
status := postJSON(t, s, "/drive/v2/volumes/"+u.VolumeID()+"/links", sessionHeader(sess), &linksRequest{LinkIDs: []string{folder, file}}, &resp)
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
require.Len(t, resp.Links, 2)
|
||||
for _, l := range resp.Links {
|
||||
assert.Subset(t, keysOf(t, l), []string{"Link", "Folder", "File", "Sharing", "Membership"})
|
||||
assert.Subset(t, keysOf(t, l["Link"]), []string{
|
||||
"LinkID", "Type", "ParentLinkID", "State", "CreateTime", "ModifyTime", "TrashTime",
|
||||
"Name", "NameHash", "NodeKey", "NodePassphrase", "NodePassphraseSignature",
|
||||
"SignatureEmail", "NameSignatureEmail", "OwnedBy", "ThirdParty", "Sdk",
|
||||
})
|
||||
if l["File"] == nil {
|
||||
assert.Subset(t, keysOf(t, l["Folder"]), []string{"NodeHashKey", "XAttr", "IsImported"})
|
||||
continue
|
||||
}
|
||||
f := l["File"]
|
||||
assert.Subset(t, keysOf(t, f), []string{"ActiveRevision", "TotalEncryptedSize", "ContentKeyPacket", "MediaType", "ContentKeyPacketSignature"})
|
||||
assert.Subset(t, keysOf(t, f.(map[string]any)["ActiveRevision"]), []string{
|
||||
"RevisionID", "CreateTime", "EncryptedSize", "SignatureEmail", "XAttr", "IsImported",
|
||||
"Thumbnails", "ThirdParty", "Sdk", "ManifestSignature",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChildrenPaging(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
sess := s.NewSession(u)
|
||||
s.SetChildrenPageSize(100)
|
||||
parent := u.SeedFolder(u.RootID(), "parent")
|
||||
want := make(map[string]bool)
|
||||
for i := range 250 {
|
||||
want[u.SeedFolder(parent, fmt.Sprintf("child %03d", i))] = true
|
||||
}
|
||||
trashed := u.SeedFolder(parent, "trashed")
|
||||
s.mu.Lock()
|
||||
u.drive.links[trashed].state = linkStateTrashed
|
||||
s.mu.Unlock()
|
||||
s.ResetRequests()
|
||||
|
||||
got := make(map[string]bool)
|
||||
anchor, pages := "", 0
|
||||
for {
|
||||
path := "/drive/v2/volumes/" + u.VolumeID() + "/folders/" + parent + "/children"
|
||||
if anchor != "" {
|
||||
path += "?AnchorID=" + anchor
|
||||
}
|
||||
var page childrenResponse
|
||||
status, body := getJSON(t, s, sess, path, &page)
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
pages++
|
||||
for _, id := range page.LinkIDs {
|
||||
assert.False(t, got[id], "duplicate %s", id)
|
||||
got[id] = true
|
||||
}
|
||||
if !page.More {
|
||||
assert.Nil(t, page.AnchorID)
|
||||
break
|
||||
}
|
||||
require.NotNil(t, page.AnchorID)
|
||||
require.Len(t, page.LinkIDs, 100)
|
||||
anchor = *page.AnchorID
|
||||
}
|
||||
assert.Equal(t, 3, pages)
|
||||
assert.Equal(t, want, got)
|
||||
assert.Equal(t, 3, s.Requests()["GET /drive/v2/volumes/{volumeID}/folders/{linkID}/children"])
|
||||
}
|
||||
|
||||
func TestLinksBatchLimit(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
sess := s.NewSession(u)
|
||||
a := u.SeedFolder(u.RootID(), "a")
|
||||
b := u.SeedFile(u.RootID(), "b", []byte("b"), time.Now())
|
||||
path := "/drive/v2/volumes/" + u.VolumeID() + "/links"
|
||||
|
||||
ids := []string{a, b}
|
||||
for len(ids) < maxLinksPerRequest {
|
||||
ids = append(ids, randomID())
|
||||
}
|
||||
var resp linksResponse
|
||||
require.Equal(t, http.StatusOK, postJSON(t, s, path, sessionHeader(sess), &linksRequest{LinkIDs: ids}, &resp))
|
||||
require.Len(t, resp.Links, 2, "unknown IDs are left out")
|
||||
assert.Equal(t, a, resp.Links[0].Link.LinkID)
|
||||
assert.NotNil(t, resp.Links[0].Folder)
|
||||
assert.Equal(t, b, resp.Links[1].Link.LinkID)
|
||||
require.NotNil(t, resp.Links[1].File)
|
||||
assert.NotNil(t, resp.Links[1].File.ActiveRevision)
|
||||
|
||||
var apiErr api.Error
|
||||
ids = append(ids, randomID())
|
||||
assert.Equal(t, http.StatusUnprocessableEntity, postJSON(t, s, path, sessionHeader(sess), &linksRequest{LinkIDs: ids}, &apiErr))
|
||||
assert.Equal(t, api.CodeInvalidValue, apiErr.Code)
|
||||
}
|
||||
|
||||
// getBlock fetches a block URL with the given storage token, sent
|
||||
// unless empty.
|
||||
func getBlock(t *testing.T, s *Server, bareURL, token string) (int, []byte) {
|
||||
t.Helper()
|
||||
header := map[string]string{"x-pm-appversion": testAppVersion}
|
||||
if token != "" {
|
||||
header["pm-storage-token"] = token
|
||||
}
|
||||
require.True(t, strings.HasPrefix(bareURL, s.URL()+"/"), "block URL %s is not on the test server", bareURL)
|
||||
resp, body := do(t, s, "GET", strings.TrimPrefix(bareURL, s.URL()), header)
|
||||
return resp.StatusCode, body
|
||||
}
|
||||
|
||||
func TestBlockDownloadNeedsToken(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
sess := s.NewSession(u)
|
||||
content := patterned(blockSize + 100)
|
||||
file := u.SeedFile(u.RootID(), "file", content, time.Now())
|
||||
|
||||
var links linksResponse
|
||||
require.Equal(t, http.StatusOK, postJSON(t, s, "/drive/v2/volumes/"+u.VolumeID()+"/links", sessionHeader(sess), &linksRequest{LinkIDs: []string{file}}, &links))
|
||||
revisionPath := "/drive/v2/volumes/" + u.VolumeID() + "/files/" + file + "/revisions/" + links.Links[0].File.ActiveRevision.RevisionID
|
||||
|
||||
var rev revisionResponse
|
||||
status, body := getJSON(t, s, sess, revisionPath+"?NoBlockUrls=true", &rev)
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
require.Len(t, rev.Revision.Blocks, 2)
|
||||
for _, b := range rev.Revision.Blocks {
|
||||
assert.Nil(t, b.Token)
|
||||
assert.Nil(t, b.BareURL)
|
||||
}
|
||||
require.NotNil(t, rev.Revision.ManifestSignature)
|
||||
|
||||
status, body = getJSON(t, s, sess, revisionPath+"?PageSize=1&FromBlockIndex=2", &rev)
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
require.Len(t, rev.Revision.Blocks, 1)
|
||||
b := rev.Revision.Blocks[0]
|
||||
assert.Equal(t, 2, b.Index)
|
||||
require.NotNil(t, b.Token)
|
||||
require.NotNil(t, b.BareURL)
|
||||
|
||||
status, _ = getBlock(t, s, *b.BareURL, "")
|
||||
assert.Equal(t, http.StatusForbidden, status)
|
||||
status, _ = getBlock(t, s, *b.BareURL, "wrong")
|
||||
assert.Equal(t, http.StatusForbidden, status)
|
||||
status, data := getBlock(t, s, *b.BareURL, *b.Token)
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
sum := sha256.Sum256(data)
|
||||
assert.Equal(t, b.Hash, base64.StdEncoding.EncodeToString(sum[:]))
|
||||
status, _ = getBlock(t, s, s.URL()+"/storage/blocks/unknown", *b.Token)
|
||||
assert.Equal(t, http.StatusNotFound, status)
|
||||
}
|
||||
|
||||
func TestQuota(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
sess := s.NewSession(u)
|
||||
u.SeedFile(u.RootID(), "a", patterned(1000), time.Now())
|
||||
u.SeedFile(u.RootID(), "b", patterned(10), time.Now())
|
||||
|
||||
var want int64
|
||||
v := u.view()
|
||||
for _, l := range v.d.links {
|
||||
for _, r := range l.revisions {
|
||||
want += r.size()
|
||||
}
|
||||
}
|
||||
require.NotZero(t, want)
|
||||
|
||||
var resp struct {
|
||||
User struct {
|
||||
UsedSpace int64
|
||||
MaxSpace int64
|
||||
ProductUsedSpace struct{ Drive int64 }
|
||||
}
|
||||
}
|
||||
status, body := getJSON(t, s, sess, "/core/v4/users", &resp)
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
assert.Equal(t, want, resp.User.UsedSpace)
|
||||
assert.Equal(t, want, resp.User.ProductUsedSpace.Drive)
|
||||
assert.Equal(t, DefaultMaxSpace, resp.User.MaxSpace)
|
||||
}
|
||||
|
||||
// volumeBootstrap returns a create volume request for the primary
|
||||
// address of u made as the SDK's generateVolumeBootstrap does.
|
||||
func volumeBootstrap(u *User) *createVolumeRequest {
|
||||
addr := u.addresses[0]
|
||||
addrKey := addr.keys[0].key
|
||||
share := newDriveKey(false)
|
||||
sharePass := sealPassphrase(share.passphrase, addrKey, addrKey)
|
||||
root := newDriveKey(false)
|
||||
rootPass := sealPassphrase(root.passphrase, share.key, addrKey)
|
||||
_, hashKey := newHashKey(root.key)
|
||||
return &createVolumeRequest{
|
||||
AddressID: addr.id,
|
||||
AddressKeyID: &addr.keys[0].id,
|
||||
ShareKey: share.armored,
|
||||
SharePassphrase: sharePass.armored,
|
||||
SharePassphraseSignature: sharePass.signature,
|
||||
FolderName: encryptArmored([]byte("root"), share.key, addrKey, false),
|
||||
FolderKey: root.armored,
|
||||
FolderPassphrase: rootPass.armored,
|
||||
FolderPassphraseSignature: rootPass.signature,
|
||||
FolderHashKey: hashKey,
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccountWithoutDrive(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUserWithoutDrive("new", "pass")
|
||||
sess := s.NewSession(u)
|
||||
|
||||
status, body := getJSON(t, s, sess, "/drive/v2/shares/my-files", nil)
|
||||
assert.Equal(t, http.StatusUnprocessableEntity, status)
|
||||
assert.Equal(t, api.CodeNotExists, decodeError(t, body).Code)
|
||||
assert.Equal(t, "Volume not found", decodeError(t, body).Message)
|
||||
|
||||
impostor := generateKey("impostor", "impostor@proton.test")
|
||||
for name, spoil := range map[string]func(req *createVolumeRequest){
|
||||
"share passphrase to another key": func(req *createVolumeRequest) {
|
||||
req.SharePassphrase = encryptArmored([]byte("x"), impostor, nil, false)
|
||||
},
|
||||
"share passphrase signed by another key": func(req *createVolumeRequest) {
|
||||
other := volumeBootstrap(u)
|
||||
req.SharePassphraseSignature = other.SharePassphraseSignature
|
||||
},
|
||||
"folder name missing": func(req *createVolumeRequest) {
|
||||
req.FolderName = ""
|
||||
},
|
||||
"hash key missing": func(req *createVolumeRequest) {
|
||||
req.FolderHashKey = ""
|
||||
},
|
||||
"unknown address": func(req *createVolumeRequest) {
|
||||
req.AddressID = "unknown"
|
||||
},
|
||||
} {
|
||||
req := volumeBootstrap(u)
|
||||
spoil(req)
|
||||
var apiErr api.Error
|
||||
assert.Equal(t, http.StatusUnprocessableEntity, postJSON(t, s, "/drive/volumes", sessionHeader(sess), req, &apiErr), name)
|
||||
assert.Equal(t, api.CodeInvalidValue, apiErr.Code, name)
|
||||
}
|
||||
|
||||
var created getVolumeResponse
|
||||
status = postJSON(t, s, "/drive/volumes", sessionHeader(sess), volumeBootstrap(u), &created)
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
assert.Equal(t, u.VolumeID(), created.Volume.ID)
|
||||
assert.Equal(t, u.ShareID(), created.Volume.Share.ShareID)
|
||||
assert.Equal(t, u.RootID(), created.Volume.Share.LinkID)
|
||||
|
||||
var apiErr api.Error
|
||||
assert.Equal(t, http.StatusUnprocessableEntity, postJSON(t, s, "/drive/volumes", sessionHeader(sess), volumeBootstrap(u), &apiErr))
|
||||
assert.Equal(t, api.CodeAlreadyExists, apiErr.Code)
|
||||
|
||||
var myFiles myFilesResponse
|
||||
status, body = getJSON(t, s, sess, "/drive/v2/shares/my-files", &myFiles)
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
assert.Equal(t, u.RootID(), myFiles.Link.Link.LinkID)
|
||||
|
||||
assert.Empty(t, u.Dump())
|
||||
id := u.SeedFile(u.RootID(), "after", []byte("works"), time.Now())
|
||||
dump := u.Dump()
|
||||
require.Len(t, dump, 1)
|
||||
assert.Equal(t, id, dump[0].ID)
|
||||
assert.Equal(t, "after", dump[0].Path)
|
||||
assert.NoError(t, dump[0].Err)
|
||||
}
|
||||
|
||||
// TestBlockBodyIsReadable checks a downloaded block is the exact data
|
||||
// packet that decrypts with the content key.
|
||||
func TestBlockBodyIsReadable(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
sess := s.NewSession(u)
|
||||
file := u.SeedFile(u.RootID(), "file", []byte("block body"), time.Now())
|
||||
v := u.view()
|
||||
l := v.d.links[file]
|
||||
key, err := v.unlock(file)
|
||||
require.NoError(t, err)
|
||||
sk, err := decryptContentKey(l.contentKeyPacket, l.contentKeyPacketSignature, key, keyRing(key), time.Time{})
|
||||
require.NoError(t, err)
|
||||
|
||||
var rev revisionResponse
|
||||
status, body := getJSON(t, s, sess, "/drive/v2/volumes/"+u.VolumeID()+"/files/"+file+"/revisions/"+l.activeRevision().id, &rev)
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
b := rev.Revision.Blocks[0]
|
||||
status, data := getBlock(t, s, *b.BareURL, *b.Token)
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
plain, err := decryptBlock(data, *b.EncSignature, sk, key, keyRing(u.addresses[0].keys[0].key), time.Time{})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "block body", string(plain))
|
||||
}
|
||||
|
||||
func TestSeedRevision(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
ids := map[string]string{
|
||||
"current": u.SeedFile(u.RootID(), "current", []byte("old"), time.Now()),
|
||||
"legacy": u.SeedLegacyFile(u.RootID(), "legacy", []byte("old")),
|
||||
}
|
||||
content := patterned(blockSize + 5)
|
||||
for name, id := range ids {
|
||||
old := u.view().d.links[id].activeRevision().id
|
||||
rev := u.SeedRevision(id, content, time.Now())
|
||||
l := u.view().d.links[id]
|
||||
assert.Equal(t, rev, l.activeRevision().id, name)
|
||||
require.Len(t, l.revisions, 2, name)
|
||||
assert.Equal(t, old, l.revisions[0].id, name)
|
||||
assert.Equal(t, revisionStateObsolete, l.revisions[0].state, name)
|
||||
}
|
||||
dump := u.Dump()
|
||||
require.Len(t, dump, 2)
|
||||
for _, d := range dump {
|
||||
assert.NoError(t, d.Err, d.Path)
|
||||
assert.True(t, bytes.Equal(content, d.Content), d.Path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeedForgedFolder(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
forged := u.SeedForgedFolder(u.RootID(), "forged")
|
||||
nested := u.SeedAnonymousFolder(forged, "nested")
|
||||
file := u.SeedAnonymousFile(nested, "file", []byte("forged content"), time.Now())
|
||||
for _, n := range u.Dump() {
|
||||
switch n.ID {
|
||||
case forged:
|
||||
assert.Error(t, n.Err, "the forged key's signature verifies")
|
||||
case nested, file:
|
||||
assert.NoError(t, n.Err, n.Path)
|
||||
}
|
||||
if n.ID == file {
|
||||
assert.Equal(t, "forged/nested/file", n.Path)
|
||||
assert.Equal(t, "forged content", string(n.Content))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestForgeShareAndRoot(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
other := s.AddUser("other", "pass")
|
||||
u.ForgeShareCreator(other, other.Email())
|
||||
u.ForgeRootSigner(other, other.Email())
|
||||
v := u.view()
|
||||
assert.Equal(t, other.Email(), v.d.creatorEmail)
|
||||
_, err := v.unlockShare()
|
||||
assert.NoError(t, err, "the share verifies against the other user's key")
|
||||
root := v.d.links[v.d.rootID]
|
||||
assert.Equal(t, other.Email(), root.signatureEmail)
|
||||
_, err = v.unlock(v.d.rootID)
|
||||
assert.NoError(t, err, "the root verifies against the other user's key")
|
||||
id := u.SeedFile(u.RootID(), "file", []byte("x"), time.Now())
|
||||
for _, n := range u.Dump() {
|
||||
if n.ID == id {
|
||||
assert.NoError(t, n.Err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeKeysAs(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
other := s.AddUser("other", "pass")
|
||||
sess := s.NewSession(u)
|
||||
s.ServeKeysAs("USER@proton.test", other)
|
||||
var resp api.PublicKeysResponse
|
||||
status, body := getJSON(t, s, sess, "/core/v4/keys/all?Email=USER@proton.test", &resp)
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
require.NotEmpty(t, resp.Address.Keys)
|
||||
want, err := other.addresses[0].keys[0].key.GetArmoredPublicKey()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, want, resp.Address.Keys[0].PublicKey)
|
||||
}
|
||||
|
||||
func TestNodeSecrets(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
id := u.SeedFile(u.RootID(), "file", []byte("secret"), time.Now())
|
||||
passphrase, sessionKey := u.NodeSecrets(id)
|
||||
|
||||
v := u.view()
|
||||
l := v.d.links[id]
|
||||
key, err := crypto.NewPrivateKeyFromArmored(l.nodeKey, passphrase)
|
||||
require.NoError(t, err, "the passphrase unlocks the node key")
|
||||
sk, err := decryptContentKey(l.contentKeyPacket, l.contentKeyPacketSignature, key, keyRing(key), time.Time{})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, sk.Key, sessionKey)
|
||||
}
|
||||
|
||||
func TestIssuedSecrets(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
sess := s.NewSession(u)
|
||||
file := u.SeedFile(u.RootID(), "file", []byte("x"), time.Now())
|
||||
assert.Equal(t, []string{sess.AccessToken, sess.RefreshToken}, s.IssuedSecrets())
|
||||
|
||||
var rev revisionResponse
|
||||
status, body := getJSON(t, s, sess, "/drive/v2/volumes/"+u.VolumeID()+"/files/"+file+"/revisions/"+u.view().d.links[file].activeRevision().id, &rev)
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
token := *rev.Revision.Blocks[0].Token
|
||||
assert.Equal(t, []string{token}, s.IssuedSecrets(StorageToken))
|
||||
assert.Equal(t, []string{sess.AccessToken, sess.RefreshToken}, s.IssuedSecrets(SessionToken))
|
||||
assert.Equal(t, []string{sess.AccessToken, sess.RefreshToken, token}, s.IssuedSecrets())
|
||||
assert.Empty(t, s.IssuedSecrets(ForkSelector))
|
||||
}
|
||||
|
||||
func TestStoredFileRoundTrip(t *testing.T) {
|
||||
s := New(t)
|
||||
addrPass := newPassphrase()
|
||||
root := newDriveKey(false)
|
||||
_, hashKey := newHashKey(root.key)
|
||||
keys := FixedKeys{
|
||||
AddressKey: lockKey(generateKey("fixed", "fixed@proton.test"), addrPass),
|
||||
AddressKeyPassphrase: string(addrPass),
|
||||
RootKey: root.armored,
|
||||
RootKeyPassphrase: string(root.passphrase),
|
||||
RootHashKey: hashKey,
|
||||
}
|
||||
// Two users holding the same keys can read each other's stored files.
|
||||
one := s.AddUserWithKeys("one", "pass", keys)
|
||||
two := s.AddUserWithKeys("two", "pass", keys)
|
||||
content := patterned(blockSize + 5)
|
||||
id := one.SeedFile(one.RootID(), "f", content, time.Now())
|
||||
|
||||
f := one.StoredFile(id)
|
||||
assert.Equal(t, "one@proton.test", f.Link.SignatureEmail)
|
||||
require.Len(t, f.Revision.Blocks, 2)
|
||||
for _, b := range f.Revision.Blocks {
|
||||
sum := sha256.Sum256(b.Data)
|
||||
assert.Equal(t, sum[:], b.Hash)
|
||||
}
|
||||
two.SeedStoredFile(two.RootID(), f)
|
||||
for _, u := range []*User{one, two} {
|
||||
dump := u.Dump()
|
||||
require.Len(t, dump, 1)
|
||||
assert.NoError(t, dump[0].Err)
|
||||
assert.Equal(t, "f", dump[0].Path)
|
||||
assert.True(t, bytes.Equal(content, dump[0].Content))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,760 @@
|
||||
package protontest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ProtonMail/go-crypto/openpgp/armor"
|
||||
"github.com/ProtonMail/go-crypto/openpgp/packet"
|
||||
"github.com/ProtonMail/go-crypto/openpgp/s2k"
|
||||
"github.com/rclone/rclone/backend/protondrive/api"
|
||||
)
|
||||
|
||||
// OpenPGP packet tags checked in client-written values.
|
||||
const (
|
||||
tagPKESK = 1
|
||||
tagSignature = 2
|
||||
tagSecretKey = 5
|
||||
tagSecretSubkey = 7
|
||||
tagSEIPD = 18
|
||||
)
|
||||
|
||||
// s2kUsageSHA1 is the S2K usage octet of a secret key locked with CFB
|
||||
// and a SHA-1 check, as ordinary (not AEAD) key locking writes.
|
||||
const s2kUsageSHA1 = 254
|
||||
|
||||
// Wire types of the mutating routes, named after and shaped like the
|
||||
// schemas in the SDK's driveTypes.ts.
|
||||
|
||||
type createFolderRequest struct {
|
||||
ParentLinkID string
|
||||
NodeKey string
|
||||
NodeHashKey string
|
||||
NodePassphrase string
|
||||
NodePassphraseSignature string
|
||||
SignatureEmail *string
|
||||
Name string
|
||||
Hash string
|
||||
XAttr *string
|
||||
}
|
||||
|
||||
type folderIDJSON struct {
|
||||
ID string
|
||||
}
|
||||
|
||||
type createFolderResponse struct {
|
||||
Folder folderIDJSON
|
||||
Code int
|
||||
}
|
||||
|
||||
type renameRequest struct {
|
||||
Name string
|
||||
Hash *string
|
||||
NameSignatureEmail *string
|
||||
OriginalHash *string
|
||||
}
|
||||
|
||||
type moveRequest struct {
|
||||
Name string
|
||||
NodePassphrase string
|
||||
Hash string
|
||||
ParentLinkID string
|
||||
OriginalHash string
|
||||
NameSignatureEmail *string
|
||||
ContentHash *string
|
||||
NodePassphraseSignature *string
|
||||
SignatureEmail *string
|
||||
}
|
||||
|
||||
type copyRequest struct {
|
||||
Name string
|
||||
NodePassphrase string
|
||||
Hash string
|
||||
TargetVolumeID string
|
||||
TargetParentLinkID string
|
||||
NameSignatureEmail *string
|
||||
NodePassphraseSignature *string
|
||||
SignatureEmail *string
|
||||
}
|
||||
|
||||
type copyResponse struct {
|
||||
LinkID string
|
||||
Code int
|
||||
}
|
||||
|
||||
type codeJSON struct {
|
||||
Code int
|
||||
Error string `json:",omitempty"`
|
||||
}
|
||||
|
||||
type linkResultJSON struct {
|
||||
LinkID string
|
||||
Response codeJSON
|
||||
}
|
||||
|
||||
type multiResponse struct {
|
||||
Code int
|
||||
Responses []linkResultJSON
|
||||
}
|
||||
|
||||
type checkHashesRequest struct {
|
||||
Hashes []string
|
||||
ClientUID []string
|
||||
}
|
||||
|
||||
type pendingHashJSON struct {
|
||||
Hash string
|
||||
RevisionID string
|
||||
LinkID string
|
||||
ClientUID *string
|
||||
}
|
||||
|
||||
type availableHashesResponse struct {
|
||||
AvailableHashes []string
|
||||
PendingHashes []pendingHashJSON
|
||||
Code int
|
||||
}
|
||||
|
||||
type conflictDetailsJSON struct {
|
||||
ConflictLinkID string
|
||||
ConflictRevisionID *string
|
||||
ConflictDraftRevisionID *string
|
||||
ConflictDraftClientUID *string
|
||||
}
|
||||
|
||||
type conflictResponse struct {
|
||||
Code int
|
||||
Error string
|
||||
Details conflictDetailsJSON
|
||||
}
|
||||
|
||||
// registerMutations registers the routes which change a drive.
|
||||
func (s *Server) registerMutations() {
|
||||
s.handleDrive("POST", "/drive/v2/volumes/{volumeID}/folders", s.handleCreateFolder)
|
||||
s.handleDrive("PUT", "/drive/v2/volumes/{volumeID}/links/{linkID}/rename", s.handleRename)
|
||||
s.handleDrive("PUT", "/drive/v2/volumes/{volumeID}/links/{linkID}/move", s.handleMove)
|
||||
s.handleDrive("POST", "/drive/volumes/{volumeID}/links/{linkID}/copy", s.handleCopy)
|
||||
s.handleDrive("POST", "/drive/v2/volumes/{volumeID}/trash_multiple", s.handleTrash)
|
||||
s.handleDrive("DELETE", "/drive/volumes/{volumeID}/trash", s.handleEmptyTrash)
|
||||
s.handleDrive("POST", "/drive/v2/volumes/{volumeID}/links/{linkID}/checkAvailableHashes", s.handleCheckAvailableHashes)
|
||||
}
|
||||
|
||||
// decodeRequest decodes the JSON body of r into v, or answers HTTP 400
|
||||
// and returns false.
|
||||
func decodeRequest(w http.ResponseWriter, r *http.Request, v any) bool {
|
||||
if err := json.NewDecoder(r.Body).Decode(v); err != nil {
|
||||
writeError(w, http.StatusBadRequest, api.CodeInvalidValue, "protontest: bad request body: "+err.Error())
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// writeInvalid answers HTTP 422 code 2001 for a request failing check.
|
||||
func writeInvalid(w http.ResponseWriter, err error) {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidValue, "protontest: "+err.Error())
|
||||
}
|
||||
|
||||
// writeConflict answers HTTP 422 code 2500 for a request whose name
|
||||
// hash is taken by l.
|
||||
func writeConflict(w http.ResponseWriter, l *link) {
|
||||
details := conflictDetailsJSON{ConflictLinkID: l.id}
|
||||
for _, r := range l.revisions {
|
||||
switch r.state {
|
||||
case revisionStateActive:
|
||||
details.ConflictRevisionID = nullable(r.id)
|
||||
case revisionStateDraft:
|
||||
details.ConflictDraftRevisionID = nullable(r.id)
|
||||
details.ConflictDraftClientUID = nullable(r.clientUID)
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusUnprocessableEntity, &conflictResponse{
|
||||
Code: api.CodeAlreadyExists,
|
||||
Error: "A file or folder with that name already exists",
|
||||
Details: details,
|
||||
})
|
||||
}
|
||||
|
||||
// activeLink returns the active link id, or nil.
|
||||
func (d *drive) activeLink(id string) *link {
|
||||
if l := d.links[id]; l != nil && l.state == linkStateActive {
|
||||
return l
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// activeFolder returns the active folder id, or nil.
|
||||
func (d *drive) activeFolder(id string) *link {
|
||||
if l := d.activeLink(id); l != nil && l.typ == linkTypeFolder {
|
||||
return l
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// conflict returns the link other than except in folder parentID
|
||||
// which is not trashed and has the name hash hash, or nil.
|
||||
func (d *drive) conflict(parentID, hash, except string) *link {
|
||||
for _, l := range d.links {
|
||||
if l.parentID == parentID && l.nameHash == hash && l.state != linkStateTrashed && l.id != except {
|
||||
return l
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// within reports whether link id is ancestorID or below it.
|
||||
func (d *drive) within(id, ancestorID string) bool {
|
||||
for l := d.links[id]; l != nil; l = d.links[l.parentID] {
|
||||
if l.id == ancestorID {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// inTrash reports whether l or one of its ancestors is trashed.
|
||||
func (d *drive) inTrash(l *link) bool {
|
||||
for ; l != nil; l = d.links[l.parentID] {
|
||||
if l.state == linkStateTrashed {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ownsEmailLocked reports whether email is one of u's addresses.
|
||||
// srv.mu must be held.
|
||||
func (u *User) ownsEmailLocked(email string) bool {
|
||||
return u.addressByEmailLocked(email) != nil
|
||||
}
|
||||
|
||||
// checkHash returns an error unless hash is a lower-case hex SHA-256.
|
||||
func checkHash(hash string) error {
|
||||
if len(hash) != 64 || strings.ToLower(hash) != hash {
|
||||
return fmt.Errorf("hash %q is not lower-case hex SHA-256", hash)
|
||||
}
|
||||
if _, err := hex.DecodeString(hash); err != nil {
|
||||
return fmt.Errorf("hash %q is not lower-case hex SHA-256", hash)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkSigner returns an error unless email is one of u's addresses.
|
||||
func checkSigner(u *User, field string, email *string) error {
|
||||
if email == nil || !u.ownsEmailLocked(*email) {
|
||||
return fmt.Errorf("%s must be an address of the user", field)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkMessages checks every named armored message with checkMessage.
|
||||
func checkMessages(messages map[string]string) error {
|
||||
for field, armored := range messages {
|
||||
if err := checkMessage(armored); err != nil {
|
||||
return fmt.Errorf("%s: %w", field, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkMessage returns an error unless armored is an OpenPGP message of
|
||||
// v3 PKESKs followed by one SEIPDv1 packet.
|
||||
//
|
||||
// Official clients write names, passphrases, hash keys and extended
|
||||
// attributes in that format and not all of them read SEIPDv2 or v6
|
||||
// PKESKs there, whatever the recipient key accepts.
|
||||
func checkMessage(armored string) error {
|
||||
if armored == "" {
|
||||
return errors.New("missing")
|
||||
}
|
||||
block, err := armor.Decode(strings.NewReader(armored))
|
||||
if err != nil {
|
||||
return fmt.Errorf("not armored: %w", err)
|
||||
}
|
||||
if block.Type != "PGP MESSAGE" {
|
||||
return fmt.Errorf("armor type %q, want PGP MESSAGE", block.Type)
|
||||
}
|
||||
packets := packet.NewOpaqueReader(block.Body)
|
||||
keyPackets, dataPackets := 0, 0
|
||||
for {
|
||||
op, err := packets.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("malformed packet: %w", err)
|
||||
}
|
||||
if dataPackets > 0 {
|
||||
return errors.New("packet after the encrypted data")
|
||||
}
|
||||
version := -1
|
||||
if len(op.Contents) > 0 {
|
||||
version = int(op.Contents[0])
|
||||
}
|
||||
switch op.Tag {
|
||||
case tagPKESK:
|
||||
if version != 3 {
|
||||
return fmt.Errorf("v%d PKESK, want v3", version)
|
||||
}
|
||||
keyPackets++
|
||||
case tagSEIPD:
|
||||
if version != 1 {
|
||||
return fmt.Errorf("SEIPDv%d, want SEIPDv1", version)
|
||||
}
|
||||
dataPackets++
|
||||
default:
|
||||
return fmt.Errorf("unexpected packet with tag %d", op.Tag)
|
||||
}
|
||||
}
|
||||
if keyPackets == 0 || dataPackets == 0 {
|
||||
return errors.New("want PKESKs and an encrypted data packet")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkFolderKey returns an error unless armored is a v4 secret key
|
||||
// whose signatures don't advertise AEAD, with every secret key packet
|
||||
// locked with CFB and an iterated and salted S2K, as official clients
|
||||
// generate folder keys. The official web app can't use a node key
|
||||
// locked with Argon2 or AEAD.
|
||||
func checkFolderKey(armored string) error {
|
||||
return checkNodeKey(armored, false)
|
||||
}
|
||||
|
||||
// checkFileKey returns an error unless armored is a v4 secret key
|
||||
// locked as checkFolderKey requires. Unlike a folder key, a file key
|
||||
// advertises AEAD, for its content.
|
||||
func checkFileKey(armored string) error {
|
||||
return checkNodeKey(armored, true)
|
||||
}
|
||||
|
||||
// checkNodeKey does the work of checkFolderKey and checkFileKey; aead
|
||||
// allows the key to advertise AEAD.
|
||||
func checkNodeKey(armored string, aead bool) error {
|
||||
block, err := armor.Decode(strings.NewReader(armored))
|
||||
if err != nil {
|
||||
return fmt.Errorf("not armored: %w", err)
|
||||
}
|
||||
if block.Type != "PGP PRIVATE KEY BLOCK" {
|
||||
return fmt.Errorf("armor type %q, want PGP PRIVATE KEY BLOCK", block.Type)
|
||||
}
|
||||
packets := packet.NewOpaqueReader(block.Body)
|
||||
secretKeys := 0
|
||||
for {
|
||||
op, err := packets.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("malformed packet: %w", err)
|
||||
}
|
||||
switch op.Tag {
|
||||
case tagSecretKey, tagSecretSubkey:
|
||||
if err := checkSecretKeyPacket(op); err != nil {
|
||||
return err
|
||||
}
|
||||
secretKeys++
|
||||
case tagSignature:
|
||||
p, err := op.Parse()
|
||||
sig, ok := p.(*packet.Signature)
|
||||
if err != nil || !ok {
|
||||
return fmt.Errorf("malformed signature: %v", err)
|
||||
}
|
||||
if !aead && (sig.SEIPDv2 || len(sig.PreferredCipherSuites) > 0) {
|
||||
return errors.New("key advertises AEAD (SEIPDv2)")
|
||||
}
|
||||
}
|
||||
}
|
||||
if secretKeys == 0 {
|
||||
return errors.New("no secret key")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkSecretKeyPacket checks one secret key or subkey packet for
|
||||
// checkNodeKey.
|
||||
func checkSecretKeyPacket(op *packet.OpaquePacket) error {
|
||||
if len(op.Contents) == 0 || op.Contents[0] != 4 {
|
||||
return errors.New("not a v4 key")
|
||||
}
|
||||
p, err := op.Parse()
|
||||
pk, ok := p.(*packet.PrivateKey)
|
||||
if err != nil || !ok {
|
||||
return fmt.Errorf("malformed secret key: %v", err)
|
||||
}
|
||||
// The packet starts with the public key body, which a v4 key hashes
|
||||
// after 0x99 and a two-octet length.
|
||||
var pub bytes.Buffer
|
||||
if err := pk.PublicKey.SerializeForHash(&pub); err != nil {
|
||||
return fmt.Errorf("malformed secret key: %w", err)
|
||||
}
|
||||
rest := op.Contents[pub.Len()-3:]
|
||||
if len(rest) < 3 {
|
||||
return errors.New("malformed secret key: truncated")
|
||||
}
|
||||
switch usage := rest[0]; usage {
|
||||
case s2kUsageSHA1:
|
||||
case 0:
|
||||
return errors.New("secret key is not locked")
|
||||
case 253:
|
||||
return errors.New("secret key is locked with AEAD")
|
||||
default:
|
||||
return fmt.Errorf("secret key S2K usage %d, want %d", usage, s2kUsageSHA1)
|
||||
}
|
||||
if mode := s2k.Mode(rest[2]); mode != s2k.IteratedSaltedS2K {
|
||||
return fmt.Errorf("secret key S2K mode %d, want iterated and salted", mode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// keySignature returns the node passphrase signature and its signer
|
||||
// that l has once moved or copied, given the request's sig and email.
|
||||
//
|
||||
// As the SDK's moveNode and copyNode, a request gives them if and only
|
||||
// if l's key is anonymous: an anonymous key is then signed by the
|
||||
// user, and a signed one keeps its signature, as the passphrase is
|
||||
// unchanged.
|
||||
func keySignature(u *User, l *link, sig, email *string) (string, string, error) {
|
||||
if l.signatureEmail != "" {
|
||||
if (sig != nil && *sig != "") || (email != nil && *email != "") {
|
||||
return "", "", errors.New("NodePassphraseSignature and SignatureEmail must not be given for a signed node")
|
||||
}
|
||||
return l.nodePass.signature, l.signatureEmail, nil
|
||||
}
|
||||
if sig == nil || *sig == "" {
|
||||
return "", "", errors.New("NodePassphraseSignature is required for an anonymous node")
|
||||
}
|
||||
if err := checkSigner(u, "SignatureEmail", email); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return *sig, *email, nil
|
||||
}
|
||||
|
||||
// handleCreateFolder creates a folder from key material the client
|
||||
// generated.
|
||||
func (s *Server) handleCreateFolder(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
|
||||
var req createFolderRequest
|
||||
if !decodeRequest(w, r, &req) {
|
||||
return
|
||||
}
|
||||
parent := d.activeFolder(req.ParentLinkID)
|
||||
if parent == nil {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "Parent folder not found")
|
||||
return
|
||||
}
|
||||
messages := map[string]string{
|
||||
"Name": req.Name,
|
||||
"NodePassphrase": req.NodePassphrase,
|
||||
"NodeHashKey": req.NodeHashKey,
|
||||
}
|
||||
if req.XAttr != nil {
|
||||
messages["XAttr"] = *req.XAttr
|
||||
}
|
||||
err := errors.Join(
|
||||
checkHash(req.Hash),
|
||||
checkSigner(u, "SignatureEmail", req.SignatureEmail),
|
||||
checkMessages(messages),
|
||||
)
|
||||
if err == nil && req.NodePassphraseSignature == "" {
|
||||
err = errors.New("NodePassphraseSignature is required")
|
||||
}
|
||||
if err == nil {
|
||||
err = annotateField("NodeKey", checkFolderKey(req.NodeKey))
|
||||
}
|
||||
if err != nil {
|
||||
writeInvalid(w, err)
|
||||
return
|
||||
}
|
||||
if other := d.conflict(parent.id, req.Hash, ""); other != nil {
|
||||
writeConflict(w, other)
|
||||
return
|
||||
}
|
||||
now := time.Now().Unix()
|
||||
l := &link{
|
||||
id: randomID(),
|
||||
parentID: parent.id,
|
||||
typ: linkTypeFolder,
|
||||
state: linkStateActive,
|
||||
createTime: now,
|
||||
modifyTime: now,
|
||||
name: req.Name,
|
||||
nameHash: req.Hash,
|
||||
nodeKey: req.NodeKey,
|
||||
nodePass: sealedPassphrase{armored: req.NodePassphrase, signature: req.NodePassphraseSignature},
|
||||
signatureEmail: *req.SignatureEmail,
|
||||
nameSignatureEmail: *req.SignatureEmail,
|
||||
hashKey: req.NodeHashKey,
|
||||
}
|
||||
if req.XAttr != nil {
|
||||
l.xattr = *req.XAttr
|
||||
}
|
||||
d.addLink(l)
|
||||
s.lagLocked(l.id, nil)
|
||||
writeJSON(w, http.StatusOK, &createFolderResponse{Folder: folderIDJSON{ID: l.id}, Code: api.CodeOK})
|
||||
}
|
||||
|
||||
// annotateField returns err prefixed with field, or nil.
|
||||
func annotateField(field string, err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s: %w", field, err)
|
||||
}
|
||||
|
||||
// handleRename renames a link within its folder.
|
||||
func (s *Server) handleRename(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
|
||||
var req renameRequest
|
||||
if !decodeRequest(w, r, &req) {
|
||||
return
|
||||
}
|
||||
l := d.activeLink(r.PathValue("linkID"))
|
||||
switch {
|
||||
case l == nil:
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "File or folder not found")
|
||||
return
|
||||
case l.parentID == "":
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidRequirements, "protontest: the root can't be renamed")
|
||||
return
|
||||
case req.OriginalHash == nil || *req.OriginalHash != l.nameHash:
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidRequirements, "protontest: OriginalHash doesn't match: the node changed")
|
||||
return
|
||||
}
|
||||
hash := ""
|
||||
if req.Hash != nil {
|
||||
hash = *req.Hash
|
||||
}
|
||||
err := errors.Join(
|
||||
checkHash(hash),
|
||||
checkSigner(u, "NameSignatureEmail", req.NameSignatureEmail),
|
||||
checkMessages(map[string]string{"Name": req.Name}),
|
||||
)
|
||||
if err != nil {
|
||||
writeInvalid(w, err)
|
||||
return
|
||||
}
|
||||
if other := d.conflict(l.parentID, hash, l.id); other != nil {
|
||||
writeConflict(w, other)
|
||||
return
|
||||
}
|
||||
l.name, l.nameHash, l.nameSignatureEmail = req.Name, hash, *req.NameSignatureEmail
|
||||
l.modifyTime = time.Now().Unix()
|
||||
writeJSON(w, http.StatusOK, &codeJSON{Code: api.CodeOK})
|
||||
}
|
||||
|
||||
// handleMove moves a link to another folder, possibly renaming it.
|
||||
func (s *Server) handleMove(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
|
||||
var req moveRequest
|
||||
if !decodeRequest(w, r, &req) {
|
||||
return
|
||||
}
|
||||
l := d.activeLink(r.PathValue("linkID"))
|
||||
parent := d.activeFolder(req.ParentLinkID)
|
||||
switch {
|
||||
case l == nil:
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "File or folder not found")
|
||||
return
|
||||
case l.parentID == "":
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidRequirements, "protontest: the root can't be moved")
|
||||
return
|
||||
case parent == nil:
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "Parent folder not found")
|
||||
return
|
||||
case d.within(parent.id, l.id):
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidRequirements, "protontest: a folder can't be moved into itself")
|
||||
return
|
||||
case req.OriginalHash != l.nameHash:
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidRequirements, "protontest: OriginalHash doesn't match: the node changed")
|
||||
return
|
||||
}
|
||||
err := errors.Join(
|
||||
checkHash(req.Hash),
|
||||
checkSigner(u, "NameSignatureEmail", req.NameSignatureEmail),
|
||||
checkMessages(map[string]string{"Name": req.Name, "NodePassphrase": req.NodePassphrase}),
|
||||
)
|
||||
if err != nil {
|
||||
writeInvalid(w, err)
|
||||
return
|
||||
}
|
||||
sig, signer, err := keySignature(u, l, req.NodePassphraseSignature, req.SignatureEmail)
|
||||
if err != nil {
|
||||
writeInvalid(w, err)
|
||||
return
|
||||
}
|
||||
if other := d.conflict(parent.id, req.Hash, l.id); other != nil {
|
||||
writeConflict(w, other)
|
||||
return
|
||||
}
|
||||
l.parentID, l.name, l.nameHash, l.nameSignatureEmail = parent.id, req.Name, req.Hash, *req.NameSignatureEmail
|
||||
l.nodePass = sealedPassphrase{armored: req.NodePassphrase, signature: sig}
|
||||
l.signatureEmail = signer
|
||||
l.modifyTime = time.Now().Unix()
|
||||
writeJSON(w, http.StatusOK, &codeJSON{Code: api.CodeOK})
|
||||
}
|
||||
|
||||
// handleCopy copies a link, and a folder's active descendants, into a
|
||||
// folder. Only the name and passphrase of the copy come from the
|
||||
// client; its key, content and descendants are the server's copies.
|
||||
func (s *Server) handleCopy(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
|
||||
var req copyRequest
|
||||
if !decodeRequest(w, r, &req) {
|
||||
return
|
||||
}
|
||||
l := d.activeLink(r.PathValue("linkID"))
|
||||
parent := d.activeFolder(req.TargetParentLinkID)
|
||||
switch {
|
||||
case l == nil:
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "File or folder not found")
|
||||
return
|
||||
case l.parentID == "":
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidRequirements, "protontest: the root can't be copied")
|
||||
return
|
||||
case req.TargetVolumeID != d.volumeID:
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "Volume not found")
|
||||
return
|
||||
case parent == nil:
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "Parent folder not found")
|
||||
return
|
||||
case d.within(parent.id, l.id):
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidRequirements, "protontest: a folder can't be copied into itself")
|
||||
return
|
||||
}
|
||||
err := errors.Join(
|
||||
checkHash(req.Hash),
|
||||
checkSigner(u, "NameSignatureEmail", req.NameSignatureEmail),
|
||||
checkMessages(map[string]string{"Name": req.Name, "NodePassphrase": req.NodePassphrase}),
|
||||
)
|
||||
if err != nil {
|
||||
writeInvalid(w, err)
|
||||
return
|
||||
}
|
||||
sig, signer, err := keySignature(u, l, req.NodePassphraseSignature, req.SignatureEmail)
|
||||
if err != nil {
|
||||
writeInvalid(w, err)
|
||||
return
|
||||
}
|
||||
if other := d.conflict(parent.id, req.Hash, ""); other != nil {
|
||||
writeConflict(w, other)
|
||||
return
|
||||
}
|
||||
c := d.copyLink(l, parent.id, time.Now().Unix())
|
||||
c.name, c.nameHash, c.nameSignatureEmail = req.Name, req.Hash, *req.NameSignatureEmail
|
||||
c.nodePass = sealedPassphrase{armored: req.NodePassphrase, signature: sig}
|
||||
c.signatureEmail = signer
|
||||
s.lagLocked(c.id, nil)
|
||||
writeJSON(w, http.StatusOK, ©Response{LinkID: c.id, Code: api.CodeOK})
|
||||
}
|
||||
|
||||
// copyLink stores and returns a copy of l in folder parentID with its
|
||||
// active revision and, for a folder, copies of its active descendants.
|
||||
// Blocks are shared, as they are immutable.
|
||||
func (d *drive) copyLink(l *link, parentID string, now int64) *link {
|
||||
c := *l
|
||||
c.id, c.parentID = randomID(), parentID
|
||||
c.state, c.createTime, c.modifyTime, c.trashTime = linkStateActive, now, now, 0
|
||||
c.revisions = nil
|
||||
if r := l.activeRevision(); r != nil {
|
||||
rc := *r
|
||||
rc.id = randomID()
|
||||
rc.blocks = slices.Clone(r.blocks)
|
||||
c.revisions = []*revision{&rc}
|
||||
}
|
||||
var children []*link
|
||||
for _, child := range d.links {
|
||||
if child.parentID == l.id && child.state == linkStateActive {
|
||||
children = append(children, child)
|
||||
}
|
||||
}
|
||||
sort.Slice(children, func(i, j int) bool { return children[i].seq < children[j].seq })
|
||||
d.addLink(&c)
|
||||
for _, child := range children {
|
||||
d.copyLink(child, c.id, now)
|
||||
}
|
||||
return &c
|
||||
}
|
||||
|
||||
// handleTrash trashes up to 100 links, answering for each.
|
||||
func (s *Server) handleTrash(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
|
||||
var req linksRequest
|
||||
if !decodeRequest(w, r, &req) {
|
||||
return
|
||||
}
|
||||
if len(req.LinkIDs) > maxLinksPerRequest {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidValue, fmt.Sprintf("protontest: %d link IDs, at most %d allowed", len(req.LinkIDs), maxLinksPerRequest))
|
||||
return
|
||||
}
|
||||
now := time.Now().Unix()
|
||||
resp := &multiResponse{Code: api.CodeOKMany, Responses: []linkResultJSON{}}
|
||||
for _, id := range req.LinkIDs {
|
||||
result := linkResultJSON{LinkID: id, Response: codeJSON{Code: api.CodeOK}}
|
||||
l := d.activeLink(id)
|
||||
switch {
|
||||
case l == nil:
|
||||
result.Response = codeJSON{Code: api.CodeNotExists, Error: "File or folder not found"}
|
||||
case l.parentID == "":
|
||||
result.Response = codeJSON{Code: api.CodeInvalidRequirements, Error: "protontest: the root can't be trashed"}
|
||||
default:
|
||||
l.state, l.trashTime, l.modifyTime = linkStateTrashed, now, now
|
||||
}
|
||||
resp.Responses = append(resp.Responses, result)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
// handleEmptyTrash deletes every trashed link with its descendants.
|
||||
func (s *Server) handleEmptyTrash(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
|
||||
var doomed []string
|
||||
for id, l := range d.links {
|
||||
if d.inTrash(l) {
|
||||
doomed = append(doomed, id)
|
||||
}
|
||||
}
|
||||
for _, id := range doomed {
|
||||
delete(d.links, id)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, &codeJSON{Code: api.CodeOK})
|
||||
}
|
||||
|
||||
// handleCheckAvailableHashes reports which name hashes are free in a
|
||||
// folder, and the drafts holding the others unless they belong to one
|
||||
// of the given ClientUIDs.
|
||||
func (s *Server) handleCheckAvailableHashes(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
|
||||
var req checkHashesRequest
|
||||
if !decodeRequest(w, r, &req) {
|
||||
return
|
||||
}
|
||||
parent := d.activeFolder(r.PathValue("linkID"))
|
||||
if parent == nil {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "Folder not found")
|
||||
return
|
||||
}
|
||||
resp := &availableHashesResponse{AvailableHashes: []string{}, PendingHashes: []pendingHashJSON{}, Code: api.CodeOK}
|
||||
for _, hash := range req.Hashes {
|
||||
other := d.conflict(parent.id, hash, "")
|
||||
if other == nil {
|
||||
resp.AvailableHashes = append(resp.AvailableHashes, hash)
|
||||
continue
|
||||
}
|
||||
if other.state != linkStateDraft {
|
||||
continue
|
||||
}
|
||||
for _, rev := range other.revisions {
|
||||
if rev.state == revisionStateDraft && !slices.Contains(req.ClientUID, rev.clientUID) {
|
||||
resp.PendingHashes = append(resp.PendingHashes, pendingHashJSON{
|
||||
Hash: hash,
|
||||
RevisionID: rev.id,
|
||||
LinkID: other.id,
|
||||
ClientUID: nullable(rev.clientUID),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
@@ -0,0 +1,542 @@
|
||||
package protontest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ProtonMail/go-crypto/openpgp/armor"
|
||||
"github.com/ProtonMail/go-crypto/openpgp/packet"
|
||||
"github.com/ProtonMail/go-crypto/openpgp/s2k"
|
||||
"github.com/ProtonMail/gopenpgp/v3/crypto"
|
||||
"github.com/ProtonMail/gopenpgp/v3/profile"
|
||||
"github.com/rclone/rclone/backend/protondrive/api"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// sendJSON sends in as JSON with method to path as sess and returns
|
||||
// the HTTP status and body.
|
||||
func sendJSON(t *testing.T, s *Server, sess Session, method, path string, in any) (int, []byte) {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(in)
|
||||
require.NoError(t, err)
|
||||
req, err := http.NewRequest(method, s.URL()+path, bytes.NewReader(body))
|
||||
require.NoError(t, err)
|
||||
for k, v := range sessionHeader(sess) {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
resp, err := s.Client().Do(req)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
out, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
return resp.StatusCode, out
|
||||
}
|
||||
|
||||
// requireRejected fails the test unless the response is HTTP 422 with
|
||||
// code, and its message contains each of want.
|
||||
func requireRejected(t *testing.T, code, status int, body []byte, want ...string) {
|
||||
t.Helper()
|
||||
require.Equal(t, http.StatusUnprocessableEntity, status, "body: %s", body)
|
||||
e := decodeError(t, body)
|
||||
assert.Equal(t, code, e.Code, "body: %s", body)
|
||||
for _, w := range want {
|
||||
assert.Contains(t, e.Message, w)
|
||||
}
|
||||
}
|
||||
|
||||
// newFolderRequest returns a create folder request for name in
|
||||
// parentID made as the SDK's createFolder does.
|
||||
func newFolderRequest(u *User, parentID, name string) *createFolderRequest {
|
||||
v := u.view()
|
||||
parentKey, parentHashKey := v.folderKeys(parentID)
|
||||
node := newDriveKey(false)
|
||||
pass := sealPassphrase(node.passphrase, parentKey, v.addrKey)
|
||||
_, hashKey := newHashKey(node.key)
|
||||
xattr := encryptArmored([]byte(`{"Common":{"ModificationTime":"2024-01-02T03:04:05.000Z"}}`), node.key, v.addrKey, true)
|
||||
return &createFolderRequest{
|
||||
ParentLinkID: parentID,
|
||||
NodeKey: node.armored,
|
||||
NodeHashKey: hashKey,
|
||||
NodePassphrase: pass.armored,
|
||||
NodePassphraseSignature: pass.signature,
|
||||
SignatureEmail: &v.email,
|
||||
Name: encryptArmored([]byte(name), parentKey, v.addrKey, false),
|
||||
Hash: lookupHash(parentHashKey, name),
|
||||
XAttr: &xattr,
|
||||
}
|
||||
}
|
||||
|
||||
// messagePackets returns the packets of an armored message.
|
||||
func messagePackets(t *testing.T, armored string) []*packet.OpaquePacket {
|
||||
t.Helper()
|
||||
block, err := armor.Decode(strings.NewReader(armored))
|
||||
require.NoError(t, err)
|
||||
var out []*packet.OpaquePacket
|
||||
r := packet.NewOpaqueReader(block.Body)
|
||||
for {
|
||||
op, err := r.Next()
|
||||
if err == io.EOF {
|
||||
return out
|
||||
}
|
||||
require.NoError(t, err)
|
||||
out = append(out, op)
|
||||
}
|
||||
}
|
||||
|
||||
// armorPackets returns packets as an armored message.
|
||||
func armorPackets(t *testing.T, packets ...*packet.OpaquePacket) string {
|
||||
t.Helper()
|
||||
var out bytes.Buffer
|
||||
w, err := armor.Encode(&out, "PGP MESSAGE", nil)
|
||||
require.NoError(t, err)
|
||||
for _, op := range packets {
|
||||
require.NoError(t, op.Serialize(w))
|
||||
}
|
||||
require.NoError(t, w.Close())
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// aeadMessages returns messages in the formats clients must not write
|
||||
// for names, passphrases, hash keys and extended attributes, by name.
|
||||
func aeadMessages(t *testing.T) map[string]string {
|
||||
t.Helper()
|
||||
// Encrypting to a file key, which advertises AEAD, gives a v6 PKESK
|
||||
// and SEIPDv2, as gopenpgp does unless told otherwise.
|
||||
eh, err := aeadPGP().Encryption().Recipient(newDriveKey(true).key).New()
|
||||
require.NoError(t, err)
|
||||
msg, err := eh.Encrypt([]byte("value"))
|
||||
require.NoError(t, err)
|
||||
armored, err := msg.Armor()
|
||||
require.NoError(t, err)
|
||||
v6 := messagePackets(t, armored)
|
||||
require.Len(t, v6, 2)
|
||||
require.Equal(t, []byte{6, 2}, []byte{v6[0].Contents[0], v6[1].Contents[0]})
|
||||
v3 := messagePackets(t, encryptArmored([]byte("value"), newDriveKey(false).key, nil, false))
|
||||
require.Len(t, v3, 2)
|
||||
require.Equal(t, []byte{3, 1}, []byte{v3[0].Contents[0], v3[1].Contents[0]})
|
||||
return map[string]string{
|
||||
"v6 PKESK and SEIPDv2": armored,
|
||||
"v3 PKESK and SEIPDv2": armorPackets(t, v3[0], v6[1]),
|
||||
"v6 PKESK and SEIPDv1": armorPackets(t, v6[0], v3[1]),
|
||||
}
|
||||
}
|
||||
|
||||
// lockedWith returns a new v4 node key locked by a handle of p.
|
||||
func lockedWith(t *testing.T, p *profile.Custom) string {
|
||||
t.Helper()
|
||||
key, err := crypto.PGP().KeyGeneration().AddUserId(driveKeyName, "").New().GenerateKey()
|
||||
require.NoError(t, err)
|
||||
locked, err := crypto.PGPWithProfile(p).LockKey(key, newPassphrase())
|
||||
require.NoError(t, err)
|
||||
armored, err := locked.Armor()
|
||||
require.NoError(t, err)
|
||||
return armored
|
||||
}
|
||||
|
||||
func TestCreateFolderFormats(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
sess := s.NewSession(u)
|
||||
path := "/drive/v2/volumes/" + u.VolumeID() + "/folders"
|
||||
post := func(req *createFolderRequest) (int, []byte) {
|
||||
return sendJSON(t, s, sess, "POST", path, req)
|
||||
}
|
||||
|
||||
for variant, bad := range aeadMessages(t) {
|
||||
for field, spoil := range map[string]func(req *createFolderRequest){
|
||||
"Name": func(req *createFolderRequest) { req.Name = bad },
|
||||
"NodePassphrase": func(req *createFolderRequest) { req.NodePassphrase = bad },
|
||||
"NodeHashKey": func(req *createFolderRequest) { req.NodeHashKey = bad },
|
||||
"XAttr": func(req *createFolderRequest) { req.XAttr = &bad },
|
||||
} {
|
||||
t.Run(field+" "+variant, func(t *testing.T) {
|
||||
req := newFolderRequest(u, u.RootID(), "dir")
|
||||
spoil(req)
|
||||
status, body := post(req)
|
||||
requireRejected(t, api.CodeInvalidValue, status, body, field)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
aeadLocked := profile.Default()
|
||||
aeadLocked.AeadKeyEncryption = &packet.AEADConfig{}
|
||||
argon2 := profile.Default()
|
||||
argon2.AeadKeyEncryption = &packet.AEADConfig{}
|
||||
argon2.S2kKeyEncryption = &s2k.Config{S2KMode: s2k.Argon2S2K, Argon2Config: &s2k.Argon2Config{}}
|
||||
salted := profile.Default()
|
||||
salted.S2kKeyEncryption = &s2k.Config{S2KMode: s2k.SaltedS2K, PassphraseIsHighEntropy: true}
|
||||
v6Key, err := crypto.PGPWithProfile(profile.RFC9580()).KeyGeneration().AddUserId(driveKeyName, "").New().GenerateKey()
|
||||
require.NoError(t, err)
|
||||
v6Locked, err := crypto.PGPWithProfile(profile.RFC9580()).LockKey(v6Key, newPassphrase())
|
||||
require.NoError(t, err)
|
||||
v6Armored, err := v6Locked.Armor()
|
||||
require.NoError(t, err)
|
||||
unlocked, err := generateKey(driveKeyName, "").Armor()
|
||||
require.NoError(t, err)
|
||||
for name, tc := range map[string]struct {
|
||||
key string
|
||||
want string
|
||||
}{
|
||||
"advertises AEAD": {newDriveKey(true).armored, "advertises AEAD"},
|
||||
"v6": {v6Armored, "not a v4 key"},
|
||||
"AEAD locked": {lockedWith(t, aeadLocked), "locked with AEAD"},
|
||||
"Argon2": {lockedWith(t, argon2), "locked with AEAD"},
|
||||
"salted S2K": {lockedWith(t, salted), "S2K mode 1"},
|
||||
"not locked": {unlocked, "not locked"},
|
||||
} {
|
||||
t.Run("NodeKey "+name, func(t *testing.T) {
|
||||
req := newFolderRequest(u, u.RootID(), "dir")
|
||||
req.NodeKey = tc.key
|
||||
status, body := post(req)
|
||||
requireRejected(t, api.CodeInvalidValue, status, body, "NodeKey", tc.want)
|
||||
})
|
||||
}
|
||||
|
||||
for name, spoil := range map[string]func(req *createFolderRequest){
|
||||
"no hash key": func(req *createFolderRequest) { req.NodeHashKey = "" },
|
||||
"no signature": func(req *createFolderRequest) { req.NodePassphraseSignature = "" },
|
||||
"bad hash": func(req *createFolderRequest) { req.Hash = "ABC" },
|
||||
"no signature email": func(req *createFolderRequest) { req.SignatureEmail = nil },
|
||||
"someone else's email": func(req *createFolderRequest) { other := "other@proton.test"; req.SignatureEmail = &other },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
req := newFolderRequest(u, u.RootID(), "dir")
|
||||
spoil(req)
|
||||
status, body := post(req)
|
||||
requireRejected(t, api.CodeInvalidValue, status, body)
|
||||
})
|
||||
}
|
||||
assert.Empty(t, u.Dump())
|
||||
|
||||
req := newFolderRequest(u, u.RootID(), "dir")
|
||||
req.ParentLinkID = "unknown"
|
||||
status, body := post(req)
|
||||
requireRejected(t, api.CodeNotExists, status, body)
|
||||
|
||||
var created createFolderResponse
|
||||
status, body = post(newFolderRequest(u, u.RootID(), "dir"))
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
require.NoError(t, json.Unmarshal(body, &created))
|
||||
dump := u.Dump()
|
||||
require.Len(t, dump, 1)
|
||||
assert.Equal(t, created.Folder.ID, dump[0].ID)
|
||||
assert.Equal(t, "dir", dump[0].Path)
|
||||
assert.NoError(t, dump[0].Err)
|
||||
assert.True(t, time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC).Equal(dump[0].ModTime))
|
||||
|
||||
// The name hash must be unique among the folder's untrashed nodes.
|
||||
fileID := u.SeedFile(u.RootID(), "file", []byte("x"), time.Now())
|
||||
status, body = post(newFolderRequest(u, u.RootID(), "file"))
|
||||
requireRejected(t, api.CodeAlreadyExists, status, body)
|
||||
var conflict conflictResponse
|
||||
require.NoError(t, json.Unmarshal(body, &conflict))
|
||||
assert.Equal(t, fileID, conflict.Details.ConflictLinkID)
|
||||
require.NotNil(t, conflict.Details.ConflictRevisionID)
|
||||
assert.Equal(t, u.view().d.links[fileID].activeRevision().id, *conflict.Details.ConflictRevisionID)
|
||||
}
|
||||
|
||||
// renameMoveCopy holds a user with folders a and b, a signed file f in
|
||||
// a and an anonymous file anon in a, and a session.
|
||||
type renameMoveCopy struct {
|
||||
s *Server
|
||||
u *User
|
||||
sess Session
|
||||
a, b, f, anon string
|
||||
volume string
|
||||
validMsg, aeadMsg string
|
||||
}
|
||||
|
||||
func newRenameMoveCopy(t *testing.T) *renameMoveCopy {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
x := &renameMoveCopy{s: s, u: u, sess: s.NewSession(u), volume: u.VolumeID()}
|
||||
x.a = u.SeedFolder(u.RootID(), "a")
|
||||
x.b = u.SeedFolder(u.RootID(), "b")
|
||||
x.f = u.SeedFile(x.a, "f", []byte("f"), time.Now())
|
||||
x.anon = u.SeedAnonymousFile(x.a, "anon", []byte("anon"), time.Now())
|
||||
// The fake doesn't decrypt what it is sent, so any message in the
|
||||
// right format will do.
|
||||
x.validMsg = encryptArmored([]byte("value"), newDriveKey(false).key, nil, false)
|
||||
x.aeadMsg = aeadMessages(t)["v6 PKESK and SEIPDv2"]
|
||||
return x
|
||||
}
|
||||
|
||||
// hash returns the stored name hash of link id.
|
||||
func (x *renameMoveCopy) hash(id string) string {
|
||||
return x.u.view().d.links[id].nameHash
|
||||
}
|
||||
|
||||
func TestRenameRules(t *testing.T) {
|
||||
x := newRenameMoveCopy(t)
|
||||
path := func(id string) string { return "/drive/v2/volumes/" + x.volume + "/links/" + id + "/rename" }
|
||||
email := x.u.Email()
|
||||
newHash := lookupHash([]byte("k"), "new")
|
||||
req := func(original string) *renameRequest {
|
||||
return &renameRequest{Name: x.validMsg, Hash: &newHash, NameSignatureEmail: &email, OriginalHash: &original}
|
||||
}
|
||||
|
||||
status, body := sendJSON(t, x.s, x.sess, "PUT", path("unknown"), req(""))
|
||||
requireRejected(t, api.CodeNotExists, status, body)
|
||||
status, body = sendJSON(t, x.s, x.sess, "PUT", path(x.f), req("stale"))
|
||||
requireRejected(t, api.CodeInvalidRequirements, status, body, "OriginalHash")
|
||||
status, body = sendJSON(t, x.s, x.sess, "PUT", path(x.u.RootID()), req(""))
|
||||
requireRejected(t, api.CodeInvalidRequirements, status, body)
|
||||
bad := req(x.hash(x.f))
|
||||
bad.Name = x.aeadMsg
|
||||
status, body = sendJSON(t, x.s, x.sess, "PUT", path(x.f), bad)
|
||||
requireRejected(t, api.CodeInvalidValue, status, body, "Name")
|
||||
taken := req(x.hash(x.f))
|
||||
anonHash := x.hash(x.anon)
|
||||
taken.Hash = &anonHash
|
||||
status, body = sendJSON(t, x.s, x.sess, "PUT", path(x.f), taken)
|
||||
requireRejected(t, api.CodeAlreadyExists, status, body)
|
||||
|
||||
// Renaming to its own name is no conflict.
|
||||
same := req(x.hash(x.f))
|
||||
ownHash := x.hash(x.f)
|
||||
same.Hash = &ownHash
|
||||
status, body = sendJSON(t, x.s, x.sess, "PUT", path(x.f), same)
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
status, body = sendJSON(t, x.s, x.sess, "PUT", path(x.f), req(x.hash(x.f)))
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
assert.Equal(t, newHash, x.hash(x.f))
|
||||
}
|
||||
|
||||
func TestMoveRules(t *testing.T) {
|
||||
x := newRenameMoveCopy(t)
|
||||
path := func(id string) string { return "/drive/v2/volumes/" + x.volume + "/links/" + id + "/move" }
|
||||
email := x.u.Email()
|
||||
other := "other@proton.test"
|
||||
sig := "signature"
|
||||
req := func(id, parentID string) *moveRequest {
|
||||
return &moveRequest{
|
||||
Name: x.validMsg,
|
||||
NodePassphrase: x.validMsg,
|
||||
Hash: lookupHash([]byte("k"), "new"),
|
||||
ParentLinkID: parentID,
|
||||
OriginalHash: x.hash(id),
|
||||
NameSignatureEmail: &email,
|
||||
}
|
||||
}
|
||||
sub := x.u.SeedFolder(x.a, "sub")
|
||||
|
||||
for name, tc := range map[string]struct {
|
||||
id string
|
||||
req *moveRequest
|
||||
code int
|
||||
reason string
|
||||
}{
|
||||
"unknown link": {"unknown", req(x.f, x.b), api.CodeNotExists, ""},
|
||||
"unknown parent": {x.f, req(x.f, "unknown"), api.CodeNotExists, ""},
|
||||
"parent is a file": {x.anon, req(x.anon, x.f), api.CodeNotExists, ""},
|
||||
"into itself": {x.a, req(x.a, x.a), api.CodeInvalidRequirements, "into itself"},
|
||||
"into a descendant": {x.a, req(x.a, sub), api.CodeInvalidRequirements, "into itself"},
|
||||
"the root": {x.u.RootID(), req(x.u.RootID(), x.b), api.CodeInvalidRequirements, ""},
|
||||
"stale original hash": {x.f, func() *moveRequest { r := req(x.f, x.b); r.OriginalHash = "stale"; return r }(), api.CodeInvalidRequirements, "OriginalHash"},
|
||||
"AEAD name": {x.f, func() *moveRequest { r := req(x.f, x.b); r.Name = x.aeadMsg; return r }(), api.CodeInvalidValue, "Name"},
|
||||
"AEAD passphrase": {x.f, func() *moveRequest { r := req(x.f, x.b); r.NodePassphrase = x.aeadMsg; return r }(), api.CodeInvalidValue, "NodePassphrase"},
|
||||
"name signed by another": {x.f, func() *moveRequest { r := req(x.f, x.b); r.NameSignatureEmail = &other; return r }(), api.CodeInvalidValue, "NameSignatureEmail"},
|
||||
"signed node re-signed": {x.f, func() *moveRequest {
|
||||
r := req(x.f, x.b)
|
||||
r.NodePassphraseSignature, r.SignatureEmail = &sig, &email
|
||||
return r
|
||||
}(), api.CodeInvalidValue, "must not be given"},
|
||||
"anonymous node unsigned": {x.anon, req(x.anon, x.b), api.CodeInvalidValue, "anonymous"},
|
||||
"anonymous node signed by another": {x.anon, func() *moveRequest {
|
||||
r := req(x.anon, x.b)
|
||||
r.NodePassphraseSignature, r.SignatureEmail = &sig, &other
|
||||
return r
|
||||
}(), api.CodeInvalidValue, "SignatureEmail"},
|
||||
"name taken": {x.f, func() *moveRequest { r := req(x.f, x.a); r.Hash = x.hash(x.anon); return r }(), api.CodeAlreadyExists, ""},
|
||||
} {
|
||||
status, body := sendJSON(t, x.s, x.sess, "PUT", path(tc.id), tc.req)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
requireRejected(t, tc.code, status, body, tc.reason)
|
||||
})
|
||||
}
|
||||
|
||||
// A signed node keeps its passphrase signature; an anonymous one
|
||||
// takes the one given.
|
||||
before := *x.u.view().d.links[x.f]
|
||||
status, body := sendJSON(t, x.s, x.sess, "PUT", path(x.f), req(x.f, x.b))
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
after := x.u.view().d.links[x.f]
|
||||
assert.Equal(t, x.b, after.parentID)
|
||||
assert.Equal(t, before.nodePass.signature, after.nodePass.signature)
|
||||
assert.Equal(t, before.signatureEmail, after.signatureEmail)
|
||||
assert.Equal(t, x.validMsg, after.nodePass.armored)
|
||||
|
||||
r := req(x.anon, x.b)
|
||||
r.Hash = lookupHash([]byte("k"), "anon")
|
||||
r.NodePassphraseSignature, r.SignatureEmail = &sig, &email
|
||||
status, body = sendJSON(t, x.s, x.sess, "PUT", path(x.anon), r)
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
anon := x.u.view().d.links[x.anon]
|
||||
assert.Equal(t, sig, anon.nodePass.signature)
|
||||
assert.Equal(t, email, anon.signatureEmail)
|
||||
assert.Equal(t, email, anon.nameSignatureEmail)
|
||||
}
|
||||
|
||||
func TestCopyRules(t *testing.T) {
|
||||
x := newRenameMoveCopy(t)
|
||||
path := func(id string) string { return "/drive/volumes/" + x.volume + "/links/" + id + "/copy" }
|
||||
email := x.u.Email()
|
||||
req := func(parentID string) *copyRequest {
|
||||
return ©Request{
|
||||
Name: x.validMsg,
|
||||
NodePassphrase: x.validMsg,
|
||||
Hash: lookupHash([]byte("k"), "copy"),
|
||||
TargetVolumeID: x.volume,
|
||||
TargetParentLinkID: parentID,
|
||||
NameSignatureEmail: &email,
|
||||
}
|
||||
}
|
||||
status, body := sendJSON(t, x.s, x.sess, "POST", path("unknown"), req(x.b))
|
||||
requireRejected(t, api.CodeNotExists, status, body)
|
||||
wrongVolume := req(x.b)
|
||||
wrongVolume.TargetVolumeID = "unknown"
|
||||
status, body = sendJSON(t, x.s, x.sess, "POST", path(x.f), wrongVolume)
|
||||
requireRejected(t, api.CodeNotExists, status, body)
|
||||
status, body = sendJSON(t, x.s, x.sess, "POST", path(x.a), req(x.a))
|
||||
requireRejected(t, api.CodeInvalidRequirements, status, body, "into itself")
|
||||
status, body = sendJSON(t, x.s, x.sess, "POST", path(x.anon), req(x.b))
|
||||
requireRejected(t, api.CodeInvalidValue, status, body, "anonymous")
|
||||
aead := req(x.b)
|
||||
aead.NodePassphrase = x.aeadMsg
|
||||
status, body = sendJSON(t, x.s, x.sess, "POST", path(x.f), aead)
|
||||
requireRejected(t, api.CodeInvalidValue, status, body, "NodePassphrase")
|
||||
taken := req(x.a)
|
||||
taken.Hash = x.hash(x.f)
|
||||
status, body = sendJSON(t, x.s, x.sess, "POST", path(x.f), taken)
|
||||
requireRejected(t, api.CodeAlreadyExists, status, body)
|
||||
|
||||
status, body = sendJSON(t, x.s, x.sess, "POST", path(x.f), req(x.b))
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
var resp copyResponse
|
||||
require.NoError(t, json.Unmarshal(body, &resp))
|
||||
v := x.u.view()
|
||||
src, cp := v.d.links[x.f], v.d.links[resp.LinkID]
|
||||
require.NotNil(t, cp)
|
||||
assert.Equal(t, x.b, cp.parentID)
|
||||
assert.Equal(t, src.nodeKey, cp.nodeKey)
|
||||
assert.Equal(t, src.contentKeyPacket, cp.contentKeyPacket)
|
||||
assert.Equal(t, src.nodePass.signature, cp.nodePass.signature)
|
||||
assert.NotEqual(t, src.activeRevision().id, cp.activeRevision().id)
|
||||
assert.Equal(t, src.activeRevision().blocks, cp.activeRevision().blocks)
|
||||
}
|
||||
|
||||
func TestTrashAndEmptyTrash(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
sess := s.NewSession(u)
|
||||
dir := u.SeedFolder(u.RootID(), "dir")
|
||||
child := u.SeedFile(dir, "child", []byte("c"), time.Now())
|
||||
kept := u.SeedFile(u.RootID(), "kept", []byte("k"), time.Now())
|
||||
|
||||
var resp multiResponse
|
||||
status := postJSON(t, s, "/drive/v2/volumes/"+u.VolumeID()+"/trash_multiple", sessionHeader(sess),
|
||||
&linksRequest{LinkIDs: []string{dir, "unknown", u.RootID()}}, &resp)
|
||||
require.Equal(t, http.StatusOK, status)
|
||||
assert.Equal(t, api.CodeOKMany, resp.Code)
|
||||
require.Len(t, resp.Responses, 3)
|
||||
assert.Equal(t, linkResultJSON{LinkID: dir, Response: codeJSON{Code: api.CodeOK}}, resp.Responses[0])
|
||||
assert.Equal(t, api.CodeNotExists, resp.Responses[1].Response.Code)
|
||||
assert.NotEmpty(t, resp.Responses[1].Response.Error)
|
||||
assert.Equal(t, api.CodeInvalidRequirements, resp.Responses[2].Response.Code)
|
||||
|
||||
states := map[string]int{}
|
||||
for _, n := range u.Dump() {
|
||||
require.NoError(t, n.Err)
|
||||
states[n.ID] = n.State
|
||||
}
|
||||
assert.Equal(t, map[string]int{dir: linkStateTrashed, child: linkStateActive, kept: linkStateActive}, states)
|
||||
|
||||
status, body := sendJSON(t, s, sess, "DELETE", "/drive/volumes/"+u.VolumeID()+"/trash", nil)
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
dump := u.Dump()
|
||||
require.Len(t, dump, 1)
|
||||
assert.Equal(t, kept, dump[0].ID)
|
||||
}
|
||||
|
||||
func TestCheckAvailableHashes(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
sess := s.NewSession(u)
|
||||
root := u.RootID()
|
||||
u.SeedFile(root, "taken", []byte("x"), time.Now())
|
||||
trashed := u.SeedFile(root, "trashed", []byte("x"), time.Now())
|
||||
_, hashKey := u.view().folderKeys(root)
|
||||
draftID, draftRev := randomID(), randomID()
|
||||
s.mu.Lock()
|
||||
d := u.mustDriveLocked()
|
||||
d.links[trashed].state = linkStateTrashed
|
||||
d.addLink(&link{
|
||||
id: draftID,
|
||||
parentID: root,
|
||||
typ: linkTypeFile,
|
||||
state: linkStateDraft,
|
||||
nameHash: lookupHash(hashKey, "draft"),
|
||||
revisions: []*revision{{id: draftRev, state: revisionStateDraft, clientUID: "mine"}},
|
||||
})
|
||||
s.mu.Unlock()
|
||||
|
||||
hashes := []string{lookupHash(hashKey, "free"), lookupHash(hashKey, "taken"), lookupHash(hashKey, "trashed"), lookupHash(hashKey, "draft")}
|
||||
path := "/drive/v2/volumes/" + u.VolumeID() + "/links/" + root + "/checkAvailableHashes"
|
||||
var resp availableHashesResponse
|
||||
require.Equal(t, http.StatusOK, postJSON(t, s, path, sessionHeader(sess), &checkHashesRequest{Hashes: hashes}, &resp))
|
||||
assert.Equal(t, []string{hashes[0], hashes[2]}, resp.AvailableHashes)
|
||||
mine := "mine"
|
||||
assert.Equal(t, []pendingHashJSON{{Hash: hashes[3], RevisionID: draftRev, LinkID: draftID, ClientUID: &mine}}, resp.PendingHashes)
|
||||
|
||||
resp = availableHashesResponse{}
|
||||
require.Equal(t, http.StatusOK, postJSON(t, s, path, sessionHeader(sess), &checkHashesRequest{Hashes: hashes, ClientUID: []string{"mine"}}, &resp))
|
||||
assert.Equal(t, []string{hashes[0], hashes[2]}, resp.AvailableHashes)
|
||||
assert.Empty(t, resp.PendingHashes)
|
||||
|
||||
var apiErr api.Error
|
||||
assert.Equal(t, http.StatusUnprocessableEntity, postJSON(t, s, "/drive/v2/volumes/"+u.VolumeID()+"/links/unknown/checkAvailableHashes", sessionHeader(sess), &checkHashesRequest{}, &apiErr))
|
||||
assert.Equal(t, api.CodeNotExists, apiErr.Code)
|
||||
}
|
||||
|
||||
func TestDumpChecksNameHash(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
id := u.SeedFile(u.RootID(), "file", []byte("x"), time.Now())
|
||||
require.NoError(t, u.Dump()[0].Err)
|
||||
s.mu.Lock()
|
||||
u.mustDriveLocked().links[id].nameHash = lookupHash([]byte("other key"), "file")
|
||||
s.mu.Unlock()
|
||||
err := u.Dump()[0].Err
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "name hash")
|
||||
}
|
||||
|
||||
// TestMutationWireNames checks the backend's request types decode into
|
||||
// the fake's without unknown fields.
|
||||
func TestMutationWireNames(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
client any
|
||||
fake any
|
||||
}{
|
||||
{&api.CreateFolderRequest{XAttr: "x"}, &createFolderRequest{}},
|
||||
{&api.RenameRequest{}, &renameRequest{}},
|
||||
{&api.MoveRequest{NodePassphraseSignature: "x", SignatureEmail: "x"}, &moveRequest{}},
|
||||
{&api.CopyRequest{NodePassphraseSignature: "x", SignatureEmail: "x"}, ©Request{}},
|
||||
{&api.LinksRequest{}, &linksRequest{}},
|
||||
{&api.MultiLinkResponse{Responses: []api.LinkResult{{}}}, &multiResponse{}},
|
||||
{&api.CreateFolderResponse{}, &createFolderResponse{}},
|
||||
{&api.CopyResponse{}, ©Response{}},
|
||||
} {
|
||||
data, err := json.Marshal(tc.client)
|
||||
require.NoError(t, err)
|
||||
dec := json.NewDecoder(bytes.NewReader(data))
|
||||
dec.DisallowUnknownFields()
|
||||
assert.NoError(t, dec.Decode(tc.fake), "%T", tc.client)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,438 @@
|
||||
// Package protontest is a fake Proton Drive API server for tests.
|
||||
//
|
||||
// It must not import the protondrive backend, whose tests use it.
|
||||
package protontest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ProtonMail/gopenpgp/v3/crypto"
|
||||
"github.com/rclone/rclone/backend/protondrive/api"
|
||||
)
|
||||
|
||||
var (
|
||||
// appVersionRe is Proton's published x-pm-appversion grammar
|
||||
// (SDK README, "Usage Guidelines") with the name fixed to rclone.
|
||||
appVersionRe = regexp.MustCompile(`^external-drive-rclone@[0-9]+\.[0-9]+\.[0-9]+(-(stable|beta|alpha))?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$`)
|
||||
|
||||
// legacyRouteRe matches the removed share-scoped routes and
|
||||
// everything beneath them.
|
||||
legacyRouteRe = regexp.MustCompile(`^/drive/shares/[^/]+/(links|folders|files|events)(/|$)`)
|
||||
)
|
||||
|
||||
// legacyRequests counts the requests for legacy routes, per "METHOD
|
||||
// path", made to every server in the process, so a test binary can
|
||||
// check none was made after all its tests have run.
|
||||
var legacyRequests = struct {
|
||||
mu sync.Mutex
|
||||
counts map[string]int
|
||||
}{counts: make(map[string]int)}
|
||||
|
||||
// LegacyRequests returns a copy of the count, per "METHOD path", of
|
||||
// the requests for removed share-scoped routes made to every server in
|
||||
// the process since it started. ResetRequests doesn't change it.
|
||||
func LegacyRequests() map[string]int {
|
||||
legacyRequests.mu.Lock()
|
||||
defer legacyRequests.mu.Unlock()
|
||||
out := make(map[string]int, len(legacyRequests.counts))
|
||||
for k, v := range legacyRequests.counts {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Fault describes a scripted failure for requests to one route.
|
||||
type Fault struct {
|
||||
Method string // HTTP method to match; empty matches any
|
||||
Pattern string // route pattern as registered, e.g. "/tests/ping"
|
||||
Skip int // matching requests let through before faulting
|
||||
Times int // matching requests faulted after Skip; 0 means forever
|
||||
Status int // HTTP status to return; 0 with Drop false only holds the request
|
||||
Code int // Proton code in the error body
|
||||
RetryAfter int // Retry-After header in seconds, if non-zero
|
||||
Drop bool // close the connection instead of responding
|
||||
Delay time.Duration // wait before handling the request
|
||||
Gather bool // hold each faulted request until all Times have arrived
|
||||
AfterHandler bool // run the real handler, then replace its response
|
||||
|
||||
// Hold, if set, makes each faulted request run the real handler on
|
||||
// arrival and get its response only once Hold is closed, so the
|
||||
// response is as of arrival. Held, if set, is sent to once the
|
||||
// response is ready and held.
|
||||
Hold <-chan struct{}
|
||||
Held chan<- struct{}
|
||||
}
|
||||
|
||||
// fault is an injected Fault with the count of requests it has matched.
|
||||
type fault struct {
|
||||
Fault
|
||||
seen int
|
||||
gathered chan struct{} // closed when the last of Times has arrived, if Gather
|
||||
}
|
||||
|
||||
// fires records a matching request and reports whether it is faulted.
|
||||
func (f *fault) fires() bool {
|
||||
f.seen++
|
||||
n := f.seen - f.Skip
|
||||
if f.Gather && n == f.Times {
|
||||
close(f.gathered)
|
||||
}
|
||||
return n > 0 && (f.Times == 0 || n <= f.Times)
|
||||
}
|
||||
|
||||
// Server is a fake Proton API server.
|
||||
type Server struct {
|
||||
srv *httptest.Server
|
||||
mux *http.ServeMux
|
||||
|
||||
mu sync.Mutex
|
||||
requests map[string]int // count per "METHOD pattern"
|
||||
faults []*fault
|
||||
clockSkew time.Duration
|
||||
users map[string]*User // by username
|
||||
sessions map[string]*session // by UID
|
||||
srp map[string]*srpLogin // by SRPSession
|
||||
forks map[string]*fork // by Selector
|
||||
|
||||
childrenPageSize int
|
||||
downloads map[string]*download // block URLs by ID
|
||||
unlockedKeys map[string]*crypto.Key // by keyCacheID
|
||||
uploads map[string]*uploadURL // block upload URLs by ID
|
||||
storageUploads int // block uploads in progress
|
||||
storagePeak int // most block uploads in progress at once
|
||||
secrets []issuedSecret // every secret issued, in order
|
||||
keysAs map[string]*User // whose keys GET /core/v4/keys/all serves for an email, overriding the owner's
|
||||
readLag int // reads of a node after a change which don't see it; see SetReadLag
|
||||
readLagView LagView // what lagging reads of a changed node see
|
||||
lagging map[string]*lagView // nodes whose reads lag, by ID
|
||||
|
||||
// driveRequirements is sent as x-pm-drive-requirements headers if
|
||||
// not empty; see SetDriveRequirements.
|
||||
driveRequirements []string
|
||||
}
|
||||
|
||||
// New starts a fake server which is closed when the test ends.
|
||||
func New(t testing.TB) *Server {
|
||||
s := newServer()
|
||||
s.srv = httptest.NewServer(http.HandlerFunc(s.serveHTTP))
|
||||
t.Cleanup(s.Close)
|
||||
return s
|
||||
}
|
||||
|
||||
// NewStandalone starts a fake server listening on addr, such as
|
||||
// "127.0.0.1:0", for use outside tests. Call Close to stop it.
|
||||
func NewStandalone(addr string) (*Server, error) {
|
||||
l, err := net.Listen("tcp", addr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("protontest: listen: %w", err)
|
||||
}
|
||||
s := newServer()
|
||||
s.srv = httptest.NewUnstartedServer(http.HandlerFunc(s.serveHTTP))
|
||||
_ = s.srv.Listener.Close()
|
||||
s.srv.Listener = l
|
||||
s.srv.Start()
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// newServer returns a server with its routes registered, not yet
|
||||
// listening.
|
||||
func newServer() *Server {
|
||||
s := &Server{
|
||||
mux: http.NewServeMux(),
|
||||
requests: make(map[string]int),
|
||||
users: make(map[string]*User),
|
||||
sessions: make(map[string]*session),
|
||||
srp: make(map[string]*srpLogin),
|
||||
forks: make(map[string]*fork),
|
||||
|
||||
childrenPageSize: defaultChildrenPageSize,
|
||||
downloads: make(map[string]*download),
|
||||
unlockedKeys: make(map[string]*crypto.Key),
|
||||
uploads: make(map[string]*uploadURL),
|
||||
lagging: make(map[string]*lagView),
|
||||
}
|
||||
s.handle("GET", "/tests/ping", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]int{"Code": api.CodeOK})
|
||||
})
|
||||
s.registerAuth()
|
||||
s.registerAccount()
|
||||
s.registerDrive()
|
||||
return s
|
||||
}
|
||||
|
||||
// Close stops the server and blocks until all its requests are done.
|
||||
func (s *Server) Close() {
|
||||
s.srv.Close()
|
||||
}
|
||||
|
||||
// URL returns the base URL of the server.
|
||||
func (s *Server) URL() string {
|
||||
return s.srv.URL
|
||||
}
|
||||
|
||||
// Client returns an HTTP client configured for the server.
|
||||
func (s *Server) Client() *http.Client {
|
||||
return s.srv.Client()
|
||||
}
|
||||
|
||||
// Requests returns a copy of the request count per "METHOD pattern",
|
||||
// e.g. "GET /drive/v2/volumes/{volumeID}/folders/{linkID}/children".
|
||||
//
|
||||
// Requests for unregistered or legacy routes are counted under their
|
||||
// path.
|
||||
func (s *Server) Requests() map[string]int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := make(map[string]int, len(s.requests))
|
||||
for k, v := range s.requests {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ResetRequests zeroes the request counts and the peak of
|
||||
// StorageUploadPeak.
|
||||
func (s *Server) ResetRequests() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
clear(s.requests)
|
||||
s.storagePeak = s.storageUploads
|
||||
}
|
||||
|
||||
// ServeKeysAs makes GET /core/v4/keys/all serve owner's primary
|
||||
// address keys for email, as a hostile server could, whoever owns
|
||||
// email.
|
||||
func (s *Server) ServeKeysAs(email string, owner *User) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.keysAs == nil {
|
||||
s.keysAs = make(map[string]*User)
|
||||
}
|
||||
s.keysAs[email] = owner
|
||||
}
|
||||
|
||||
// SecretKind is a kind of secret the server issues.
|
||||
type SecretKind int
|
||||
|
||||
// Kinds of secret the server issues.
|
||||
const (
|
||||
SessionToken SecretKind = iota + 1 // an access or refresh token
|
||||
ForkSelector // the selector polling a session fork
|
||||
StorageToken // a block upload or download token
|
||||
)
|
||||
|
||||
// issuedSecret is a secret the server handed to a client.
|
||||
type issuedSecret struct {
|
||||
kind SecretKind
|
||||
value string
|
||||
}
|
||||
|
||||
// newSecretLocked returns a new random secret of kind and records it
|
||||
// for IssuedSecrets. s.mu must be held.
|
||||
func (s *Server) newSecretLocked(kind SecretKind) string {
|
||||
secret := randomID()
|
||||
s.secrets = append(s.secrets, issuedSecret{kind: kind, value: secret})
|
||||
return secret
|
||||
}
|
||||
|
||||
// IssuedSecrets returns every secret of the kinds given, or of any kind
|
||||
// if none is, which the server has handed to a client, in order.
|
||||
func (s *Server) IssuedSecrets(kinds ...SecretKind) []string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []string
|
||||
for _, secret := range s.secrets {
|
||||
if len(kinds) == 0 || slices.Contains(kinds, secret.kind) {
|
||||
out = append(out, secret.value)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Inject adds a fault. Faults are checked in the order they were added
|
||||
// and the first one that fires is applied. A Gather fault needs Times.
|
||||
func (s *Server) Inject(f Fault) {
|
||||
if f.Gather && f.Times <= 0 {
|
||||
panic("protontest: a Gather fault needs Times")
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.faults = append(s.faults, &fault{Fault: f, gathered: make(chan struct{})})
|
||||
}
|
||||
|
||||
// SetClockSkew shifts the Date header of every response by d.
|
||||
func (s *Server) SetClockSkew(d time.Duration) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.clockSkew = d
|
||||
}
|
||||
|
||||
// SetDriveRequirements makes every response carry an
|
||||
// x-pm-drive-requirements header with each of values, as Proton
|
||||
// announces upcoming requirements; none if values is empty.
|
||||
func (s *Server) SetDriveRequirements(values ...string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.driveRequirements = values
|
||||
}
|
||||
|
||||
// handle registers h for method and an http.ServeMux path pattern.
|
||||
func (s *Server) handle(method, pattern string, h http.HandlerFunc) {
|
||||
s.mux.HandleFunc(method+" "+pattern, h)
|
||||
}
|
||||
|
||||
// serveHTTP checks, counts and dispatches every request.
|
||||
func (s *Server) serveHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
_, pattern := s.mux.Handler(r)
|
||||
// Registered patterns are "METHOD /path"; keep the path part.
|
||||
_, pattern, _ = strings.Cut(pattern, " ")
|
||||
key := r.Method + " " + pattern
|
||||
if pattern == "" {
|
||||
key = r.Method + " " + r.URL.Path
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.requests[key]++
|
||||
w.Header().Set("Date", time.Now().Add(s.clockSkew).UTC().Format(http.TimeFormat))
|
||||
for _, v := range s.driveRequirements {
|
||||
w.Header().Add("x-pm-drive-requirements", v)
|
||||
}
|
||||
if pattern == storageUploadPattern {
|
||||
s.storageUploads++
|
||||
s.storagePeak = max(s.storagePeak, s.storageUploads)
|
||||
defer func() {
|
||||
s.mu.Lock()
|
||||
s.storageUploads--
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
switch {
|
||||
case legacyRouteRe.MatchString(r.URL.Path):
|
||||
legacyRequests.mu.Lock()
|
||||
legacyRequests.counts[r.Method+" "+r.URL.Path]++
|
||||
legacyRequests.mu.Unlock()
|
||||
writeError(w, http.StatusGone, 0, "protontest: legacy route "+r.URL.Path+" was removed")
|
||||
return
|
||||
case r.Header.Get("x-pm-drive-sdk-version") != "":
|
||||
writeError(w, http.StatusBadRequest, api.CodeInvalidSection, "protontest: x-pm-drive-sdk-version must not be sent")
|
||||
return
|
||||
case !appVersionRe.MatchString(r.Header.Get("x-pm-appversion")):
|
||||
writeError(w, http.StatusBadRequest, api.CodeInvalidSection, fmt.Sprintf("protontest: invalid x-pm-appversion %q", r.Header.Get("x-pm-appversion")))
|
||||
return
|
||||
case pattern == "":
|
||||
writeError(w, http.StatusNotFound, 0, "protontest: no route for "+r.Method+" "+r.URL.Path)
|
||||
return
|
||||
}
|
||||
|
||||
if f := s.activeFault(r.Method, pattern); f != nil {
|
||||
s.serveFault(w, r, f)
|
||||
return
|
||||
}
|
||||
s.mux.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
// activeFault records the request against every matching fault and
|
||||
// returns the first one that fires, or nil.
|
||||
func (s *Server) activeFault(method, pattern string) *fault {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var active *fault
|
||||
for _, f := range s.faults {
|
||||
if f.Pattern != pattern || (f.Method != "" && f.Method != method) {
|
||||
continue
|
||||
}
|
||||
if f.fires() && active == nil {
|
||||
active = f
|
||||
}
|
||||
}
|
||||
return active
|
||||
}
|
||||
|
||||
// serveFault applies f to a request for a registered route.
|
||||
func (s *Server) serveFault(w http.ResponseWriter, r *http.Request, f *fault) {
|
||||
if f.Delay > 0 || f.Gather {
|
||||
// The server notices a client giving up, and cancels the request
|
||||
// context, only once the request body has been read.
|
||||
if body, err := io.ReadAll(r.Body); err == nil {
|
||||
r.Body = io.NopCloser(bytes.NewReader(body))
|
||||
}
|
||||
}
|
||||
if f.Gather {
|
||||
select {
|
||||
case <-f.gathered:
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
if f.Delay > 0 {
|
||||
select {
|
||||
case <-time.After(f.Delay):
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
if f.Hold != nil {
|
||||
rec := httptest.NewRecorder()
|
||||
s.mux.ServeHTTP(rec, r)
|
||||
if f.Held != nil {
|
||||
f.Held <- struct{}{}
|
||||
}
|
||||
select {
|
||||
case <-f.Hold:
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
}
|
||||
for k, v := range rec.Header() {
|
||||
w.Header()[k] = v
|
||||
}
|
||||
w.WriteHeader(rec.Code)
|
||||
_, _ = w.Write(rec.Body.Bytes())
|
||||
return
|
||||
}
|
||||
if f.Status == 0 && !f.Drop {
|
||||
s.mux.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
if f.AfterHandler {
|
||||
s.mux.ServeHTTP(httptest.NewRecorder(), r)
|
||||
}
|
||||
if f.Drop {
|
||||
conn, _, err := http.NewResponseController(w).Hijack()
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("protontest: drop connection: %v", err))
|
||||
}
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
if f.RetryAfter > 0 {
|
||||
w.Header().Set("Retry-After", strconv.Itoa(f.RetryAfter))
|
||||
}
|
||||
writeError(w, f.Status, f.Code, "protontest: injected fault")
|
||||
}
|
||||
|
||||
// writeJSON writes v as a JSON response with the given HTTP status.
|
||||
func writeJSON(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
|
||||
// writeError writes an api.Error response.
|
||||
func writeError(w http.ResponseWriter, status, code int, msg string) {
|
||||
writeJSON(w, status, &api.Error{Code: code, Message: msg})
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
package protontest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/protondrive/api"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const testAppVersion = "external-drive-rclone@1.77.0-stable"
|
||||
|
||||
// do sends a request with the given headers and returns the response
|
||||
// and its body.
|
||||
func do(t *testing.T, s *Server, method, path string, header map[string]string) (*http.Response, []byte) {
|
||||
t.Helper()
|
||||
req, err := http.NewRequest(method, s.URL()+path, nil)
|
||||
require.NoError(t, err)
|
||||
for k, v := range header {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
resp, err := s.Client().Do(req)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
return resp, body
|
||||
}
|
||||
|
||||
// ping calls GET /tests/ping with a valid app version.
|
||||
func ping(t *testing.T, s *Server) (*http.Response, []byte) {
|
||||
t.Helper()
|
||||
return do(t, s, "GET", "/tests/ping", map[string]string{"x-pm-appversion": testAppVersion})
|
||||
}
|
||||
|
||||
func decodeError(t *testing.T, body []byte) api.Error {
|
||||
t.Helper()
|
||||
var e api.Error
|
||||
require.NoError(t, json.Unmarshal(body, &e), "body: %s", body)
|
||||
return e
|
||||
}
|
||||
|
||||
func TestLegacyRoutesGone(t *testing.T) {
|
||||
s := New(t)
|
||||
for _, path := range []string{
|
||||
"/drive/shares/abc/links/def",
|
||||
"/drive/shares/abc/folders/def/children",
|
||||
"/drive/shares/abc/files",
|
||||
"/drive/shares/abc/events/latest",
|
||||
} {
|
||||
for _, method := range []string{"GET", "POST"} {
|
||||
resp, _ := do(t, s, method, path, map[string]string{"x-pm-appversion": testAppVersion})
|
||||
assert.Equal(t, http.StatusGone, resp.StatusCode, "%s %s", method, path)
|
||||
assert.Equal(t, 1, s.Requests()[method+" "+path], "%s %s counted", method, path)
|
||||
}
|
||||
}
|
||||
|
||||
// The share itself is not a legacy route.
|
||||
resp, _ := do(t, s, "GET", "/drive/shares/abc", map[string]string{"x-pm-appversion": testAppVersion})
|
||||
assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
|
||||
}
|
||||
|
||||
func TestLegacyRequestsProcessWide(t *testing.T) {
|
||||
before := LegacyRequests()
|
||||
s1, s2 := New(t), New(t)
|
||||
do(t, s1, "GET", "/drive/shares/abc/links/def", map[string]string{"x-pm-appversion": testAppVersion})
|
||||
do(t, s2, "GET", "/drive/shares/abc/links/def", map[string]string{"x-pm-appversion": testAppVersion})
|
||||
do(t, s2, "POST", "/drive/shares/abc/files", nil)
|
||||
do(t, s2, "GET", "/drive/shares/abc", nil)
|
||||
s1.ResetRequests()
|
||||
s2.ResetRequests()
|
||||
|
||||
after := LegacyRequests()
|
||||
assert.Equal(t, before["GET /drive/shares/abc/links/def"]+2, after["GET /drive/shares/abc/links/def"])
|
||||
assert.Equal(t, before["POST /drive/shares/abc/files"]+1, after["POST /drive/shares/abc/files"], "counted whatever its headers")
|
||||
assert.NotContains(t, after, "GET /drive/shares/abc")
|
||||
after["changed"] = 1
|
||||
assert.NotContains(t, LegacyRequests(), "changed", "a copy is returned")
|
||||
}
|
||||
|
||||
func TestUnknownRoute(t *testing.T) {
|
||||
s := New(t)
|
||||
resp, body := do(t, s, "GET", "/core/v4/nothing", map[string]string{"x-pm-appversion": testAppVersion})
|
||||
assert.Equal(t, http.StatusNotFound, resp.StatusCode)
|
||||
assert.Equal(t, "application/json", resp.Header.Get("Content-Type"))
|
||||
assert.Contains(t, decodeError(t, body).Message, "GET /core/v4/nothing")
|
||||
}
|
||||
|
||||
func TestAppVersionEnforced(t *testing.T) {
|
||||
s := New(t)
|
||||
for _, version := range []string{
|
||||
"",
|
||||
"web-drive@5.2.0",
|
||||
"external-drive-rclone@1.77.0-dev",
|
||||
"external-drive-other@1.77.0-stable",
|
||||
"external-drive-rclone@1.77-stable",
|
||||
"external-drive-rclone@1.77.0-stable+",
|
||||
} {
|
||||
header := map[string]string{}
|
||||
if version != "" {
|
||||
header["x-pm-appversion"] = version
|
||||
}
|
||||
resp, body := do(t, s, "GET", "/tests/ping", header)
|
||||
assert.Equal(t, http.StatusBadRequest, resp.StatusCode, "version %q", version)
|
||||
assert.Equal(t, api.CodeInvalidSection, decodeError(t, body).Code, "version %q", version)
|
||||
}
|
||||
|
||||
for _, version := range []string{
|
||||
"external-drive-rclone@1.77.0-stable",
|
||||
"external-drive-rclone@1.77.0-beta+abc",
|
||||
"external-drive-rclone@1.77.0-alpha",
|
||||
"external-drive-rclone@1.77.0",
|
||||
"external-drive-rclone@1.77.0-alpha+abc123f.dirty",
|
||||
} {
|
||||
resp, body := do(t, s, "GET", "/tests/ping", map[string]string{"x-pm-appversion": version})
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode, "version %q", version)
|
||||
assert.JSONEq(t, `{"Code":1000}`, string(body), "version %q", version)
|
||||
}
|
||||
|
||||
resp, _ := do(t, s, "GET", "/tests/ping", map[string]string{
|
||||
"x-pm-appversion": testAppVersion,
|
||||
"x-pm-drive-sdk-version": "js@0.23.0",
|
||||
})
|
||||
assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
|
||||
}
|
||||
|
||||
func TestFaultInjection(t *testing.T) {
|
||||
t.Run("Times", func(t *testing.T) {
|
||||
s := New(t)
|
||||
s.Inject(Fault{Pattern: "/tests/ping", Times: 2, Status: 429, RetryAfter: 1})
|
||||
for i := range 2 {
|
||||
resp, _ := ping(t, s)
|
||||
assert.Equal(t, http.StatusTooManyRequests, resp.StatusCode, "call %d", i)
|
||||
assert.Equal(t, "1", resp.Header.Get("Retry-After"), "call %d", i)
|
||||
}
|
||||
resp, _ := ping(t, s)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
assert.Empty(t, resp.Header.Get("Retry-After"))
|
||||
})
|
||||
|
||||
t.Run("Skip", func(t *testing.T) {
|
||||
s := New(t)
|
||||
s.Inject(Fault{Pattern: "/tests/ping", Skip: 1, Times: 1, Status: 503, Code: 2028})
|
||||
resp, _ := ping(t, s)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
resp, body := ping(t, s)
|
||||
assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
|
||||
assert.Equal(t, api.CodeFeatureBlocked, decodeError(t, body).Code)
|
||||
resp, _ = ping(t, s)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
})
|
||||
|
||||
t.Run("Forever", func(t *testing.T) {
|
||||
s := New(t)
|
||||
s.Inject(Fault{Pattern: "/tests/ping", Status: 500})
|
||||
for range 5 {
|
||||
resp, _ := ping(t, s)
|
||||
assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Method", func(t *testing.T) {
|
||||
s := New(t)
|
||||
s.Inject(Fault{Method: "POST", Pattern: "/tests/ping", Status: 500})
|
||||
resp, _ := ping(t, s)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
})
|
||||
|
||||
t.Run("Drop", func(t *testing.T) {
|
||||
s := New(t)
|
||||
s.Inject(Fault{Pattern: "/tests/ping", Times: 1, Drop: true})
|
||||
req, err := http.NewRequest("GET", s.URL()+"/tests/ping", nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("x-pm-appversion", testAppVersion)
|
||||
resp, err := s.Client().Do(req)
|
||||
if err == nil {
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
require.Error(t, err)
|
||||
resp2, _ := ping(t, s)
|
||||
assert.Equal(t, http.StatusOK, resp2.StatusCode)
|
||||
})
|
||||
|
||||
t.Run("Delay", func(t *testing.T) {
|
||||
s := New(t)
|
||||
s.Inject(Fault{Pattern: "/tests/ping", Times: 1, Delay: 100 * time.Millisecond})
|
||||
start := time.Now()
|
||||
resp, _ := ping(t, s)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
assert.GreaterOrEqual(t, time.Since(start), 100*time.Millisecond)
|
||||
})
|
||||
|
||||
t.Run("Gather", func(t *testing.T) {
|
||||
s := New(t)
|
||||
s.Inject(Fault{Pattern: "/tests/ping", Times: 2, Gather: true})
|
||||
req, err := http.NewRequest("GET", s.URL()+"/tests/ping", nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("x-pm-appversion", testAppVersion)
|
||||
first := make(chan int)
|
||||
go func() {
|
||||
resp, err := s.Client().Do(req)
|
||||
if err != nil {
|
||||
first <- 0
|
||||
return
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
first <- resp.StatusCode
|
||||
}()
|
||||
select {
|
||||
case <-first:
|
||||
t.Fatal("first request answered before the second arrived")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
resp, _ := ping(t, s)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
assert.Equal(t, http.StatusOK, <-first)
|
||||
// Requests after Times are not held.
|
||||
resp, _ = ping(t, s)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
})
|
||||
|
||||
t.Run("Hold", func(t *testing.T) {
|
||||
s := New(t)
|
||||
n := 0
|
||||
s.handle("GET", "/tests/count", func(w http.ResponseWriter, r *http.Request) {
|
||||
n++
|
||||
writeJSON(w, http.StatusOK, map[string]int{"Code": api.CodeOK, "N": n})
|
||||
})
|
||||
release := make(chan struct{})
|
||||
held := make(chan struct{}, 1)
|
||||
s.Inject(Fault{Pattern: "/tests/count", Times: 1, Hold: release, Held: held})
|
||||
first := make(chan []byte)
|
||||
go func() {
|
||||
_, body := do(t, s, "GET", "/tests/count", map[string]string{"x-pm-appversion": testAppVersion})
|
||||
first <- body
|
||||
}()
|
||||
<-held
|
||||
// The held request was handled on arrival; later ones pass.
|
||||
_, body := do(t, s, "GET", "/tests/count", map[string]string{"x-pm-appversion": testAppVersion})
|
||||
assert.JSONEq(t, `{"Code":1000,"N":2}`, string(body))
|
||||
select {
|
||||
case <-first:
|
||||
t.Fatal("held request answered before release")
|
||||
default:
|
||||
}
|
||||
close(release)
|
||||
assert.JSONEq(t, `{"Code":1000,"N":1}`, string(<-first))
|
||||
})
|
||||
|
||||
t.Run("AfterHandler", func(t *testing.T) {
|
||||
s := New(t)
|
||||
called := 0
|
||||
s.handle("POST", "/tests/commit", func(w http.ResponseWriter, r *http.Request) {
|
||||
called++
|
||||
writeJSON(w, http.StatusOK, map[string]int{"Code": api.CodeOK})
|
||||
})
|
||||
s.Inject(Fault{Pattern: "/tests/commit", Times: 1, Status: 500, AfterHandler: true})
|
||||
resp, body := do(t, s, "POST", "/tests/commit", map[string]string{"x-pm-appversion": testAppVersion})
|
||||
assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
|
||||
assert.NotContains(t, string(body), `"Code":1000`)
|
||||
assert.Equal(t, 1, called)
|
||||
})
|
||||
}
|
||||
|
||||
func TestRequestCounting(t *testing.T) {
|
||||
s := New(t)
|
||||
s.handle("GET", "/tests/volumes/{volumeID}/folders/{linkID}/children", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"Code": api.CodeOK, "Volume": r.PathValue("volumeID")})
|
||||
})
|
||||
header := map[string]string{"x-pm-appversion": testAppVersion}
|
||||
_, body := do(t, s, "GET", "/tests/volumes/v1/folders/l1/children", header)
|
||||
assert.JSONEq(t, `{"Code":1000,"Volume":"v1"}`, string(body))
|
||||
do(t, s, "GET", "/tests/volumes/v2/folders/l2/children", header)
|
||||
ping(t, s)
|
||||
|
||||
assert.Equal(t, map[string]int{
|
||||
"GET /tests/volumes/{volumeID}/folders/{linkID}/children": 2,
|
||||
"GET /tests/ping": 1,
|
||||
}, s.Requests())
|
||||
|
||||
// The returned map is a copy.
|
||||
s.Requests()["GET /tests/ping"] = 99
|
||||
assert.Equal(t, 1, s.Requests()["GET /tests/ping"])
|
||||
|
||||
s.ResetRequests()
|
||||
assert.Empty(t, s.Requests())
|
||||
ping(t, s)
|
||||
assert.Equal(t, map[string]int{"GET /tests/ping": 1}, s.Requests())
|
||||
}
|
||||
|
||||
func TestClockSkew(t *testing.T) {
|
||||
s := New(t)
|
||||
check := func(skew time.Duration) {
|
||||
resp, _ := ping(t, s)
|
||||
date, err := http.ParseTime(resp.Header.Get("Date"))
|
||||
require.NoError(t, err)
|
||||
assert.WithinDuration(t, time.Now().Add(skew), date, 5*time.Second)
|
||||
}
|
||||
check(0)
|
||||
s.SetClockSkew(-2 * time.Hour)
|
||||
check(-2 * time.Hour)
|
||||
s.SetClockSkew(30 * time.Minute)
|
||||
check(30 * time.Minute)
|
||||
}
|
||||
|
||||
func TestErrorJSON(t *testing.T) {
|
||||
e := api.Error{Status: 422, Code: api.CodeAlreadyExists, Message: "exists", Details: json.RawMessage(`{"ConflictLinkID":"x"}`)}
|
||||
b, err := json.Marshal(&e)
|
||||
require.NoError(t, err)
|
||||
assert.JSONEq(t, `{"Code":2500,"Error":"exists","Details":{"ConflictLinkID":"x"}}`, string(b))
|
||||
assert.True(t, strings.Contains(e.Error(), "exists") && strings.Contains(e.Error(), "2500") && strings.Contains(e.Error(), "422"), e.Error())
|
||||
}
|
||||
|
||||
func TestNewStandalone(t *testing.T) {
|
||||
s, err := NewStandalone("127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
assert.False(t, strings.HasSuffix(s.URL(), ":0"), "URL %q must carry the bound port", s.URL())
|
||||
resp, _ := ping(t, s)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
|
||||
s.Close()
|
||||
_, err = s.Client().Get(s.URL() + "/tests/ping")
|
||||
assert.Error(t, err, "server still answering after Close")
|
||||
|
||||
_, err = NewStandalone("not an address")
|
||||
assert.Error(t, err)
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
package protontest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"time"
|
||||
|
||||
"github.com/ProtonMail/gopenpgp/v3/crypto"
|
||||
)
|
||||
|
||||
// StoredFile is a file node as the server stores it: the encrypted
|
||||
// metadata and content a client sent, with nothing decrypted.
|
||||
type StoredFile struct {
|
||||
Link StoredLink
|
||||
Revision StoredRevision // the active revision
|
||||
}
|
||||
|
||||
// StoredLink is the encrypted metadata of a file node.
|
||||
type StoredLink struct {
|
||||
Name string // armored, encrypted to the parent key
|
||||
NameHash string
|
||||
NodeKey string // armored, locked with the node passphrase
|
||||
NodePassphrase string // armored, encrypted to the parent key
|
||||
NodePassphraseSignature string
|
||||
SignatureEmail string
|
||||
NameSignatureEmail string
|
||||
MediaType string
|
||||
ContentKeyPacket string // base64
|
||||
ContentKeyPacketSignature string
|
||||
CreateTime int64 // Unix seconds; the time of seeding if zero
|
||||
ModifyTime int64 // Unix seconds; the time of seeding if zero
|
||||
}
|
||||
|
||||
// StoredRevision is a revision of a file node.
|
||||
type StoredRevision struct {
|
||||
ManifestSignature string
|
||||
XAttr string // armored; empty if none
|
||||
SignatureEmail string
|
||||
CreateTime int64 // Unix seconds; the time of seeding if zero
|
||||
Blocks []StoredBlock // in index order, from 1
|
||||
}
|
||||
|
||||
// StoredBlock is an encrypted block of a revision.
|
||||
type StoredBlock struct {
|
||||
Hash []byte // SHA-256 of Data
|
||||
EncSignature string
|
||||
SignatureEmail string
|
||||
Data []byte `json:"-"`
|
||||
}
|
||||
|
||||
// StoredFile returns file id with its active revision as stored. It
|
||||
// panics if id is not a file with an active revision.
|
||||
func (u *User) StoredFile(id string) StoredFile {
|
||||
u.srv.mu.Lock()
|
||||
defer u.srv.mu.Unlock()
|
||||
l := u.mustDriveLocked().links[id]
|
||||
if l == nil || l.typ != linkTypeFile {
|
||||
panic("protontest: no file " + id)
|
||||
}
|
||||
rev := l.activeRevision()
|
||||
if rev == nil {
|
||||
panic("protontest: file " + id + " has no active revision")
|
||||
}
|
||||
f := StoredFile{
|
||||
Link: StoredLink{
|
||||
Name: l.name,
|
||||
NameHash: l.nameHash,
|
||||
NodeKey: l.nodeKey,
|
||||
NodePassphrase: l.nodePass.armored,
|
||||
NodePassphraseSignature: l.nodePass.signature,
|
||||
SignatureEmail: l.signatureEmail,
|
||||
NameSignatureEmail: l.nameSignatureEmail,
|
||||
MediaType: l.mediaType,
|
||||
ContentKeyPacket: l.contentKeyPacket,
|
||||
ContentKeyPacketSignature: l.contentKeyPacketSignature,
|
||||
CreateTime: l.createTime,
|
||||
ModifyTime: l.modifyTime,
|
||||
},
|
||||
Revision: StoredRevision{
|
||||
ManifestSignature: rev.manifestSignature,
|
||||
XAttr: rev.xattr,
|
||||
SignatureEmail: rev.signatureEmail,
|
||||
CreateTime: rev.createTime,
|
||||
},
|
||||
}
|
||||
for _, b := range rev.blocks {
|
||||
f.Revision.Blocks = append(f.Revision.Blocks, StoredBlock{
|
||||
Hash: bytes.Clone(b.hash),
|
||||
EncSignature: b.encSignature,
|
||||
SignatureEmail: b.signatureEmail,
|
||||
Data: bytes.Clone(b.data),
|
||||
})
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// SeedStoredFile creates a file in parentID from f, stored as given
|
||||
// without any check, and returns its ID. It panics as insertLink does.
|
||||
func (u *User) SeedStoredFile(parentID string, f StoredFile) (id string) {
|
||||
rev := &revision{
|
||||
id: randomID(),
|
||||
state: revisionStateActive,
|
||||
createTime: f.Revision.CreateTime,
|
||||
manifestSignature: f.Revision.ManifestSignature,
|
||||
xattr: f.Revision.XAttr,
|
||||
signatureEmail: f.Revision.SignatureEmail,
|
||||
}
|
||||
for i, b := range f.Revision.Blocks {
|
||||
rev.blocks = append(rev.blocks, &block{
|
||||
index: i + 1,
|
||||
hash: bytes.Clone(b.Hash),
|
||||
data: bytes.Clone(b.Data),
|
||||
encSignature: b.EncSignature,
|
||||
signatureEmail: b.SignatureEmail,
|
||||
})
|
||||
}
|
||||
if rev.createTime == 0 {
|
||||
rev.createTime = time.Now().Unix()
|
||||
}
|
||||
return u.insertLink(parentID, &link{
|
||||
typ: linkTypeFile,
|
||||
name: f.Link.Name,
|
||||
nameHash: f.Link.NameHash,
|
||||
nodeKey: f.Link.NodeKey,
|
||||
nodePass: sealedPassphrase{armored: f.Link.NodePassphrase, signature: f.Link.NodePassphraseSignature},
|
||||
signatureEmail: f.Link.SignatureEmail,
|
||||
nameSignatureEmail: f.Link.NameSignatureEmail,
|
||||
mediaType: f.Link.MediaType,
|
||||
contentKeyPacket: f.Link.ContentKeyPacket,
|
||||
contentKeyPacketSignature: f.Link.ContentKeyPacketSignature,
|
||||
createTime: f.Link.CreateTime,
|
||||
modifyTime: f.Link.ModifyTime,
|
||||
revisions: []*revision{rev},
|
||||
})
|
||||
}
|
||||
|
||||
// StoredFolder is a folder node as the server stores it. Its Link has
|
||||
// no file members.
|
||||
type StoredFolder struct {
|
||||
Link StoredLink
|
||||
HashKey string // armored, encrypted to the node key
|
||||
XAttr string // armored; empty if none
|
||||
}
|
||||
|
||||
// SeedStoredFolder creates a folder in parentID from f, stored as given
|
||||
// without any check, and returns its ID. It panics as insertLink does.
|
||||
func (u *User) SeedStoredFolder(parentID string, f StoredFolder) (id string) {
|
||||
return u.insertLink(parentID, &link{
|
||||
typ: linkTypeFolder,
|
||||
name: f.Link.Name,
|
||||
nameHash: f.Link.NameHash,
|
||||
nodeKey: f.Link.NodeKey,
|
||||
nodePass: sealedPassphrase{armored: f.Link.NodePassphrase, signature: f.Link.NodePassphraseSignature},
|
||||
signatureEmail: f.Link.SignatureEmail,
|
||||
nameSignatureEmail: f.Link.NameSignatureEmail,
|
||||
createTime: f.Link.CreateTime,
|
||||
modifyTime: f.Link.ModifyTime,
|
||||
hashKey: f.HashKey,
|
||||
xattr: f.XAttr,
|
||||
})
|
||||
}
|
||||
|
||||
// FixedKeys is key material for AddUserWithKeys, in place of keys the
|
||||
// fake would generate.
|
||||
type FixedKeys struct {
|
||||
AddressKey string // armored, locked with AddressKeyPassphrase
|
||||
AddressKeyPassphrase string
|
||||
RootKey string // armored root folder node key, locked with RootKeyPassphrase
|
||||
RootKeyPassphrase string
|
||||
RootHashKey string // armored root folder hash key, encrypted to the root key
|
||||
}
|
||||
|
||||
// AddUserWithKeys is AddUser with the primary address key and the root
|
||||
// folder's node key and hash key taken from keys, so that files stored
|
||||
// by a client holding those keys can be seeded with SeedStoredFile. The
|
||||
// user key and share key are generated as usual. It panics if a key
|
||||
// doesn't unlock.
|
||||
func (s *Server) AddUserWithKeys(username, password string, keys FixedKeys) *User {
|
||||
addrKey, err := crypto.NewPrivateKeyFromArmored(keys.AddressKey, []byte(keys.AddressKeyPassphrase))
|
||||
mustCrypto("unlock fixed address key", err)
|
||||
rootKey, err := crypto.NewPrivateKeyFromArmored(keys.RootKey, []byte(keys.RootKeyPassphrase))
|
||||
mustCrypto("unlock fixed root key", err)
|
||||
u := newUser(s, username, password, password)
|
||||
addr := u.addresses[0]
|
||||
k := addr.keys[0]
|
||||
k.passphrase, k.armored, k.key = []byte(keys.AddressKeyPassphrase), keys.AddressKey, addrKey
|
||||
k.sealPassphrase(u.keys[0])
|
||||
root := lockedKey{key: rootKey, passphrase: []byte(keys.RootKeyPassphrase), armored: keys.RootKey}
|
||||
u.drive = s.newDriveWithRoot(addr, root, keys.RootHashKey)
|
||||
s.addUser(u)
|
||||
return u
|
||||
}
|
||||
@@ -0,0 +1,696 @@
|
||||
package protontest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"slices"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/ProtonMail/go-crypto/openpgp/packet"
|
||||
"github.com/ProtonMail/gopenpgp/v3/crypto"
|
||||
"github.com/rclone/rclone/backend/protondrive/api"
|
||||
)
|
||||
|
||||
const (
|
||||
// storageUploadPattern is the route of block uploads.
|
||||
storageUploadPattern = "/storage/upload/{token}"
|
||||
|
||||
// verificationCodeSize is the size of a draft's verification code,
|
||||
// and so of each block's verifier token.
|
||||
verificationCodeSize = 32
|
||||
|
||||
// maxUploadBlockSize is the largest encrypted block accepted.
|
||||
maxUploadBlockSize = 8 << 20
|
||||
|
||||
// intendedSizePrecision is the precision of IntendedUploadSize.
|
||||
intendedSizePrecision = 100_000
|
||||
)
|
||||
|
||||
// pendingBlock is a block of a draft revision requested for upload.
|
||||
type pendingBlock struct {
|
||||
index int
|
||||
encSignature string
|
||||
token []byte // verifier token
|
||||
hash []byte // SHA-256 the client declared; nil if none
|
||||
data []byte // nil until uploaded
|
||||
}
|
||||
|
||||
// uploadURL is a block upload URL issued by /drive/blocks.
|
||||
type uploadURL struct {
|
||||
token string
|
||||
user *User
|
||||
linkID string
|
||||
rev *revision
|
||||
block *pendingBlock
|
||||
expired bool
|
||||
}
|
||||
|
||||
// Wire types of the upload routes, named after and shaped like the
|
||||
// schemas in the SDK's driveTypes.ts.
|
||||
|
||||
type createFileRequest struct {
|
||||
ParentLinkID string
|
||||
Name string
|
||||
Hash string
|
||||
MIMEType string
|
||||
ClientUID *string
|
||||
IntendedUploadSize *int64
|
||||
NodeKey string
|
||||
NodePassphrase string
|
||||
NodePassphraseSignature string
|
||||
ContentKeyPacket string
|
||||
ContentKeyPacketSignature *string
|
||||
SignatureAddress *string
|
||||
}
|
||||
|
||||
type fileResponseJSON struct {
|
||||
ID string
|
||||
RevisionID string
|
||||
ClientUID *string
|
||||
}
|
||||
|
||||
type createFileResponse struct {
|
||||
File fileResponseJSON
|
||||
Code int
|
||||
}
|
||||
|
||||
type createRevisionRequest struct {
|
||||
CurrentRevisionID *string
|
||||
ClientUID *string
|
||||
IntendedUploadSize *int64
|
||||
}
|
||||
|
||||
type revisionIDJSON struct {
|
||||
ID string
|
||||
}
|
||||
|
||||
type createRevisionResponse struct {
|
||||
Revision revisionIDJSON
|
||||
Code int
|
||||
}
|
||||
|
||||
type verificationResponse struct {
|
||||
VerificationCode string
|
||||
ContentKeyPacket string
|
||||
Code int
|
||||
}
|
||||
|
||||
type verifierJSON struct {
|
||||
Token string
|
||||
}
|
||||
|
||||
type blockUploadInput struct {
|
||||
Index int
|
||||
Verifier *verifierJSON
|
||||
EncSignature *string
|
||||
Size *int64
|
||||
Hash *string
|
||||
}
|
||||
|
||||
type blockUploadRequest struct {
|
||||
LinkID string
|
||||
RevisionID string
|
||||
AddressID *string
|
||||
VolumeID *string
|
||||
BlockList []blockUploadInput
|
||||
ThumbnailList []struct{ Type int }
|
||||
}
|
||||
|
||||
type blockURLJSON struct {
|
||||
BareURL string
|
||||
Token string
|
||||
URL string
|
||||
Index int
|
||||
}
|
||||
|
||||
type blockUploadResponse struct {
|
||||
UploadLinks []blockURLJSON
|
||||
ThumbnailLinks []struct{}
|
||||
Code int
|
||||
}
|
||||
|
||||
type commitRevisionRequest struct {
|
||||
ManifestSignature string
|
||||
SignatureAddress *string
|
||||
XAttr *string
|
||||
ChecksumVerified bool
|
||||
Photo *struct{}
|
||||
}
|
||||
|
||||
// registerUpload registers the routes which upload files.
|
||||
func (s *Server) registerUpload() {
|
||||
s.handleDrive("POST", "/drive/v2/volumes/{volumeID}/files", s.handleCreateFile)
|
||||
s.handleDrive("POST", "/drive/v2/volumes/{volumeID}/files/{linkID}/revisions", s.handleCreateRevision)
|
||||
s.handleDrive("GET", "/drive/v2/volumes/{volumeID}/links/{linkID}/revisions/{revisionID}/verification", s.handleVerification)
|
||||
s.handleAuth("POST", "/drive/blocks", s.handleRequestUpload)
|
||||
s.handle("POST", storageUploadPattern, s.handleStorageUpload)
|
||||
s.handleDrive("PUT", "/drive/v2/volumes/{volumeID}/files/{linkID}/revisions/{revisionID}", s.handleCommit)
|
||||
s.handleDrive("DELETE", "/drive/v2/volumes/{volumeID}/files/{linkID}/revisions/{revisionID}", s.handleDeleteRevision)
|
||||
s.handleDrive("POST", "/drive/v2/volumes/{volumeID}/delete_multiple", s.handleDeleteMultiple)
|
||||
}
|
||||
|
||||
// ExpireUploadTokens makes every block upload URL issued so far answer
|
||||
// HTTP 404, as an expired one does.
|
||||
func (s *Server) ExpireUploadTokens() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, up := range s.uploads {
|
||||
up.expired = true
|
||||
}
|
||||
}
|
||||
|
||||
// StorageUploadPeak returns the most block uploads that were in
|
||||
// progress at once since the server started or ResetRequests.
|
||||
func (s *Server) StorageUploadPeak() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.storagePeak
|
||||
}
|
||||
|
||||
// nowLocked returns the server's clock. srv.mu must be held.
|
||||
func (s *Server) nowLocked() time.Time {
|
||||
return time.Now().Add(s.clockSkew)
|
||||
}
|
||||
|
||||
// newDraftRevision returns a draft revision created by clientUID.
|
||||
func newDraftRevision(clientUID *string) *revision {
|
||||
r := &revision{
|
||||
id: randomID(),
|
||||
state: revisionStateDraft,
|
||||
createTime: time.Now().Unix(),
|
||||
verificationCode: randomBytes(verificationCodeSize),
|
||||
pending: make(map[int]*pendingBlock),
|
||||
}
|
||||
if clientUID != nil {
|
||||
r.clientUID = *clientUID
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// checkIntendedSize returns an error if size, when given, is not
|
||||
// rounded as the SDK rounds it: the server must not learn the size.
|
||||
func checkIntendedSize(size *int64) error {
|
||||
if size == nil {
|
||||
return nil
|
||||
}
|
||||
if *size < 0 || (*size%intendedSizePrecision != 0 && *size != 4095) {
|
||||
return fmt.Errorf("IntendedUploadSize %d is not rounded", *size)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkContentKeyPacket returns an error unless packet64 is the base64
|
||||
// of one v6 PKESK: new content is written in the RFC 9580 format.
|
||||
func checkContentKeyPacket(packet64 string) error {
|
||||
raw, err := base64.StdEncoding.DecodeString(packet64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("not base64: %w", err)
|
||||
}
|
||||
r := packet.NewOpaqueReader(bytes.NewReader(raw))
|
||||
op, err := r.Next()
|
||||
if err != nil {
|
||||
return fmt.Errorf("malformed packet: %w", err)
|
||||
}
|
||||
if op.Tag != tagPKESK || len(op.Contents) == 0 || op.Contents[0] != 6 {
|
||||
return fmt.Errorf("packet with tag %d version %d, want a v6 PKESK", op.Tag, firstByte(op.Contents))
|
||||
}
|
||||
if _, err := r.Next(); err != io.EOF {
|
||||
return errors.New("more than one packet")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// contentKeyVersion returns the version of the PKESK of a stored
|
||||
// content key packet, or -1.
|
||||
func contentKeyVersion(packet64 string) int {
|
||||
raw, err := base64.StdEncoding.DecodeString(packet64)
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
op, err := packet.NewOpaqueReader(bytes.NewReader(raw)).Next()
|
||||
if err != nil || op.Tag != tagPKESK {
|
||||
return -1
|
||||
}
|
||||
return int(firstByte(op.Contents))
|
||||
}
|
||||
|
||||
// checkBlockFormat returns an error unless data is one SEIPD packet of
|
||||
// version: SEIPDv2 for content whose key packet is a v6 PKESK, SEIPDv1
|
||||
// for a v3 one.
|
||||
func checkBlockFormat(data []byte, version int) error {
|
||||
r := packet.NewOpaqueReader(bytes.NewReader(data))
|
||||
op, err := r.Next()
|
||||
if err != nil {
|
||||
return fmt.Errorf("malformed packet: %w", err)
|
||||
}
|
||||
if op.Tag != tagSEIPD || int(firstByte(op.Contents)) != version {
|
||||
return fmt.Errorf("packet with tag %d version %d, want SEIPDv%d", op.Tag, firstByte(op.Contents), version)
|
||||
}
|
||||
if _, err := r.Next(); err != io.EOF {
|
||||
return errors.New("more than one packet")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// firstByte returns b[0], or 0 if b is empty.
|
||||
func firstByte(b []byte) byte {
|
||||
if len(b) == 0 {
|
||||
return 0
|
||||
}
|
||||
return b[0]
|
||||
}
|
||||
|
||||
// draftRevision returns the draft revision id of file l, or nil.
|
||||
func draftRevision(l *link, id string) *revision {
|
||||
if l == nil || l.typ != linkTypeFile {
|
||||
return nil
|
||||
}
|
||||
for _, r := range l.revisions {
|
||||
if r.id == id && r.state == revisionStateDraft {
|
||||
return r
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleCreateFile creates a draft file, with a draft revision, from
|
||||
// key material the client generated.
|
||||
func (s *Server) handleCreateFile(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
|
||||
var req createFileRequest
|
||||
if !decodeRequest(w, r, &req) {
|
||||
return
|
||||
}
|
||||
parent := d.activeFolder(req.ParentLinkID)
|
||||
if parent == nil {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "Parent folder not found")
|
||||
return
|
||||
}
|
||||
err := errors.Join(
|
||||
checkHash(req.Hash),
|
||||
checkSigner(u, "SignatureAddress", req.SignatureAddress),
|
||||
checkMessages(map[string]string{"Name": req.Name, "NodePassphrase": req.NodePassphrase}),
|
||||
annotateField("ContentKeyPacket", checkContentKeyPacket(req.ContentKeyPacket)),
|
||||
checkIntendedSize(req.IntendedUploadSize),
|
||||
)
|
||||
switch {
|
||||
case err != nil:
|
||||
case req.NodePassphraseSignature == "":
|
||||
err = errors.New("NodePassphraseSignature is required")
|
||||
case req.ContentKeyPacketSignature == nil || *req.ContentKeyPacketSignature == "":
|
||||
err = errors.New("ContentKeyPacketSignature is required")
|
||||
case req.MIMEType == "":
|
||||
err = errors.New("MIMEType is required")
|
||||
default:
|
||||
err = annotateField("NodeKey", checkFileKey(req.NodeKey))
|
||||
}
|
||||
if err != nil {
|
||||
writeInvalid(w, err)
|
||||
return
|
||||
}
|
||||
if other := d.conflict(parent.id, req.Hash, ""); other != nil {
|
||||
writeConflict(w, other)
|
||||
return
|
||||
}
|
||||
now := time.Now().Unix()
|
||||
rev := newDraftRevision(req.ClientUID)
|
||||
l := &link{
|
||||
id: randomID(),
|
||||
parentID: parent.id,
|
||||
typ: linkTypeFile,
|
||||
state: linkStateDraft,
|
||||
createTime: now,
|
||||
modifyTime: now,
|
||||
name: req.Name,
|
||||
nameHash: req.Hash,
|
||||
nodeKey: req.NodeKey,
|
||||
nodePass: sealedPassphrase{armored: req.NodePassphrase, signature: req.NodePassphraseSignature},
|
||||
signatureEmail: *req.SignatureAddress,
|
||||
nameSignatureEmail: *req.SignatureAddress,
|
||||
mediaType: req.MIMEType,
|
||||
contentKeyPacket: req.ContentKeyPacket,
|
||||
contentKeyPacketSignature: *req.ContentKeyPacketSignature,
|
||||
revisions: []*revision{rev},
|
||||
}
|
||||
d.addLink(l)
|
||||
writeJSON(w, http.StatusOK, &createFileResponse{
|
||||
File: fileResponseJSON{ID: l.id, RevisionID: rev.id, ClientUID: nullable(rev.clientUID)},
|
||||
Code: api.CodeOK,
|
||||
})
|
||||
}
|
||||
|
||||
// handleCreateRevision creates a draft revision of an active file whose
|
||||
// active revision is CurrentRevisionID.
|
||||
func (s *Server) handleCreateRevision(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
|
||||
var req createRevisionRequest
|
||||
if !decodeRequest(w, r, &req) {
|
||||
return
|
||||
}
|
||||
l := d.activeLink(r.PathValue("linkID"))
|
||||
if l == nil || l.typ != linkTypeFile {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "File not found")
|
||||
return
|
||||
}
|
||||
if err := checkIntendedSize(req.IntendedUploadSize); err != nil {
|
||||
writeInvalid(w, err)
|
||||
return
|
||||
}
|
||||
active := l.activeRevision()
|
||||
if active == nil || req.CurrentRevisionID == nil || *req.CurrentRevisionID != active.id {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidRequirements, "protontest: CurrentRevisionID is not the active revision: the file changed")
|
||||
return
|
||||
}
|
||||
rev := newDraftRevision(req.ClientUID)
|
||||
l.revisions = append(l.revisions, rev)
|
||||
writeJSON(w, http.StatusOK, &createRevisionResponse{Revision: revisionIDJSON{ID: rev.id}, Code: api.CodeOK})
|
||||
}
|
||||
|
||||
// handleVerification returns the verification code and content key
|
||||
// packet of a draft revision.
|
||||
func (s *Server) handleVerification(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
|
||||
l := d.links[r.PathValue("linkID")]
|
||||
rev := draftRevision(l, r.PathValue("revisionID"))
|
||||
if rev == nil {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "Revision not found")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, &verificationResponse{
|
||||
VerificationCode: base64.StdEncoding.EncodeToString(rev.verificationCode),
|
||||
ContentKeyPacket: l.contentKeyPacket,
|
||||
Code: api.CodeOK,
|
||||
})
|
||||
}
|
||||
|
||||
// handleRequestUpload issues upload URLs on this server for blocks of a
|
||||
// draft revision. Requesting a block again replaces it, and its URL.
|
||||
func (s *Server) handleRequestUpload(w http.ResponseWriter, r *http.Request, u *User) {
|
||||
var req blockUploadRequest
|
||||
if !decodeRequest(w, r, &req) {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
d := u.drive
|
||||
if d == nil || req.VolumeID == nil || *req.VolumeID != d.volumeID {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "Volume not found")
|
||||
return
|
||||
}
|
||||
l := d.links[req.LinkID]
|
||||
rev := draftRevision(l, req.RevisionID)
|
||||
if rev == nil {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "Revision not found")
|
||||
return
|
||||
}
|
||||
if req.AddressID == nil || !slices.ContainsFunc(u.addresses, func(a *address) bool { return a.id == *req.AddressID }) {
|
||||
writeInvalid(w, errors.New("AddressID must be an address of the user"))
|
||||
return
|
||||
}
|
||||
if len(req.ThumbnailList) > 0 {
|
||||
writeInvalid(w, errors.New("protontest: thumbnails are not supported"))
|
||||
return
|
||||
}
|
||||
blocks := make([]*pendingBlock, 0, len(req.BlockList))
|
||||
for _, in := range req.BlockList {
|
||||
b, err := pendingBlockOf(in)
|
||||
if err != nil {
|
||||
writeInvalid(w, fmt.Errorf("block %d: %w", in.Index, err))
|
||||
return
|
||||
}
|
||||
blocks = append(blocks, b)
|
||||
}
|
||||
resp := &blockUploadResponse{UploadLinks: []blockURLJSON{}, ThumbnailLinks: []struct{}{}, Code: api.CodeOK}
|
||||
for _, b := range blocks {
|
||||
rev.pending[b.index] = b
|
||||
id, token := randomID(), s.newSecretLocked(StorageToken)
|
||||
s.uploads[id] = &uploadURL{token: token, user: u, linkID: l.id, rev: rev, block: b}
|
||||
bareURL := s.URL() + "/storage/upload/" + id
|
||||
resp.UploadLinks = append(resp.UploadLinks, blockURLJSON{BareURL: bareURL, Token: token, URL: bareURL, Index: b.index})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
// pendingBlockOf returns the block in an upload request, checked.
|
||||
func pendingBlockOf(in blockUploadInput) (*pendingBlock, error) {
|
||||
b := &pendingBlock{index: in.Index}
|
||||
if in.Index < 1 {
|
||||
return nil, errors.New("field Index must be at least 1")
|
||||
}
|
||||
if in.Verifier == nil {
|
||||
return nil, errors.New("field Verifier is required")
|
||||
}
|
||||
token, err := base64.StdEncoding.DecodeString(in.Verifier.Token)
|
||||
if err != nil || len(token) != verificationCodeSize {
|
||||
return nil, fmt.Errorf("Verifier.Token is not the base64 of %d bytes", verificationCodeSize)
|
||||
}
|
||||
b.token = token
|
||||
if in.EncSignature == nil || *in.EncSignature == "" {
|
||||
return nil, errors.New("EncSignature is required")
|
||||
}
|
||||
b.encSignature = *in.EncSignature
|
||||
if in.Hash != nil {
|
||||
if b.hash, err = base64.StdEncoding.DecodeString(*in.Hash); err != nil || len(b.hash) != sha256.Size {
|
||||
return nil, errors.New("field Hash is not the base64 of a SHA-256")
|
||||
}
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// handleStorageUpload stores the block of a multipart upload to a URL
|
||||
// issued by handleRequestUpload, which carries its pm-storage-token and
|
||||
// none of the session's credentials.
|
||||
func (s *Server) handleStorageUpload(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "" || r.Header.Get("x-pm-uid") != "" {
|
||||
writeError(w, http.StatusBadRequest, api.CodeInvalidValue, "protontest: session credentials must not be sent to the storage host")
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
up := s.uploads[r.PathValue("token")]
|
||||
usable := up != nil && !up.expired
|
||||
s.mu.Unlock()
|
||||
switch {
|
||||
case !usable:
|
||||
writeError(w, http.StatusNotFound, api.CodeNotExists, "protontest: no such upload URL, or it expired")
|
||||
return
|
||||
case r.Header.Get("pm-storage-token") != up.token:
|
||||
writeError(w, http.StatusForbidden, api.CodeNotEnoughPermissions, "protontest: missing or wrong pm-storage-token")
|
||||
return
|
||||
}
|
||||
data, err := readBlockPart(r)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, api.CodeInvalidValue, "protontest: bad block upload: "+err.Error())
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if up.rev.state != revisionStateDraft || up.rev.pending[up.block.index] != up.block {
|
||||
writeError(w, http.StatusNotFound, api.CodeNotExists, "protontest: the block is no longer awaited")
|
||||
return
|
||||
}
|
||||
up.block.data = data
|
||||
writeJSON(w, http.StatusOK, &codeJSON{Code: api.CodeOK})
|
||||
}
|
||||
|
||||
// readBlockPart returns the content of the one part, named Block, of
|
||||
// the multipart body of r.
|
||||
func readBlockPart(r *http.Request) ([]byte, error) {
|
||||
mr, err := r.MultipartReader()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var data []byte
|
||||
for {
|
||||
part, err := mr.NextPart()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if part.FormName() != "Block" || data != nil {
|
||||
return nil, fmt.Errorf("unexpected part %q", part.FormName())
|
||||
}
|
||||
if data, err = io.ReadAll(io.LimitReader(part, maxUploadBlockSize+1)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(data) > maxUploadBlockSize {
|
||||
return nil, fmt.Errorf("block larger than %d bytes", maxUploadBlockSize)
|
||||
}
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return nil, errors.New("no Block part, or it is empty")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// handleCommit makes a draft revision with every requested block
|
||||
// uploaded the file's active revision, after checking each block's
|
||||
// verifier token and format, the manifest signature over the uploaded
|
||||
// blocks and the extended attributes' format.
|
||||
func (s *Server) handleCommit(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
|
||||
var req commitRevisionRequest
|
||||
if !decodeRequest(w, r, &req) {
|
||||
return
|
||||
}
|
||||
l := d.links[r.PathValue("linkID")]
|
||||
rev := draftRevision(l, r.PathValue("revisionID"))
|
||||
if rev == nil {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "Revision not found")
|
||||
return
|
||||
}
|
||||
xattr := ""
|
||||
if req.XAttr != nil {
|
||||
xattr = *req.XAttr
|
||||
}
|
||||
err := errors.Join(
|
||||
checkSigner(u, "SignatureAddress", req.SignatureAddress),
|
||||
checkMessages(map[string]string{"XAttr": xattr}),
|
||||
)
|
||||
if err == nil && req.ManifestSignature == "" {
|
||||
err = errors.New("ManifestSignature is required")
|
||||
}
|
||||
if err != nil {
|
||||
writeInvalid(w, err)
|
||||
return
|
||||
}
|
||||
blocks, err := uploadedBlocks(rev, *req.SignatureAddress)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidRequirements, "protontest: "+err.Error())
|
||||
return
|
||||
}
|
||||
var manifest []byte
|
||||
var formatErrs []error
|
||||
version := 1
|
||||
if contentKeyVersion(l.contentKeyPacket) == 6 {
|
||||
version = 2
|
||||
}
|
||||
for _, b := range blocks {
|
||||
manifest = append(manifest, b.hash...)
|
||||
formatErrs = append(formatErrs, annotateField(fmt.Sprintf("block %d", b.index), checkBlockFormat(b.data, version)))
|
||||
}
|
||||
addr := u.addressByEmailLocked(*req.SignatureAddress)
|
||||
keys := make([]*crypto.Key, 0, len(addr.keys))
|
||||
for _, k := range addr.keys {
|
||||
keys = append(keys, k.key)
|
||||
}
|
||||
sigErr := verifyDetached(manifest, []byte(req.ManifestSignature), crypto.Armor, keyRing(keys...), s.nowLocked())
|
||||
if err := errors.Join(append(formatErrs, annotateField("manifest", sigErr))...); err != nil {
|
||||
writeInvalid(w, err)
|
||||
return
|
||||
}
|
||||
before := d.linkDetails(l)
|
||||
for _, other := range l.revisions {
|
||||
if other.state == revisionStateActive {
|
||||
other.state = revisionStateObsolete
|
||||
}
|
||||
}
|
||||
now := time.Now().Unix()
|
||||
rev.state, rev.createTime = revisionStateActive, now
|
||||
rev.manifestSignature, rev.xattr, rev.signatureEmail = req.ManifestSignature, xattr, *req.SignatureAddress
|
||||
rev.blocks, rev.pending, rev.verificationCode = blocks, nil, nil
|
||||
l.state, l.modifyTime = linkStateActive, now
|
||||
s.dropUploadsLocked(rev)
|
||||
s.lagLocked(l.id, &before)
|
||||
writeJSON(w, http.StatusOK, &codeJSON{Code: api.CodeOK})
|
||||
}
|
||||
|
||||
// uploadedBlocks returns the blocks of draft rev, signed by email, after
|
||||
// checking that blocks 1 to n were requested and each uploaded, with the
|
||||
// hash it was declared with, if any, and its verifier token.
|
||||
func uploadedBlocks(rev *revision, email string) ([]*block, error) {
|
||||
indexes := make([]int, 0, len(rev.pending))
|
||||
for i := range rev.pending {
|
||||
indexes = append(indexes, i)
|
||||
}
|
||||
sort.Ints(indexes)
|
||||
blocks := make([]*block, 0, len(indexes))
|
||||
for i, index := range indexes {
|
||||
p := rev.pending[index]
|
||||
switch {
|
||||
case index != i+1:
|
||||
return nil, fmt.Errorf("block %d was not requested", i+1)
|
||||
case p.data == nil:
|
||||
return nil, fmt.Errorf("block %d was not uploaded", index)
|
||||
}
|
||||
sum := sha256.Sum256(p.data)
|
||||
if p.hash != nil && !bytes.Equal(p.hash, sum[:]) {
|
||||
return nil, fmt.Errorf("block %d: declared hash doesn't match the uploaded block", index)
|
||||
}
|
||||
if want := xorPrefix(rev.verificationCode, p.data); !bytes.Equal(p.token, want) {
|
||||
return nil, fmt.Errorf("block %d: verifier token doesn't match the uploaded block", index)
|
||||
}
|
||||
blocks = append(blocks, &block{index: index, hash: sum[:], data: p.data, encSignature: p.encSignature, signatureEmail: email})
|
||||
}
|
||||
return blocks, nil
|
||||
}
|
||||
|
||||
// xorPrefix returns code XOR the start of data, padded with zeros: the
|
||||
// verifier token of a block.
|
||||
func xorPrefix(code, data []byte) []byte {
|
||||
out := make([]byte, len(code))
|
||||
for i := range code {
|
||||
out[i] = code[i]
|
||||
if i < len(data) {
|
||||
out[i] ^= data[i]
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// dropUploadsLocked forgets the upload URLs of rev. srv.mu must be
|
||||
// held.
|
||||
func (s *Server) dropUploadsLocked(rev *revision) {
|
||||
for id, up := range s.uploads {
|
||||
if up.rev == rev {
|
||||
delete(s.uploads, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// handleDeleteRevision deletes a draft revision.
|
||||
func (s *Server) handleDeleteRevision(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
|
||||
l := d.links[r.PathValue("linkID")]
|
||||
rev := draftRevision(l, r.PathValue("revisionID"))
|
||||
if rev == nil {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "Draft revision not found")
|
||||
return
|
||||
}
|
||||
l.revisions = slices.DeleteFunc(l.revisions, func(other *revision) bool { return other == rev })
|
||||
s.dropUploadsLocked(rev)
|
||||
writeJSON(w, http.StatusOK, &codeJSON{Code: api.CodeOK})
|
||||
}
|
||||
|
||||
// handleDeleteMultiple permanently deletes up to 100 draft links,
|
||||
// answering for each.
|
||||
func (s *Server) handleDeleteMultiple(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
|
||||
var req linksRequest
|
||||
if !decodeRequest(w, r, &req) {
|
||||
return
|
||||
}
|
||||
if len(req.LinkIDs) > maxLinksPerRequest {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidValue, fmt.Sprintf("protontest: %d link IDs, at most %d allowed", len(req.LinkIDs), maxLinksPerRequest))
|
||||
return
|
||||
}
|
||||
resp := &multiResponse{Code: api.CodeOKMany, Responses: []linkResultJSON{}}
|
||||
for _, id := range req.LinkIDs {
|
||||
result := linkResultJSON{LinkID: id, Response: codeJSON{Code: api.CodeOK}}
|
||||
l := d.links[id]
|
||||
switch {
|
||||
case l == nil:
|
||||
result.Response = codeJSON{Code: api.CodeNotExists, Error: "File or folder not found"}
|
||||
case l.state != linkStateDraft:
|
||||
result.Response = codeJSON{Code: api.CodeInvalidRequirements, Error: "protontest: only drafts can be deleted"}
|
||||
default:
|
||||
for _, rev := range l.revisions {
|
||||
s.dropUploadsLocked(rev)
|
||||
}
|
||||
delete(d.links, id)
|
||||
}
|
||||
resp.Responses = append(resp.Responses, result)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
@@ -0,0 +1,502 @@
|
||||
package protontest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ProtonMail/go-crypto/openpgp/packet"
|
||||
"github.com/ProtonMail/go-crypto/openpgp/s2k"
|
||||
"github.com/ProtonMail/gopenpgp/v3/crypto"
|
||||
"github.com/ProtonMail/gopenpgp/v3/profile"
|
||||
"github.com/rclone/rclone/backend/protondrive/api"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const testClientUID = "fake-test-client"
|
||||
|
||||
// uploader uploads files to the root of a user's drive as the SDK
|
||||
// does, step by step, for tests to spoil one step.
|
||||
type uploader struct {
|
||||
t *testing.T
|
||||
s *Server
|
||||
u *User
|
||||
sess Session
|
||||
volume string
|
||||
}
|
||||
|
||||
func newUploader(t *testing.T) *uploader {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
return &uploader{t: t, s: s, u: u, sess: s.NewSession(u), volume: u.VolumeID()}
|
||||
}
|
||||
|
||||
// fileDraft is a file being uploaded, with its keys.
|
||||
type fileDraft struct {
|
||||
req *createFileRequest
|
||||
node lockedKey
|
||||
ck contentKey
|
||||
signer *crypto.Key
|
||||
linkID string
|
||||
revID string
|
||||
code []byte
|
||||
content []byte
|
||||
}
|
||||
|
||||
// newFileRequest returns a create file request for name in the root
|
||||
// made as the SDK's createDraft does.
|
||||
func (x *uploader) newFileRequest(name string) *fileDraft {
|
||||
v := x.u.view()
|
||||
parentKey, parentHashKey := v.folderKeys(v.d.rootID)
|
||||
node := newDriveKey(true)
|
||||
x.s.rememberKey(node)
|
||||
pass := sealPassphrase(node.passphrase, parentKey, v.addrKey)
|
||||
ck := newContentKey(node.key, true)
|
||||
uid := testClientUID
|
||||
return &fileDraft{
|
||||
req: &createFileRequest{
|
||||
ParentLinkID: v.d.rootID,
|
||||
Name: encryptArmored([]byte(name), parentKey, v.addrKey, false),
|
||||
Hash: lookupHash(parentHashKey, name),
|
||||
MIMEType: "text/plain",
|
||||
ClientUID: &uid,
|
||||
NodeKey: node.armored,
|
||||
NodePassphrase: pass.armored,
|
||||
NodePassphraseSignature: pass.signature,
|
||||
ContentKeyPacket: ck.packet,
|
||||
ContentKeyPacketSignature: &ck.signature,
|
||||
SignatureAddress: &v.email,
|
||||
},
|
||||
node: node,
|
||||
ck: ck,
|
||||
signer: v.addrKey,
|
||||
}
|
||||
}
|
||||
|
||||
// send sends in with method to path as the user.
|
||||
func (x *uploader) send(method, path string, in any) (int, []byte) {
|
||||
return sendJSON(x.t, x.s, x.sess, method, path, in)
|
||||
}
|
||||
|
||||
// create creates the draft of f, failing the test if it can't.
|
||||
func (x *uploader) create(f *fileDraft) {
|
||||
t := x.t
|
||||
status, body := x.send("POST", "/drive/v2/volumes/"+x.volume+"/files", f.req)
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
var resp createFileResponse
|
||||
require.NoError(t, json.Unmarshal(body, &resp))
|
||||
f.linkID, f.revID = resp.File.ID, resp.File.RevisionID
|
||||
x.loadCode(f)
|
||||
}
|
||||
|
||||
// loadCode fetches the verification code of f's draft revision.
|
||||
func (x *uploader) loadCode(f *fileDraft) {
|
||||
var ver verificationResponse
|
||||
status, body := getJSON(x.t, x.s, x.sess, "/drive/v2/volumes/"+x.volume+"/links/"+f.linkID+"/revisions/"+f.revID+"/verification", &ver)
|
||||
require.Equal(x.t, http.StatusOK, status, "body: %s", body)
|
||||
code, err := base64.StdEncoding.DecodeString(ver.VerificationCode)
|
||||
require.NoError(x.t, err)
|
||||
require.Len(x.t, code, verificationCodeSize)
|
||||
assert.Equal(x.t, f.ck.packet, ver.ContentKeyPacket)
|
||||
f.code = code
|
||||
}
|
||||
|
||||
// testBlock is an encrypted block with its upload request entry.
|
||||
type testBlock struct {
|
||||
data []byte
|
||||
in blockUploadInput
|
||||
}
|
||||
|
||||
// encrypt encrypts content as f's blocks of blockSize.
|
||||
func (f *fileDraft) encrypt(content []byte) []testBlock {
|
||||
f.content = content
|
||||
var blocks []testBlock
|
||||
for i := 0; i < len(content); i += blockSize {
|
||||
data, encSig := f.ck.encryptBlock(content[i:min(i+blockSize, len(content))], f.node.key, f.signer)
|
||||
blocks = append(blocks, testBlock{data: data, in: blockUploadInput{
|
||||
Index: len(blocks) + 1,
|
||||
Verifier: &verifierJSON{Token: base64.StdEncoding.EncodeToString(xorPrefix(f.code, data))},
|
||||
EncSignature: &encSig,
|
||||
}})
|
||||
}
|
||||
return blocks
|
||||
}
|
||||
|
||||
// requestURLs requests upload URLs for blocks of f.
|
||||
func (x *uploader) requestURLs(f *fileDraft, blocks []testBlock) (int, []byte, []blockURLJSON) {
|
||||
addressID := x.u.addresses[0].id
|
||||
req := &blockUploadRequest{LinkID: f.linkID, RevisionID: f.revID, AddressID: &addressID, VolumeID: &x.volume}
|
||||
for _, b := range blocks {
|
||||
req.BlockList = append(req.BlockList, b.in)
|
||||
}
|
||||
status, body := x.send("POST", "/drive/blocks", req)
|
||||
var resp blockUploadResponse
|
||||
if status == http.StatusOK {
|
||||
require.NoError(x.t, json.Unmarshal(body, &resp))
|
||||
}
|
||||
return status, body, resp.UploadLinks
|
||||
}
|
||||
|
||||
// putBlock uploads data to link as the SDK does, with extra headers,
|
||||
// and returns the HTTP status.
|
||||
func (x *uploader) putBlock(link blockURLJSON, data []byte, header map[string]string) int {
|
||||
t := x.t
|
||||
var body bytes.Buffer
|
||||
mw := multipart.NewWriter(&body)
|
||||
w, err := mw.CreateFormFile("Block", "blob")
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write(data)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, mw.Close())
|
||||
req, err := http.NewRequest("POST", link.BareURL, &body)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
req.Header.Set("x-pm-appversion", testAppVersion)
|
||||
req.Header.Set("pm-storage-token", link.Token)
|
||||
for k, v := range header {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
resp, err := x.s.Client().Do(req)
|
||||
require.NoError(t, err)
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
return resp.StatusCode
|
||||
}
|
||||
|
||||
// uploadAll requests URLs for blocks and uploads each.
|
||||
func (x *uploader) uploadAll(f *fileDraft, blocks []testBlock) {
|
||||
status, body, links := x.requestURLs(f, blocks)
|
||||
require.Equal(x.t, http.StatusOK, status, "body: %s", body)
|
||||
require.Len(x.t, links, len(blocks))
|
||||
for i, link := range links {
|
||||
require.Equal(x.t, blocks[i].in.Index, link.Index)
|
||||
require.Equal(x.t, http.StatusOK, x.putBlock(link, blocks[i].data, nil))
|
||||
}
|
||||
}
|
||||
|
||||
// commitRequest returns the commit request of f with blocks, as the SDK
|
||||
// makes it.
|
||||
func (f *fileDraft) commitRequest(blocks []testBlock) *commitRevisionRequest {
|
||||
var manifest []byte
|
||||
var sizes []int64
|
||||
for i, b := range blocks {
|
||||
sum := sha256.Sum256(b.data)
|
||||
manifest = append(manifest, sum[:]...)
|
||||
sizes = append(sizes, int64(min(blockSize, len(f.content)-i*blockSize)))
|
||||
}
|
||||
xattr := encryptArmored(newFileXAttr(f.content, sizes, "2024-01-02T03:04:05.000Z"), f.node.key, f.signer, true)
|
||||
return &commitRevisionRequest{
|
||||
ManifestSignature: signArmored(manifest, f.signer),
|
||||
SignatureAddress: f.req.SignatureAddress,
|
||||
XAttr: &xattr,
|
||||
}
|
||||
}
|
||||
|
||||
// commit commits f's draft revision with req.
|
||||
func (x *uploader) commit(f *fileDraft, req *commitRevisionRequest) (int, []byte) {
|
||||
return x.send("PUT", "/drive/v2/volumes/"+x.volume+"/files/"+f.linkID+"/revisions/"+f.revID, req)
|
||||
}
|
||||
|
||||
func TestCreateFileFormats(t *testing.T) {
|
||||
x := newUploader(t)
|
||||
post := func(f *fileDraft) (int, []byte) {
|
||||
return x.send("POST", "/drive/v2/volumes/"+x.volume+"/files", f.req)
|
||||
}
|
||||
for variant, bad := range aeadMessages(t) {
|
||||
for field, spoil := range map[string]func(req *createFileRequest){
|
||||
"Name": func(req *createFileRequest) { req.Name = bad },
|
||||
"NodePassphrase": func(req *createFileRequest) { req.NodePassphrase = bad },
|
||||
} {
|
||||
t.Run(field+" "+variant, func(t *testing.T) {
|
||||
f := x.newFileRequest("f")
|
||||
spoil(f.req)
|
||||
status, body := post(f)
|
||||
requireRejected(t, api.CodeInvalidValue, status, body, field)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
aeadLocked := profile.Default()
|
||||
aeadLocked.AeadKeyEncryption = &packet.AEADConfig{}
|
||||
salted := profile.Default()
|
||||
salted.S2kKeyEncryption = &s2k.Config{S2KMode: s2k.SaltedS2K, PassphraseIsHighEntropy: true}
|
||||
v6Key, err := crypto.PGPWithProfile(profile.RFC9580()).KeyGeneration().AddUserId(driveKeyName, "").New().GenerateKey()
|
||||
require.NoError(t, err)
|
||||
v6Locked, err := crypto.PGPWithProfile(profile.RFC9580()).LockKey(v6Key, newPassphrase())
|
||||
require.NoError(t, err)
|
||||
v6Armored, err := v6Locked.Armor()
|
||||
require.NoError(t, err)
|
||||
for name, tc := range map[string]struct {
|
||||
key string
|
||||
want string
|
||||
}{
|
||||
"v6": {v6Armored, "not a v4 key"},
|
||||
"AEAD locked": {lockedWith(t, aeadLocked), "locked with AEAD"},
|
||||
"salted S2K": {lockedWith(t, salted), "S2K mode 1"},
|
||||
} {
|
||||
t.Run("NodeKey "+name, func(t *testing.T) {
|
||||
f := x.newFileRequest("f")
|
||||
f.req.NodeKey = tc.key
|
||||
status, body := post(f)
|
||||
requireRejected(t, api.CodeInvalidValue, status, body, "NodeKey", tc.want)
|
||||
})
|
||||
}
|
||||
|
||||
legacy := newContentKey(newDriveKey(false).key, false)
|
||||
unrounded := int64(123_456)
|
||||
for name, tc := range map[string]struct {
|
||||
spoil func(req *createFileRequest)
|
||||
want string
|
||||
}{
|
||||
"v3 content key packet": {func(req *createFileRequest) { req.ContentKeyPacket = legacy.packet }, "v6 PKESK"},
|
||||
"no content key sig": {func(req *createFileRequest) { req.ContentKeyPacketSignature = nil }, "ContentKeyPacketSignature"},
|
||||
"no passphrase sig": {func(req *createFileRequest) { req.NodePassphraseSignature = "" }, "NodePassphraseSignature"},
|
||||
"unrounded size": {func(req *createFileRequest) { req.IntendedUploadSize = &unrounded }, "IntendedUploadSize"},
|
||||
"no signer": {func(req *createFileRequest) { req.SignatureAddress = nil }, "SignatureAddress"},
|
||||
"no media type": {func(req *createFileRequest) { req.MIMEType = "" }, "MIMEType"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f := x.newFileRequest("f")
|
||||
tc.spoil(f.req)
|
||||
status, body := post(f)
|
||||
requireRejected(t, api.CodeInvalidValue, status, body, tc.want)
|
||||
})
|
||||
}
|
||||
assert.Empty(t, x.u.Drafts())
|
||||
|
||||
// The name hash must be unique, and a conflict names the draft and
|
||||
// its creator.
|
||||
f := x.newFileRequest("f")
|
||||
rounded := int64(200_000)
|
||||
f.req.IntendedUploadSize = &rounded
|
||||
x.create(f)
|
||||
assert.Equal(t, []string{f.linkID}, x.u.Drafts())
|
||||
status, body := post(x.newFileRequest("f"))
|
||||
requireRejected(t, api.CodeAlreadyExists, status, body)
|
||||
var conflict conflictResponse
|
||||
require.NoError(t, json.Unmarshal(body, &conflict))
|
||||
assert.Equal(t, f.linkID, conflict.Details.ConflictLinkID)
|
||||
assert.Nil(t, conflict.Details.ConflictRevisionID)
|
||||
require.NotNil(t, conflict.Details.ConflictDraftRevisionID)
|
||||
assert.Equal(t, f.revID, *conflict.Details.ConflictDraftRevisionID)
|
||||
require.NotNil(t, conflict.Details.ConflictDraftClientUID)
|
||||
assert.Equal(t, testClientUID, *conflict.Details.ConflictDraftClientUID)
|
||||
assert.Empty(t, x.u.Dump(), "a draft is dumped")
|
||||
}
|
||||
|
||||
func TestCommitChecks(t *testing.T) {
|
||||
x := newUploader(t)
|
||||
content := patterned(blockSize + 1000)
|
||||
n := 0
|
||||
// prepare returns a new draft with content encrypted, its blocks
|
||||
// and their commit request.
|
||||
prepare := func() (*fileDraft, []testBlock, *commitRevisionRequest) {
|
||||
n++
|
||||
f := x.newFileRequest("f" + strconv.Itoa(n))
|
||||
x.create(f)
|
||||
blocks := f.encrypt(content)
|
||||
return f, blocks, f.commitRequest(blocks)
|
||||
}
|
||||
impostor := generateKey("impostor", "impostor@proton.test")
|
||||
empty := ""
|
||||
aead := aeadMessages(t)["v6 PKESK and SEIPDv2"]
|
||||
legacy := newContentKey(newDriveKey(false).key, false)
|
||||
otherHash := base64.StdEncoding.EncodeToString(make([]byte, sha256.Size))
|
||||
for name, tc := range map[string]struct {
|
||||
spoil func(f *fileDraft, blocks []testBlock, req *commitRevisionRequest) []testBlock
|
||||
code int
|
||||
want string
|
||||
}{
|
||||
"AEAD XAttr": {func(f *fileDraft, blocks []testBlock, req *commitRevisionRequest) []testBlock {
|
||||
req.XAttr = &aead
|
||||
return blocks
|
||||
}, api.CodeInvalidValue, "XAttr"},
|
||||
"empty XAttr": {func(f *fileDraft, blocks []testBlock, req *commitRevisionRequest) []testBlock {
|
||||
req.XAttr = &empty
|
||||
return blocks
|
||||
}, api.CodeInvalidValue, "XAttr"},
|
||||
"no XAttr": {func(f *fileDraft, blocks []testBlock, req *commitRevisionRequest) []testBlock {
|
||||
req.XAttr = nil
|
||||
return blocks
|
||||
}, api.CodeInvalidValue, "XAttr"},
|
||||
"verifier": {func(f *fileDraft, blocks []testBlock, req *commitRevisionRequest) []testBlock {
|
||||
token := xorPrefix(f.code, blocks[1].data)
|
||||
token[5] ^= 1
|
||||
blocks[1].in.Verifier.Token = base64.StdEncoding.EncodeToString(token)
|
||||
return blocks
|
||||
}, api.CodeInvalidRequirements, "block 2: verifier token"},
|
||||
"block not uploaded": {func(f *fileDraft, blocks []testBlock, req *commitRevisionRequest) []testBlock {
|
||||
_, _, links := x.requestURLs(f, blocks[1:])
|
||||
require.Len(t, links, 1)
|
||||
return blocks[:1]
|
||||
}, api.CodeInvalidRequirements, "block 2 was not uploaded"},
|
||||
"index gap": {func(f *fileDraft, blocks []testBlock, req *commitRevisionRequest) []testBlock {
|
||||
blocks[1].in.Index = 3
|
||||
return blocks
|
||||
}, api.CodeInvalidRequirements, "block 2 was not requested"},
|
||||
"declared hash": {func(f *fileDraft, blocks []testBlock, req *commitRevisionRequest) []testBlock {
|
||||
blocks[0].in.Hash = &otherHash
|
||||
return blocks
|
||||
}, api.CodeInvalidRequirements, "declared hash"},
|
||||
"manifest": {func(f *fileDraft, blocks []testBlock, req *commitRevisionRequest) []testBlock {
|
||||
req.ManifestSignature = signArmored([]byte("not the manifest"), f.signer)
|
||||
return blocks
|
||||
}, api.CodeInvalidValue, "manifest"},
|
||||
"manifest signer": {func(f *fileDraft, blocks []testBlock, req *commitRevisionRequest) []testBlock {
|
||||
var manifest []byte
|
||||
for _, b := range blocks {
|
||||
sum := sha256.Sum256(b.data)
|
||||
manifest = append(manifest, sum[:]...)
|
||||
}
|
||||
req.ManifestSignature = signArmored(manifest, impostor)
|
||||
return blocks
|
||||
}, api.CodeInvalidValue, "manifest"},
|
||||
"SEIPDv1 block": {func(f *fileDraft, blocks []testBlock, req *commitRevisionRequest) []testBlock {
|
||||
data, _ := legacy.encryptBlock(content[blockSize:], f.node.key, f.signer)
|
||||
blocks[1].data = data
|
||||
blocks[1].in.Verifier.Token = base64.StdEncoding.EncodeToString(xorPrefix(f.code, data))
|
||||
*req = *f.commitRequest(blocks)
|
||||
return blocks
|
||||
}, api.CodeInvalidValue, "block 2: packet with tag 18 version 1, want SEIPDv2"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f, blocks, req := prepare()
|
||||
blocks = tc.spoil(f, blocks, req)
|
||||
x.uploadAll(f, blocks)
|
||||
status, body := x.commit(f, req)
|
||||
requireRejected(t, tc.code, status, body, tc.want)
|
||||
assert.Contains(t, x.u.Drafts(), f.linkID)
|
||||
})
|
||||
}
|
||||
assert.Empty(t, x.u.Dump(), "a draft was committed")
|
||||
|
||||
f, blocks, req := prepare()
|
||||
x.uploadAll(f, blocks)
|
||||
status, body := x.commit(f, req)
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
dump := x.u.Dump()
|
||||
require.Len(t, dump, 1)
|
||||
assert.NoError(t, dump[0].Err)
|
||||
assert.Equal(t, f.linkID, dump[0].ID)
|
||||
assert.True(t, bytes.Equal(content, dump[0].Content))
|
||||
assert.NotContains(t, x.u.Drafts(), f.linkID)
|
||||
|
||||
// A committed revision can't be committed again or deleted as a
|
||||
// draft.
|
||||
status, body = x.commit(f, req)
|
||||
requireRejected(t, api.CodeNotExists, status, body)
|
||||
status, body = x.send("POST", "/drive/v2/volumes/"+x.volume+"/delete_multiple", &linksRequest{LinkIDs: []string{f.linkID}})
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
var multi multiResponse
|
||||
require.NoError(t, json.Unmarshal(body, &multi))
|
||||
require.Len(t, multi.Responses, 1)
|
||||
assert.Equal(t, api.CodeInvalidRequirements, multi.Responses[0].Response.Code)
|
||||
status, body = x.send("DELETE", "/drive/v2/volumes/"+x.volume+"/files/"+f.linkID+"/revisions/"+f.revID, nil)
|
||||
requireRejected(t, api.CodeNotExists, status, body)
|
||||
require.Len(t, x.u.Dump(), 1)
|
||||
}
|
||||
|
||||
func TestStorageUploadRules(t *testing.T) {
|
||||
x := newUploader(t)
|
||||
f := x.newFileRequest("f")
|
||||
x.create(f)
|
||||
blocks := f.encrypt([]byte("content"))
|
||||
status, body, links := x.requestURLs(f, blocks)
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
require.Len(t, links, 1)
|
||||
link := links[0]
|
||||
|
||||
assert.Equal(t, http.StatusBadRequest, x.putBlock(link, blocks[0].data, sessionHeader(x.sess)), "session credentials accepted")
|
||||
wrong := link
|
||||
wrong.Token = "wrong"
|
||||
assert.Equal(t, http.StatusForbidden, x.putBlock(wrong, blocks[0].data, nil))
|
||||
x.s.ExpireUploadTokens()
|
||||
assert.Equal(t, http.StatusNotFound, x.putBlock(link, blocks[0].data, nil))
|
||||
|
||||
// A block requested again gets a URL which works.
|
||||
_, _, links = x.requestURLs(f, blocks)
|
||||
require.Len(t, links, 1)
|
||||
require.Equal(t, http.StatusOK, x.putBlock(links[0], blocks[0].data, nil))
|
||||
status, body = x.commit(f, f.commitRequest(blocks))
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
dump := x.u.Dump()
|
||||
require.Len(t, dump, 1)
|
||||
require.NoError(t, dump[0].Err)
|
||||
|
||||
// Uploads to a committed revision are refused.
|
||||
assert.Equal(t, http.StatusNotFound, x.putBlock(links[0], blocks[0].data, nil))
|
||||
|
||||
// Another user's AddressID is refused.
|
||||
other := x.s.AddUser("other", "pass")
|
||||
g := x.newFileRequest("g")
|
||||
x.create(g)
|
||||
addressID := other.addresses[0].id
|
||||
status, body = x.send("POST", "/drive/blocks", &blockUploadRequest{LinkID: g.linkID, RevisionID: g.revID, AddressID: &addressID, VolumeID: &x.volume, BlockList: []blockUploadInput{g.encrypt([]byte("x"))[0].in}})
|
||||
requireRejected(t, api.CodeInvalidValue, status, body, "AddressID")
|
||||
}
|
||||
|
||||
func TestRevisionRules(t *testing.T) {
|
||||
x := newUploader(t)
|
||||
id := x.u.SeedFile(x.u.RootID(), "f", []byte("old"), time.Now())
|
||||
current := x.u.view().d.links[id].activeRevision().id
|
||||
path := "/drive/v2/volumes/" + x.volume + "/files/" + id + "/revisions"
|
||||
stale := "stale"
|
||||
status, body := x.send("POST", path, &createRevisionRequest{CurrentRevisionID: &stale})
|
||||
requireRejected(t, api.CodeInvalidRequirements, status, body, "CurrentRevisionID")
|
||||
status, body = x.send("POST", path, &createRevisionRequest{})
|
||||
requireRejected(t, api.CodeInvalidRequirements, status, body, "CurrentRevisionID")
|
||||
|
||||
uid := testClientUID
|
||||
status, body = x.send("POST", path, &createRevisionRequest{CurrentRevisionID: ¤t, ClientUID: &uid})
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
var resp createRevisionResponse
|
||||
require.NoError(t, json.Unmarshal(body, &resp))
|
||||
assert.Equal(t, []string{resp.Revision.ID}, x.u.Drafts())
|
||||
|
||||
// The name conflict of a file with a draft revision names both.
|
||||
status, body = x.send("POST", "/drive/v2/volumes/"+x.volume+"/files", x.newFileRequest("f").req)
|
||||
requireRejected(t, api.CodeAlreadyExists, status, body)
|
||||
var conflict conflictResponse
|
||||
require.NoError(t, json.Unmarshal(body, &conflict))
|
||||
require.NotNil(t, conflict.Details.ConflictRevisionID)
|
||||
assert.Equal(t, current, *conflict.Details.ConflictRevisionID)
|
||||
require.NotNil(t, conflict.Details.ConflictDraftRevisionID)
|
||||
assert.Equal(t, resp.Revision.ID, *conflict.Details.ConflictDraftRevisionID)
|
||||
|
||||
status, body = x.send("DELETE", path+"/"+resp.Revision.ID, nil)
|
||||
require.Equal(t, http.StatusOK, status, "body: %s", body)
|
||||
assert.Empty(t, x.u.Drafts())
|
||||
status, body = x.send("DELETE", path+"/"+current, nil)
|
||||
requireRejected(t, api.CodeNotExists, status, body)
|
||||
dump := x.u.Dump()
|
||||
require.Len(t, dump, 1)
|
||||
assert.Equal(t, []byte("old"), dump[0].Content)
|
||||
}
|
||||
|
||||
// TestUploadWireNames checks the backend's upload request types decode
|
||||
// into the fake's without unknown fields.
|
||||
func TestUploadWireNames(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
client any
|
||||
fake any
|
||||
}{
|
||||
{&api.CreateFileRequest{}, &createFileRequest{}},
|
||||
{&api.CreateRevisionRequest{}, &createRevisionRequest{}},
|
||||
{&api.BlockUploadRequest{BlockList: []api.BlockUploadInfo{{}}, ThumbnailList: []struct{}{}}, &blockUploadRequest{}},
|
||||
{&api.CommitRevisionRequest{}, &commitRevisionRequest{}},
|
||||
} {
|
||||
data, err := json.Marshal(tc.client)
|
||||
require.NoError(t, err)
|
||||
dec := json.NewDecoder(bytes.NewReader(data))
|
||||
dec.DisallowUnknownFields()
|
||||
assert.NoError(t, dec.Decode(tc.fake), "%T", tc.client)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,596 @@
|
||||
package protondrive_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/backend/protondrive"
|
||||
"github.com/rclone/rclone/backend/protondrive/protontest"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
fssync "github.com/rclone/rclone/fs/sync"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
childrenRoute = "GET /drive/v2/volumes/{volumeID}/folders/{linkID}/children"
|
||||
refreshRoute = "POST /auth/v4/refresh"
|
||||
draftRoute = "POST /drive/v2/volumes/{volumeID}/files"
|
||||
)
|
||||
|
||||
// credentialKeys are the config keys holding a saved session.
|
||||
var credentialKeys = []string{"client_uid", "client_access_token", "client_refresh_token", "client_salted_key_pass"}
|
||||
|
||||
// newFakeFs returns an Fs for remote at root.
|
||||
func newFakeFs(t *testing.T, remote, root string) fs.Fs {
|
||||
t.Helper()
|
||||
f, err := fs.NewFs(context.Background(), remote+root)
|
||||
require.NoError(t, err)
|
||||
return f
|
||||
}
|
||||
|
||||
// sessionMap returns the config of a remote on srv with a new saved
|
||||
// session of u.
|
||||
func sessionMap(srv *protontest.Server, u *protontest.User) configmap.Simple {
|
||||
sess := srv.NewSession(u)
|
||||
return configmap.Simple{
|
||||
"endpoint": srv.URL(),
|
||||
"client_uid": sess.UID,
|
||||
"client_access_token": sess.AccessToken,
|
||||
"client_refresh_token": sess.RefreshToken,
|
||||
"client_salted_key_pass": base64.StdEncoding.EncodeToString([]byte(u.KeyPassword())),
|
||||
"draft_client_uid": random.String(32),
|
||||
}
|
||||
}
|
||||
|
||||
// lockedMap is a configmap.Mapper safe for concurrent use, as rclone's
|
||||
// config file is. It records every value set.
|
||||
type lockedMap struct {
|
||||
mu sync.Mutex
|
||||
m configmap.Simple
|
||||
set map[string][]string // every value set per key, in order
|
||||
}
|
||||
|
||||
// newLockedMap returns a lockedMap holding a copy of m.
|
||||
func newLockedMap(m configmap.Simple) *lockedMap {
|
||||
l := &lockedMap{m: configmap.Simple{}, set: map[string][]string{}}
|
||||
for k, v := range m {
|
||||
l.m[k] = v
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
// Get returns the value of key.
|
||||
func (l *lockedMap) Get(key string) (string, bool) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
v, ok := l.m[key]
|
||||
return v, ok
|
||||
}
|
||||
|
||||
// Set sets key to value.
|
||||
func (l *lockedMap) Set(key, value string) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.m[key] = value
|
||||
l.set[key] = append(l.set[key], value)
|
||||
}
|
||||
|
||||
// values returns every value set for key.
|
||||
func (l *lockedMap) values(key string) []string {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return append([]string(nil), l.set[key]...)
|
||||
}
|
||||
|
||||
// processMap is one process's view of a config file shared through
|
||||
// backing: it reads its own writes at once, but they reach backing,
|
||||
// and so other processes, only after delay.
|
||||
type processMap struct {
|
||||
backing *lockedMap
|
||||
delay time.Duration
|
||||
flushes *sync.WaitGroup
|
||||
|
||||
mu sync.Mutex
|
||||
pending map[string]string // values set and not yet in backing
|
||||
}
|
||||
|
||||
// Get returns the value of key.
|
||||
func (p *processMap) Get(key string) (string, bool) {
|
||||
p.mu.Lock()
|
||||
v, ok := p.pending[key]
|
||||
p.mu.Unlock()
|
||||
if ok {
|
||||
return v, true
|
||||
}
|
||||
return p.backing.Get(key)
|
||||
}
|
||||
|
||||
// Set sets key to value, writing it to backing after p.delay.
|
||||
func (p *processMap) Set(key, value string) {
|
||||
p.mu.Lock()
|
||||
p.pending[key] = value
|
||||
p.mu.Unlock()
|
||||
p.flushes.Add(1)
|
||||
time.AfterFunc(p.delay, func() {
|
||||
defer p.flushes.Done()
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.backing.Set(key, value)
|
||||
if p.pending[key] == value {
|
||||
delete(p.pending, key)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// requireValidSession checks the session saved in m is accepted by srv.
|
||||
func requireValidSession(t *testing.T, srv *protontest.Server, m configmap.Getter) {
|
||||
t.Helper()
|
||||
uid, _ := m.Get("client_uid")
|
||||
token, _ := m.Get("client_access_token")
|
||||
req, err := http.NewRequest("GET", srv.URL()+"/core/v4/users", nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("x-pm-appversion", "external-drive-rclone@1.0.0-stable")
|
||||
req.Header.Set("x-pm-uid", uid)
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
resp, err := srv.Client().Do(req)
|
||||
require.NoError(t, err)
|
||||
_ = resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode, "the saved session is rejected")
|
||||
}
|
||||
|
||||
// listConcurrently lists the root of every Fs at once and checks each
|
||||
// listing succeeds.
|
||||
func listConcurrently(ctx context.Context, t *testing.T, fsys ...fs.Fs) {
|
||||
t.Helper()
|
||||
var wg sync.WaitGroup
|
||||
for _, f := range fsys {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_, err := f.List(ctx, "")
|
||||
assert.NoError(t, err)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// TestRegressionTwoFsRefreshRace is a regression test for rclone issues
|
||||
// #9880 and #8135: two remotes on one config refreshing at once lost
|
||||
// the session.
|
||||
func TestRegressionTwoFsRefreshRace(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
m := newLockedMap(sessionMap(srv, u))
|
||||
uid, _ := m.Get("client_uid")
|
||||
f1, err := protondrive.NewFs(ctx, "remote", "", m)
|
||||
require.NoError(t, err)
|
||||
f2, err := protondrive.NewFs(ctx, "remote", "", m)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Both refreshes reach the server with the same refresh token.
|
||||
srv.Inject(protontest.Fault{Method: "POST", Pattern: "/auth/v4/refresh", Times: 2, Gather: true})
|
||||
srv.ExpireAccessToken(uid)
|
||||
srv.ResetRequests()
|
||||
listConcurrently(ctx, t, f1, f2)
|
||||
assert.Equal(t, 2, srv.Requests()[refreshRoute])
|
||||
|
||||
requireValidSession(t, srv, m)
|
||||
for _, k := range credentialKeys {
|
||||
for _, v := range m.values(k) {
|
||||
assert.NotEmpty(t, v, "%s was blanked", k)
|
||||
}
|
||||
}
|
||||
// Both use the surviving session without refreshing again.
|
||||
f1.(fs.DirCacheFlusher).DirCacheFlush()
|
||||
f2.(fs.DirCacheFlusher).DirCacheFlush()
|
||||
listConcurrently(ctx, t, f1, f2)
|
||||
assert.Equal(t, 2, srv.Requests()[refreshRoute])
|
||||
}
|
||||
|
||||
// TestRegressionTwoProcessRefreshRace checks two processes sharing a
|
||||
// config file, which sees each one's saves only after a delay, both
|
||||
// keep working when they refresh at once.
|
||||
func TestRegressionTwoProcessRefreshRace(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
backing := newLockedMap(sessionMap(srv, u))
|
||||
uid, _ := backing.Get("client_uid")
|
||||
var flushes sync.WaitGroup
|
||||
newProcess := func() fs.Fs {
|
||||
p := &processMap{backing: backing, delay: 500 * time.Millisecond, flushes: &flushes, pending: map[string]string{}}
|
||||
f, err := protondrive.NewFs(ctx, "remote", "", p)
|
||||
require.NoError(t, err)
|
||||
return f
|
||||
}
|
||||
f1, f2 := newProcess(), newProcess()
|
||||
|
||||
srv.Inject(protontest.Fault{Method: "POST", Pattern: "/auth/v4/refresh", Times: 2, Gather: true})
|
||||
srv.ExpireAccessToken(uid)
|
||||
srv.ResetRequests()
|
||||
listConcurrently(ctx, t, f1, f2)
|
||||
flushes.Wait()
|
||||
assert.Equal(t, 2, srv.Requests()[refreshRoute], "one refresh wins, the other is rejected")
|
||||
|
||||
requireValidSession(t, srv, backing)
|
||||
for _, k := range credentialKeys {
|
||||
for _, v := range backing.values(k) {
|
||||
assert.NotEmpty(t, v, "%s was blanked", k)
|
||||
}
|
||||
}
|
||||
// The loser adopted the winner's tokens.
|
||||
f1.(fs.DirCacheFlusher).DirCacheFlush()
|
||||
f2.(fs.DirCacheFlusher).DirCacheFlush()
|
||||
listConcurrently(ctx, t, f1, f2)
|
||||
assert.Equal(t, 2, srv.Requests()[refreshRoute])
|
||||
}
|
||||
|
||||
// TestRegressionManyNewFs is a regression test for rclone issue #9816:
|
||||
// many remotes created at once were OOM-killed unlocking keys.
|
||||
//
|
||||
// All the NewFs calls are held until each has fetched its keys, so
|
||||
// their unlocks contend. That unlocks then run one at a time can't be
|
||||
// seen from the server; TestLoadAccountSerialised checks it.
|
||||
func TestRegressionManyNewFs(t *testing.T) {
|
||||
const n = 16
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
maps := make([]configmap.Simple, n)
|
||||
for i := range maps {
|
||||
maps[i] = sessionMap(srv, u)
|
||||
}
|
||||
srv.Inject(protontest.Fault{Method: "GET", Pattern: "/core/v4/addresses", Times: n, Gather: true})
|
||||
|
||||
fsys := make([]fs.Fs, n)
|
||||
var wg sync.WaitGroup
|
||||
for i := range fsys {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
f, err := protondrive.NewFs(ctx, "remote", "", maps[i])
|
||||
if assert.NoError(t, err) {
|
||||
fsys[i] = f
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if t.Failed() {
|
||||
return
|
||||
}
|
||||
listConcurrently(ctx, t, fsys...)
|
||||
// Keys are fetched, and so unlocked, once per Fs.
|
||||
assert.Equal(t, n, srv.Requests()["GET /core/v4/users"])
|
||||
assert.Equal(t, n, srv.Requests()["GET /core/v4/addresses"])
|
||||
}
|
||||
|
||||
// TestRegressionFileOverDirectory is a regression test for rclone issue
|
||||
// #9438: copying a file over a directory destroyed data.
|
||||
func TestRegressionFileOverDirectory(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
remote, _, _ := setupFakeRemote(t)
|
||||
f := newFakeFs(t, remote, "")
|
||||
inside := object.NewMemoryObject("x/inside.txt", time.Now(), []byte("inside"))
|
||||
_, err := operations.Copy(ctx, f, nil, inside.Remote(), inside)
|
||||
require.NoError(t, err)
|
||||
|
||||
src := object.NewMemoryObject("x", time.Now(), []byte("a file"))
|
||||
_, err = operations.Copy(ctx, f, nil, "x", src)
|
||||
require.ErrorIs(t, err, fs.ErrorIsDir)
|
||||
|
||||
f = newFakeFs(t, remote, "")
|
||||
entries, err := f.List(ctx, "x")
|
||||
require.NoError(t, err, "the directory survives")
|
||||
require.Len(t, entries, 1)
|
||||
assert.Equal(t, "x/inside.txt", entries[0].Remote())
|
||||
}
|
||||
|
||||
// TestRegressionDirectoryOverFile is a regression test for rclone issue
|
||||
// #9438: copying into a directory where a file of its name is
|
||||
// destroyed data.
|
||||
func TestRegressionDirectoryOverFile(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
remote, _, _ := setupFakeRemote(t)
|
||||
f := newFakeFs(t, remote, "")
|
||||
file := object.NewMemoryObject("x", time.Now(), []byte("a file"))
|
||||
_, err := operations.Copy(ctx, f, nil, "x", file)
|
||||
require.NoError(t, err)
|
||||
|
||||
src := object.NewMemoryObject("x/y", time.Now(), []byte("inside"))
|
||||
_, err = operations.Copy(ctx, f, nil, "x/y", src)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), `create folder "x": already exists`)
|
||||
|
||||
f = newFakeFs(t, remote, "")
|
||||
o, err := f.NewObject(ctx, "x")
|
||||
require.NoError(t, err, "the file survives")
|
||||
assert.Equal(t, "a file", readObject(t, o))
|
||||
}
|
||||
|
||||
// readObject returns the content of o.
|
||||
func readObject(t *testing.T, o fs.Object) string {
|
||||
t.Helper()
|
||||
rc, err := o.Open(context.Background())
|
||||
require.NoError(t, err)
|
||||
data, err := io.ReadAll(rc)
|
||||
require.NoError(t, rc.Close())
|
||||
require.NoError(t, err)
|
||||
return string(data)
|
||||
}
|
||||
|
||||
// TestRegressionDocsNodeSkipped is a regression test for rclone issue
|
||||
// #7959: a Proton Docs document made every sync fail.
|
||||
func TestRegressionDocsNodeSkipped(t *testing.T) {
|
||||
ctx := accounting.WithStatsGroup(context.Background(), "docs")
|
||||
remote, _, u := setupFakeRemote(t)
|
||||
dir := u.SeedFolder(u.RootID(), "dir")
|
||||
u.SeedDoc(dir, "notes")
|
||||
u.SeedFile(dir, "file.txt", []byte("content"), time.Now())
|
||||
|
||||
src := newFakeFs(t, remote, "dir")
|
||||
dst := newFakeFs(t, "", t.TempDir())
|
||||
require.NoError(t, fssync.Sync(ctx, dst, src, false))
|
||||
assert.Zero(t, accounting.Stats(ctx).GetErrors())
|
||||
entries, err := os.ReadDir(dst.Root())
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 1)
|
||||
assert.Equal(t, "file.txt", entries[0].Name())
|
||||
}
|
||||
|
||||
// TestRegressionBadSignatureFile is a regression test for rclone issue
|
||||
// #8003: files whose signatures didn't verify couldn't be read at all.
|
||||
func TestRegressionBadSignatureFile(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
remote, _, u := setupFakeRemote(t)
|
||||
const second = "second@proton.test"
|
||||
u.AddAddress(second)
|
||||
content := []byte("signed by another address")
|
||||
u.SeedFileContentKeySignedBy(second, time.Now(), u.RootID(), "bad.txt", content)
|
||||
|
||||
f := newFakeFs(t, remote, "")
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 1)
|
||||
o, err := f.NewObject(ctx, "bad.txt")
|
||||
require.NoError(t, err)
|
||||
_, err = o.Open(ctx)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "bad.txt")
|
||||
assert.Contains(t, err.Error(), "signature")
|
||||
assert.Contains(t, err.Error(), "ignore_signature_errors")
|
||||
|
||||
f = newFakeFs(t, strings.TrimSuffix(remote, ":")+",ignore_signature_errors=true:", "")
|
||||
o, err = f.NewObject(ctx, "bad.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, string(content), readObject(t, o))
|
||||
}
|
||||
|
||||
// TestRegressionSDKCreatedFolderResolvable is a regression test for
|
||||
// rclone issue #9622: files in folders made by Proton's own clients
|
||||
// couldn't be found by path.
|
||||
func TestRegressionSDKCreatedFolderResolvable(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
remote, _, u := setupFakeRemote(t)
|
||||
for _, names := range [][3]string{
|
||||
{"dir", "sub", "file"},
|
||||
{"Café", "é combining", "Ünïcødé file.txt"},
|
||||
} {
|
||||
dir := u.SeedFolder(u.RootID(), names[0])
|
||||
sub := u.SeedFolder(dir, names[1])
|
||||
u.SeedFile(sub, names[2], []byte(names[2]), time.Now())
|
||||
|
||||
f := newFakeFs(t, remote, "")
|
||||
o, err := f.NewObject(ctx, strings.Join(names[:], "/"))
|
||||
require.NoError(t, err, "%q", names)
|
||||
assert.Equal(t, names[2], readObject(t, o))
|
||||
}
|
||||
}
|
||||
|
||||
// blockTransfers returns the number of block uploads and downloads in
|
||||
// requests.
|
||||
func blockTransfers(requests map[string]int) int {
|
||||
n := 0
|
||||
for route, count := range requests {
|
||||
if strings.Contains(route, "/storage/") || route == "POST /drive/blocks" {
|
||||
n += count
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// TestRegressionMoveIsServerSide checks a move transfers no content.
|
||||
func TestRegressionMoveIsServerSide(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
remote, srv, _ := setupFakeRemote(t)
|
||||
f := newFakeFs(t, remote, "")
|
||||
src, err := operations.Copy(ctx, f, nil, "a.txt", object.NewMemoryObject("a.txt", time.Now(), []byte("move me")))
|
||||
require.NoError(t, err)
|
||||
|
||||
srv.ResetRequests()
|
||||
moved, err := operations.Move(ctx, f, nil, "dir/b.txt", src)
|
||||
require.NoError(t, err)
|
||||
assert.Zero(t, blockTransfers(srv.Requests()), "%v", srv.Requests())
|
||||
assert.Equal(t, "move me", readObject(t, moved))
|
||||
_, err = f.NewObject(ctx, "a.txt")
|
||||
assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
|
||||
}
|
||||
|
||||
// TestRegressionCopyIsServerSide checks a copy transfers no content.
|
||||
func TestRegressionCopyIsServerSide(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
remote, srv, _ := setupFakeRemote(t)
|
||||
f := newFakeFs(t, remote, "")
|
||||
src, err := operations.Copy(ctx, f, nil, "a.txt", object.NewMemoryObject("a.txt", time.Now(), []byte("copy me")))
|
||||
require.NoError(t, err)
|
||||
|
||||
srv.ResetRequests()
|
||||
copied, err := operations.Copy(ctx, f, nil, "dir/b.txt", src)
|
||||
require.NoError(t, err)
|
||||
assert.Zero(t, blockTransfers(srv.Requests()), "%v", srv.Requests())
|
||||
assert.Equal(t, "copy me", readObject(t, copied))
|
||||
assert.Equal(t, "copy me", readObject(t, src))
|
||||
}
|
||||
|
||||
// TestRegressionMultiThreadDownload checks a multi-thread download
|
||||
// assembles the file correctly.
|
||||
func TestRegressionMultiThreadDownload(t *testing.T) {
|
||||
ctx, ci := fs.AddConfig(context.Background())
|
||||
ci.MultiThreadStreams = 4
|
||||
ci.MultiThreadSet = true
|
||||
ci.MultiThreadCutoff = 1 << 20
|
||||
// Chunks which don't line up with the 4 MiB blocks.
|
||||
ci.MultiThreadChunkSize = 5 << 20
|
||||
remote, srv, u := setupFakeRemote(t)
|
||||
content := []byte(random.String(20 << 20))
|
||||
u.SeedFile(u.RootID(), "big.bin", content, time.Now())
|
||||
|
||||
src := newFakeFs(t, remote, "")
|
||||
o, err := src.NewObject(ctx, "big.bin")
|
||||
require.NoError(t, err)
|
||||
dst := newFakeFs(t, "", t.TempDir())
|
||||
srv.ResetRequests()
|
||||
_, err = operations.Copy(ctx, dst, nil, "big.bin", o)
|
||||
require.NoError(t, err)
|
||||
// A single stream fetches each of the 5 blocks once.
|
||||
assert.Greater(t, srv.Requests()["GET /storage/blocks/{id}"], 5, "not downloaded in parallel ranges")
|
||||
|
||||
got, err := os.ReadFile(filepath.Join(dst.Root(), "big.bin"))
|
||||
require.NoError(t, err)
|
||||
want := sha1.Sum(content)
|
||||
gotSum := sha1.Sum(got)
|
||||
assert.Equal(t, hex.EncodeToString(want[:]), hex.EncodeToString(gotSum[:]))
|
||||
}
|
||||
|
||||
// TestRegressionPutStreamUnknownSize checks an upload of unknown size.
|
||||
func TestRegressionPutStreamUnknownSize(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
remote, _, _ := setupFakeRemote(t)
|
||||
f := newFakeFs(t, remote, "")
|
||||
require.NotNil(t, f.Features().PutStream)
|
||||
content := []byte(random.String(9 << 20))
|
||||
modTime := time.Now().Truncate(time.Second)
|
||||
info := object.NewStaticObjectInfo("stream.bin", modTime, -1, true, nil, nil)
|
||||
_, err := f.Features().PutStream(ctx, bytes.NewReader(content), info)
|
||||
require.NoError(t, err)
|
||||
|
||||
o, err := newFakeFs(t, remote, "").NewObject(ctx, "stream.bin")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(len(content)), o.Size())
|
||||
assert.True(t, modTime.Equal(o.ModTime(ctx)))
|
||||
assert.True(t, readObject(t, o) == string(content), "content differs")
|
||||
}
|
||||
|
||||
// TestRegressionCaptcha checks a CAPTCHA on password sign-in tells the
|
||||
// user how to sign in with a browser.
|
||||
func TestRegressionCaptcha(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
srv.RequireCaptcha(u)
|
||||
m := configmap.Simple{
|
||||
"endpoint": srv.URL(),
|
||||
"username": "user",
|
||||
"password": obscure.MustObscure("pass"),
|
||||
}
|
||||
_, err := protondrive.NewFs(context.Background(), "myremote", "", m)
|
||||
require.Error(t, err)
|
||||
assert.True(t, fserrors.IsFatalError(err), "not fatal: %v", err)
|
||||
assert.Contains(t, err.Error(), `rclone config reconnect myremote:`)
|
||||
assert.Contains(t, err.Error(), "browser sign-in")
|
||||
}
|
||||
|
||||
// TestRegressionAppVersionRejected checks Proton refusing rclone's
|
||||
// version is a fatal error which says to upgrade.
|
||||
func TestRegressionAppVersionRejected(t *testing.T) {
|
||||
remote, srv, _ := setupFakeRemote(t)
|
||||
f := newFakeFs(t, remote, "")
|
||||
srv.Inject(protontest.Fault{Method: "GET", Pattern: "/drive/v2/volumes/{volumeID}/folders/{linkID}/children", Status: http.StatusBadRequest, Code: 5003})
|
||||
_, err := f.List(context.Background(), "")
|
||||
require.Error(t, err)
|
||||
assert.True(t, fserrors.IsFatalError(err), "not fatal: %v", err)
|
||||
assert.Contains(t, err.Error(), "upgrade rclone")
|
||||
}
|
||||
|
||||
// TestRegressionFeatureBlockedRetried checks Proton's code for a
|
||||
// feature disabled, rate-limited or blocked is retried, not fatal.
|
||||
func TestRegressionFeatureBlockedRetried(t *testing.T) {
|
||||
remote, srv, u := setupFakeRemote(t)
|
||||
u.SeedFile(u.RootID(), "a.txt", []byte("a"), time.Now())
|
||||
f := newFakeFs(t, remote, "")
|
||||
srv.Inject(protontest.Fault{Method: "GET", Pattern: "/drive/v2/volumes/{volumeID}/folders/{linkID}/children", Times: 2, Status: http.StatusUnprocessableEntity, Code: 2028})
|
||||
entries, err := f.List(context.Background(), "")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, entries, 1)
|
||||
}
|
||||
|
||||
// TestRegressionRenameResendNotFatal checks a rename whose response was
|
||||
// lost, so that its resend fails Proton's stale-state check, doesn't
|
||||
// abort the whole run as an "upgrade rclone" error.
|
||||
func TestRegressionRenameResendNotFatal(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
remote, srv, u := setupFakeRemote(t)
|
||||
u.SeedFile(u.RootID(), "old.txt", []byte("x"), time.Now())
|
||||
f := newFakeFs(t, remote, "")
|
||||
o, err := f.NewObject(ctx, "old.txt")
|
||||
require.NoError(t, err)
|
||||
srv.Inject(protontest.Fault{Method: "PUT", Pattern: "/drive/v2/volumes/{volumeID}/links/{linkID}/rename", Status: http.StatusBadGateway, AfterHandler: true, Times: 1})
|
||||
_, err = f.Features().Move(ctx, o, "new.txt")
|
||||
require.Error(t, err, "the resend saw the rename already made")
|
||||
assert.False(t, fserrors.IsFatalError(err), "fatal: %v", err)
|
||||
assert.NotContains(t, err.Error(), "upgrade rclone")
|
||||
}
|
||||
|
||||
// TestRegressionQuotaExceeded checks running out of space is reported
|
||||
// with Proton's message and not retried.
|
||||
func TestRegressionQuotaExceeded(t *testing.T) {
|
||||
remote, srv, _ := setupFakeRemote(t)
|
||||
f := newFakeFs(t, remote, "")
|
||||
srv.Inject(protontest.Fault{Method: "POST", Pattern: "/drive/v2/volumes/{volumeID}/files", Status: http.StatusUnprocessableEntity, Code: 200001})
|
||||
srv.ResetRequests()
|
||||
src := object.NewMemoryObject("a.txt", time.Now(), []byte("no room"))
|
||||
_, err := operations.Copy(context.Background(), f, nil, "a.txt", src)
|
||||
require.Error(t, err)
|
||||
assert.True(t, fserrors.IsNoRetryError(err), "retryable: %v", err)
|
||||
assert.Contains(t, err.Error(), "protontest: injected fault")
|
||||
assert.Equal(t, 1, srv.Requests()[draftRoute])
|
||||
}
|
||||
|
||||
// TestRegressionRateLimited checks listing waits out every Retry-After
|
||||
// and then succeeds.
|
||||
func TestRegressionRateLimited(t *testing.T) {
|
||||
const times, retryAfter = 5, 1
|
||||
remote, srv, u := setupFakeRemote(t)
|
||||
u.SeedFile(u.RootID(), "a.txt", []byte("a"), time.Now())
|
||||
f := newFakeFs(t, remote, "")
|
||||
srv.Inject(protontest.Fault{Method: "GET", Pattern: "/drive/v2/volumes/{volumeID}/folders/{linkID}/children", Times: times, Status: http.StatusTooManyRequests, RetryAfter: retryAfter})
|
||||
srv.ResetRequests()
|
||||
start := time.Now()
|
||||
entries, err := f.List(context.Background(), "")
|
||||
require.NoError(t, err)
|
||||
assert.GreaterOrEqual(t, time.Since(start), times*retryAfter*time.Second)
|
||||
assert.Len(t, entries, 1)
|
||||
assert.Equal(t, times+1, srv.Requests()[childrenRoute])
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,662 @@
|
||||
package protondrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ProtonMail/go-srp"
|
||||
"github.com/rclone/rclone/backend/protondrive/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
// errSessionExpired is returned when the session can no longer be
|
||||
// refreshed and the user must sign in again.
|
||||
var errSessionExpired = errors.New("session expired")
|
||||
|
||||
// errNotSignedIn is returned by an authenticated call of a session
|
||||
// which has no credentials, as after signing out.
|
||||
var errNotSignedIn = errors.New("not signed in")
|
||||
|
||||
// errCaptcha is returned by password sign-in when Proton asks for human
|
||||
// verification, which only its sign-in page can do.
|
||||
var errCaptcha = errors.New("password sign-in needs a CAPTCHA solved")
|
||||
|
||||
const (
|
||||
// srpBitLength is the bit length of Proton's SRP modulus.
|
||||
srpBitLength = 2048
|
||||
|
||||
// forkClientID identifies rclone to Proton's sign-in page.
|
||||
forkClientID = "external-drive"
|
||||
|
||||
// forkSignInURL is the sign-in page for session forks, followed by
|
||||
// the escaped payload.
|
||||
forkSignInURL = "https://account.proton.me/desktop/login?app=drive&pv=3#payload="
|
||||
)
|
||||
|
||||
var (
|
||||
// rcloneVersionRe splits an rclone version into its number and suffix.
|
||||
rcloneVersionRe = regexp.MustCompile(`^v?([0-9]+\.[0-9]+\.[0-9]+)(?:-(.*))?$`)
|
||||
|
||||
// invalidBuildCharsRe matches characters not allowed in a semver
|
||||
// build identifier.
|
||||
invalidBuildCharsRe = regexp.MustCompile(`[^0-9A-Za-z-]+`)
|
||||
)
|
||||
|
||||
// appVersionFromRcloneVersion returns the x-pm-appversion for the
|
||||
// rclone version v, as external-drive-rclone@<semver>-<channel>.
|
||||
//
|
||||
// Releases map to stable, betas to beta with their build number as
|
||||
// build metadata, and anything else to alpha. An unparsable version
|
||||
// maps to 1.0.0-stable.
|
||||
func appVersionFromRcloneVersion(v string) string {
|
||||
const prefix = "external-drive-rclone@"
|
||||
m := rcloneVersionRe.FindStringSubmatch(strings.TrimSpace(v))
|
||||
if m == nil {
|
||||
return prefix + "1.0.0-stable"
|
||||
}
|
||||
number, suffix := m[1], m[2]
|
||||
switch {
|
||||
case suffix == "":
|
||||
return prefix + number + "-stable"
|
||||
case suffix == "beta":
|
||||
return prefix + number + "-beta"
|
||||
case strings.HasPrefix(suffix, "beta."):
|
||||
var build []string
|
||||
for _, id := range strings.Split(strings.TrimPrefix(suffix, "beta."), ".") {
|
||||
id = strings.Trim(invalidBuildCharsRe.ReplaceAllString(id, "-"), "-")
|
||||
if id != "" {
|
||||
build = append(build, id)
|
||||
}
|
||||
}
|
||||
if len(build) == 0 {
|
||||
return prefix + number + "-beta"
|
||||
}
|
||||
return prefix + number + "-beta+" + strings.Join(build, ".")
|
||||
default:
|
||||
return prefix + number + "-alpha"
|
||||
}
|
||||
}
|
||||
|
||||
// credentials are the saved tokens of a session.
|
||||
type credentials struct {
|
||||
UID string
|
||||
AccessToken string
|
||||
RefreshToken string
|
||||
KeyPassword string // salted password which unlocks the user keys
|
||||
}
|
||||
|
||||
// credentialStore persists credentials, possibly shared with other
|
||||
// sessions and processes.
|
||||
type credentialStore interface {
|
||||
Load() credentials
|
||||
Save(credentials)
|
||||
}
|
||||
|
||||
// memoryStore is a credentialStore held in memory.
|
||||
type memoryStore struct {
|
||||
mu sync.Mutex
|
||||
creds credentials
|
||||
}
|
||||
|
||||
// Load returns the stored credentials.
|
||||
func (m *memoryStore) Load() credentials {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.creds
|
||||
}
|
||||
|
||||
// Save replaces the stored credentials.
|
||||
func (m *memoryStore) Save(c credentials) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.creds = c
|
||||
}
|
||||
|
||||
// session makes API calls with a Proton session, refreshing its tokens
|
||||
// when they expire.
|
||||
type session struct {
|
||||
srv *rest.Client
|
||||
store credentialStore
|
||||
adoptWait time.Duration // how long a rejected refresh waits for the store to change
|
||||
unlockHook func() // if set, called by loadAccount while unlockMu is held; for tests
|
||||
reconnect string // if set, ends errors which need the user to sign in again, saying how
|
||||
|
||||
refreshMu sync.Mutex // held while refreshing, so one refresh runs at a time
|
||||
|
||||
mu sync.Mutex
|
||||
creds credentials
|
||||
deadRefresh string // refresh token the server rejected
|
||||
serverDate time.Time // Date of the last response
|
||||
serverDateLocal time.Time // local time serverDate was received
|
||||
requirements string // x-pm-drive-requirements last logged
|
||||
}
|
||||
|
||||
// newSession returns a session for the API at baseURL using the
|
||||
// credentials in store.
|
||||
func newSession(client *http.Client, baseURL, appVersion string, store credentialStore) *session {
|
||||
s := &session{
|
||||
srv: rest.NewClient(client).SetRoot(baseURL),
|
||||
store: store,
|
||||
adoptWait: 3 * time.Second,
|
||||
creds: store.Load(),
|
||||
}
|
||||
s.srv.SetHeader("x-pm-appversion", appVersion)
|
||||
s.srv.SetHeader("Accept", "application/vnd.protonmail.v1+json")
|
||||
s.srv.SetErrorHandler(errorHandler)
|
||||
return s
|
||||
}
|
||||
|
||||
// errorHandler decodes an unsuccessful response into an *api.Error.
|
||||
func errorHandler(resp *http.Response) error {
|
||||
body, err := rest.ReadBody(resp)
|
||||
apiErr := &api.Error{Status: resp.StatusCode}
|
||||
if err == nil {
|
||||
_ = json.Unmarshal(body, apiErr)
|
||||
}
|
||||
if apiErr.Message == "" {
|
||||
apiErr.Message = resp.Status
|
||||
}
|
||||
return apiErr
|
||||
}
|
||||
|
||||
// current returns the credentials in use.
|
||||
func (s *session) current() credentials {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.creds
|
||||
}
|
||||
|
||||
// serverTime returns the server's clock: the Date of the last response
|
||||
// advanced by the local time elapsed since. It is zero before the
|
||||
// first response.
|
||||
func (s *session) serverTime() time.Time {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.serverDate.IsZero() {
|
||||
return time.Time{}
|
||||
}
|
||||
return s.serverDate.Add(time.Since(s.serverDateLocal))
|
||||
}
|
||||
|
||||
// call makes an authenticated API call with rest.Client.CallJSON
|
||||
// semantics.
|
||||
//
|
||||
// An unsuccessful HTTP status or a response Code outside 1000-1999 is
|
||||
// returned as an *api.Error. Without credentials it returns
|
||||
// errNotSignedIn and sends nothing. On HTTP 401 the session is refreshed and
|
||||
// the call retried once, so opts.Body, if set, must be an io.Seeker to
|
||||
// be retried. If the session can't be refreshed the error wraps
|
||||
// errSessionExpired.
|
||||
func (s *session) call(ctx context.Context, opts *rest.Opts, request, response any) (*http.Response, error) {
|
||||
creds := s.current()
|
||||
if creds.UID == "" {
|
||||
return nil, s.signInNeeded(errNotSignedIn)
|
||||
}
|
||||
resp, err := s.do(ctx, opts, request, response, &creds)
|
||||
var apiErr *api.Error
|
||||
if !errors.As(err, &apiErr) || apiErr.Status != http.StatusUnauthorized {
|
||||
return resp, err
|
||||
}
|
||||
if refreshErr := s.refresh(ctx, creds.AccessToken); refreshErr != nil {
|
||||
// Not the 401's response, so that a retry is decided by the
|
||||
// refresh's error.
|
||||
return nil, refreshErr
|
||||
}
|
||||
if opts.Body != nil {
|
||||
seeker, ok := opts.Body.(io.Seeker)
|
||||
if !ok {
|
||||
return resp, err
|
||||
}
|
||||
if _, seekErr := seeker.Seek(0, io.SeekStart); seekErr != nil {
|
||||
return resp, fmt.Errorf("rewind request body: %w", seekErr)
|
||||
}
|
||||
}
|
||||
creds = s.current()
|
||||
if creds.UID == "" {
|
||||
// Signed out while the call was made.
|
||||
return resp, s.signInNeeded(errNotSignedIn)
|
||||
}
|
||||
return s.do(ctx, opts, request, response, &creds)
|
||||
}
|
||||
|
||||
// callUnauthenticated makes an API call like call but without the
|
||||
// session's credentials and without refreshing.
|
||||
func (s *session) callUnauthenticated(ctx context.Context, opts *rest.Opts, request, response any) (*http.Response, error) {
|
||||
return s.do(ctx, opts, request, response, nil)
|
||||
}
|
||||
|
||||
// do makes one API call, authenticated with creds if not nil.
|
||||
func (s *session) do(ctx context.Context, opts *rest.Opts, request, response any, creds *credentials) (*http.Response, error) {
|
||||
if creds != nil {
|
||||
opts = opts.Copy()
|
||||
opts.ExtraHeaders = maps.Clone(opts.ExtraHeaders)
|
||||
if opts.ExtraHeaders == nil {
|
||||
opts.ExtraHeaders = make(map[string]string, 2)
|
||||
}
|
||||
opts.ExtraHeaders["x-pm-uid"] = creds.UID
|
||||
opts.ExtraHeaders["Authorization"] = "Bearer " + creds.AccessToken
|
||||
}
|
||||
var body json.RawMessage
|
||||
resp, err := s.srv.CallJSON(ctx, opts, request, &body)
|
||||
if resp != nil {
|
||||
s.observeDate(resp.Header.Get("Date"))
|
||||
s.observeRequirements(resp.Header.Values("x-pm-drive-requirements"))
|
||||
}
|
||||
if err != nil || opts.NoResponse {
|
||||
return resp, err
|
||||
}
|
||||
var result struct {
|
||||
Code int `json:"Code"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return resp, fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
// A body without a Code is not a Proton envelope, so is passed on.
|
||||
if result.Code != 0 && result.Code/1000 != 1 {
|
||||
apiErr := &api.Error{Status: resp.StatusCode}
|
||||
_ = json.Unmarshal(body, apiErr)
|
||||
return resp, apiErr
|
||||
}
|
||||
if response != nil {
|
||||
if err := json.Unmarshal(body, response); err != nil {
|
||||
return resp, fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// observeDate records the server's clock from a Date header.
|
||||
func (s *session) observeDate(date string) {
|
||||
t, err := http.ParseTime(date)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.serverDate = t
|
||||
s.serverDateLocal = time.Now()
|
||||
}
|
||||
|
||||
// maxRequirementsLog bounds the x-pm-drive-requirements text logged.
|
||||
const maxRequirementsLog = 1024
|
||||
|
||||
// observeRequirements logs at debug level the x-pm-drive-requirements
|
||||
// header values of a response, by which Proton announces requirements
|
||||
// it will enforce, when they differ from those last logged.
|
||||
func (s *session) observeRequirements(values []string) {
|
||||
if len(values) == 0 {
|
||||
return
|
||||
}
|
||||
v := strings.Join(values, ", ")
|
||||
if len(v) > maxRequirementsLog {
|
||||
v = v[:maxRequirementsLog]
|
||||
}
|
||||
s.mu.Lock()
|
||||
changed := v != s.requirements
|
||||
s.requirements = v
|
||||
s.mu.Unlock()
|
||||
if changed {
|
||||
fs.Debugf(nil, "protondrive: Proton announces drive requirements: %q", v)
|
||||
}
|
||||
}
|
||||
|
||||
// refresh replaces the access token rejected with a working one.
|
||||
//
|
||||
// It uses tokens another goroutine or another user of the store has
|
||||
// already refreshed if there are any, since refresh tokens are single
|
||||
// use. Otherwise it asks the server, saving the new tokens to the
|
||||
// store. If the server rejects the refresh token, it waits up to
|
||||
// s.adoptWait for another user of the store to save new tokens before
|
||||
// returning errSessionExpired. Credentials are never removed from the
|
||||
// store, because another process may have just saved new ones.
|
||||
func (s *session) refresh(ctx context.Context, rejected string) error {
|
||||
s.refreshMu.Lock()
|
||||
defer s.refreshMu.Unlock()
|
||||
|
||||
cur := s.current()
|
||||
if cur.AccessToken != rejected {
|
||||
return nil
|
||||
}
|
||||
if stored := s.store.Load(); stored.RefreshToken != "" && stored.RefreshToken != cur.RefreshToken {
|
||||
fs.Debugf(nil, "protondrive: using session tokens refreshed elsewhere")
|
||||
s.setCredentials(stored)
|
||||
return nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
dead := cur.RefreshToken == "" || cur.RefreshToken == s.deadRefresh
|
||||
s.mu.Unlock()
|
||||
if dead {
|
||||
return s.signInNeeded(errSessionExpired)
|
||||
}
|
||||
|
||||
fs.Debugf(nil, "protondrive: refreshing session")
|
||||
var out api.AuthRefreshResponse
|
||||
_, err := s.do(ctx, &rest.Opts{Method: "POST", Path: "/auth/v4/refresh"}, &api.AuthRefreshRequest{
|
||||
ResponseType: "token",
|
||||
GrantType: "refresh_token",
|
||||
RefreshToken: cur.RefreshToken,
|
||||
}, &out, &cur)
|
||||
if err != nil {
|
||||
// As the SDK does, treat a client error other than rate
|
||||
// limiting, or a timeout, as the refresh token being rejected.
|
||||
var apiErr *api.Error
|
||||
if errors.As(err, &apiErr) && apiErr.Status >= 400 && apiErr.Status < 500 &&
|
||||
apiErr.Status != http.StatusTooManyRequests && apiErr.Status != http.StatusRequestTimeout {
|
||||
// Another user of the store may have spent the same refresh
|
||||
// token and not saved its replacement yet.
|
||||
if stored, ok := s.waitForStoredRefresh(ctx, cur.RefreshToken); ok {
|
||||
fs.Debugf(nil, "protondrive: using session tokens refreshed elsewhere")
|
||||
s.setCredentials(stored)
|
||||
return nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.deadRefresh = cur.RefreshToken
|
||||
s.mu.Unlock()
|
||||
return s.signInNeeded(fmt.Errorf("%w: %w", errSessionExpired, err))
|
||||
}
|
||||
return fmt.Errorf("refresh session: %w", err)
|
||||
}
|
||||
if out.AccessToken == "" {
|
||||
return errors.New("refresh session: no access token in response")
|
||||
}
|
||||
next := cur
|
||||
next.AccessToken = out.AccessToken
|
||||
if out.UID != "" {
|
||||
next.UID = out.UID
|
||||
}
|
||||
if out.RefreshToken != "" {
|
||||
next.RefreshToken = out.RefreshToken
|
||||
}
|
||||
s.setCredentials(next)
|
||||
s.store.Save(next)
|
||||
return nil
|
||||
}
|
||||
|
||||
// signInNeeded returns err, which the user can only fix by signing in
|
||||
// again, as a fatal error ending with s.reconnect.
|
||||
func (s *session) signInNeeded(err error) error {
|
||||
if s.reconnect != "" {
|
||||
err = fmt.Errorf("%w: %s", err, s.reconnect)
|
||||
}
|
||||
return fserrors.FatalError(err)
|
||||
}
|
||||
|
||||
// waitForStoredRefresh polls the store for up to s.adoptWait for a
|
||||
// refresh token other than spent and returns those credentials. It
|
||||
// returns false if none appears or ctx is done.
|
||||
func (s *session) waitForStoredRefresh(ctx context.Context, spent string) (credentials, bool) {
|
||||
const step = 50 * time.Millisecond
|
||||
deadline := time.Now().Add(s.adoptWait)
|
||||
for {
|
||||
if stored := s.store.Load(); stored.RefreshToken != "" && stored.RefreshToken != spent {
|
||||
return stored, true
|
||||
}
|
||||
if !time.Now().Before(deadline) {
|
||||
return credentials{}, false
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return credentials{}, false
|
||||
case <-time.After(min(step, time.Until(deadline))):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// setCredentials replaces the credentials in use.
|
||||
func (s *session) setCredentials(c credentials) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.creds = c
|
||||
}
|
||||
|
||||
// signInError marks a human verification error as errCaptcha.
|
||||
func signInError(err error) error {
|
||||
var apiErr *api.Error
|
||||
if errors.As(err, &apiErr) && apiErr.Code == api.CodeHumanVerification {
|
||||
return fmt.Errorf("sign in: %w: %w", errCaptcha, err)
|
||||
}
|
||||
return fmt.Errorf("sign in: %w", err)
|
||||
}
|
||||
|
||||
// saveCredentials makes c the credentials in use and saves them to the store.
|
||||
func (s *session) saveCredentials(c credentials) {
|
||||
s.setCredentials(c)
|
||||
s.store.Save(c)
|
||||
}
|
||||
|
||||
// loginSRP signs in with a username and password using SRP, saving the
|
||||
// new credentials to the store only if sign-in succeeds.
|
||||
//
|
||||
// The key password derives from mailboxPassword if set, else from
|
||||
// password. twoFA is called for a TOTP code only if the account needs
|
||||
// one. A request for human verification returns an error wrapping
|
||||
// errCaptcha.
|
||||
func (s *session) loginSRP(ctx context.Context, username, password, mailboxPassword string, twoFA func() (string, error)) (err error) {
|
||||
var info api.AuthInfoResponse
|
||||
_, err = s.callUnauthenticated(ctx, &rest.Opts{Method: "POST", Path: "/core/v4/auth/info"}, &api.AuthInfoRequest{
|
||||
Intent: "Proton",
|
||||
Username: username,
|
||||
}, &info)
|
||||
if err != nil {
|
||||
return signInError(err)
|
||||
}
|
||||
if info.Modulus == "" || info.ServerEphemeral == "" || info.SRPSession == "" {
|
||||
return errors.New("sign in: incomplete SRP challenge")
|
||||
}
|
||||
srpAuth, err := srp.NewAuth(info.Version, username, []byte(password), info.Salt, info.Modulus, info.ServerEphemeral)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sign in: %w", err)
|
||||
}
|
||||
proofs, err := srpAuth.GenerateProofs(srpBitLength)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sign in: %w", err)
|
||||
}
|
||||
|
||||
var auth api.AuthResponse
|
||||
_, err = s.callUnauthenticated(ctx, &rest.Opts{Method: "POST", Path: "/core/v4/auth"}, &api.AuthRequest{
|
||||
Username: username,
|
||||
ClientEphemeral: base64.StdEncoding.EncodeToString(proofs.ClientEphemeral),
|
||||
ClientProof: base64.StdEncoding.EncodeToString(proofs.ClientProof),
|
||||
SRPSession: info.SRPSession,
|
||||
PersistentCookies: 1,
|
||||
Payload: map[string]string{},
|
||||
}, &auth)
|
||||
if err != nil {
|
||||
return signInError(err)
|
||||
}
|
||||
serverProof, err := base64.StdEncoding.DecodeString(auth.ServerProof)
|
||||
if err != nil || subtle.ConstantTimeCompare(serverProof, proofs.ExpectedServerProof) != 1 {
|
||||
return errors.New("sign in: server proof verification failed")
|
||||
}
|
||||
creds := credentials{UID: auth.UID, AccessToken: auth.AccessToken, RefreshToken: auth.RefreshToken}
|
||||
if creds.UID == "" || creds.AccessToken == "" || creds.RefreshToken == "" {
|
||||
return errors.New("sign in: no session tokens in response")
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
// Don't leave a half signed in session behind.
|
||||
_, _ = s.do(ctx, &rest.Opts{Method: "DELETE", Path: "/core/v4/auth"}, nil, nil, &creds)
|
||||
}
|
||||
}()
|
||||
|
||||
if auth.TwoFA.Enabled != 0 {
|
||||
if auth.TwoFA.Enabled&api.TwoFATOTP == 0 {
|
||||
return errors.New("sign in: password sign-in supports only TOTP two-factor authentication: choose browser sign-in")
|
||||
}
|
||||
if twoFA == nil {
|
||||
return errors.New("sign in: a two-factor authentication code is required")
|
||||
}
|
||||
var code string
|
||||
code, err = twoFA()
|
||||
if err != nil {
|
||||
return fmt.Errorf("sign in: %w", err)
|
||||
}
|
||||
_, err = s.do(ctx, &rest.Opts{Method: "POST", Path: "/core/v4/auth/2fa"}, &api.Auth2FARequest{TwoFactorCode: code}, nil, &creds)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sign in: two-factor authentication: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
keyPassword := password
|
||||
if mailboxPassword != "" {
|
||||
keyPassword = mailboxPassword
|
||||
} else if auth.PasswordMode == api.PasswordModeTwo {
|
||||
return errors.New("sign in: the account has a separate mailbox password: set mailbox_password")
|
||||
}
|
||||
var user api.UserResponse
|
||||
if _, err = s.do(ctx, &rest.Opts{Method: "GET", Path: "/core/v4/users"}, nil, &user, &creds); err != nil {
|
||||
return fmt.Errorf("sign in: get user: %w", err)
|
||||
}
|
||||
var salts api.KeySaltsResponse
|
||||
if _, err = s.do(ctx, &rest.Opts{Method: "GET", Path: "/core/v4/keys/salts"}, nil, &salts, &creds); err != nil {
|
||||
return fmt.Errorf("sign in: get key salts: %w", err)
|
||||
}
|
||||
salt, ok := primaryKeySalt(user.User.Keys, salts.KeySalts)
|
||||
if !ok {
|
||||
return errors.New("sign in: no key salt for the account")
|
||||
}
|
||||
salted, err := computeKeyPassword([]byte(keyPassword), salt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sign in: %w", err)
|
||||
}
|
||||
creds.KeyPassword = string(salted)
|
||||
s.saveCredentials(creds)
|
||||
return nil
|
||||
}
|
||||
|
||||
// primaryKeySalt returns the salt of the primary user key in keys.
|
||||
//
|
||||
// The SDK uses the first salt, which is wrong for an account whose
|
||||
// user keys have different salts, so the first salt is only the
|
||||
// fallback when none matches the primary key.
|
||||
func primaryKeySalt(keys []api.UserKey, salts []api.KeySalt) (string, bool) {
|
||||
for _, key := range keys {
|
||||
if key.Primary != 1 {
|
||||
continue
|
||||
}
|
||||
for _, salt := range salts {
|
||||
if salt.ID == key.ID {
|
||||
return salt.KeySalt, true
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(salts) == 0 {
|
||||
return "", false
|
||||
}
|
||||
return salts[0].KeySalt, true
|
||||
}
|
||||
|
||||
// forkRequest is a pending browser sign-in.
|
||||
type forkRequest struct {
|
||||
UserCode string // code shown on the sign-in page
|
||||
Selector string // polls the fork for the new session
|
||||
URL string // sign-in page for the user to open
|
||||
key []byte // AES-256 key the page encrypts the key password with
|
||||
}
|
||||
|
||||
// forkStart starts a browser sign-in by requesting a session fork.
|
||||
func (s *session) forkStart(ctx context.Context) (*forkRequest, error) {
|
||||
var out api.ForkInitResponse
|
||||
if _, err := s.callUnauthenticated(ctx, &rest.Opts{Method: "GET", Path: "/auth/v4/sessions/forks"}, nil, &out); err != nil {
|
||||
return nil, fmt.Errorf("browser sign-in: %w", err)
|
||||
}
|
||||
if out.UserCode == "" || out.Selector == "" {
|
||||
return nil, errors.New("browser sign-in: no fork in response")
|
||||
}
|
||||
key := make([]byte, 32)
|
||||
if _, err := rand.Read(key); err != nil {
|
||||
return nil, fmt.Errorf("browser sign-in: %w", err)
|
||||
}
|
||||
payload := "0:" + out.UserCode + ":" + base64.StdEncoding.EncodeToString(key) + ":" + forkClientID
|
||||
return &forkRequest{
|
||||
UserCode: out.UserCode,
|
||||
Selector: out.Selector,
|
||||
// The SDK escapes the payload with encodeURIComponent, which
|
||||
// QueryEscape matches for the characters a payload contains.
|
||||
URL: forkSignInURL + url.QueryEscape(payload),
|
||||
key: key,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// forkPoll waits for the user to complete the browser sign-in f,
|
||||
// polling every interval for up to timeout, then saves the new
|
||||
// credentials to the store.
|
||||
func (s *session) forkPoll(ctx context.Context, f *forkRequest, interval, timeout time.Duration) error {
|
||||
pollCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
stopped := func() error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("browser sign-in: not completed within %v", timeout)
|
||||
}
|
||||
opts := &rest.Opts{Method: "GET", Path: "/auth/v4/sessions/forks/" + url.PathEscape(f.Selector)}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-pollCtx.Done():
|
||||
return stopped()
|
||||
case <-ticker.C:
|
||||
}
|
||||
var out api.ForkStatusResponse
|
||||
_, err := s.callUnauthenticated(pollCtx, opts, nil, &out)
|
||||
var apiErr *api.Error
|
||||
if errors.As(err, &apiErr) && apiErr.Status == http.StatusUnprocessableEntity {
|
||||
// The fork isn't approved yet.
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
if pollCtx.Err() != nil {
|
||||
return stopped()
|
||||
}
|
||||
return fmt.Errorf("browser sign-in: %w", err)
|
||||
}
|
||||
keyPassword, err := decryptForkPayload(out.Payload, f.key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("browser sign-in: %w", err)
|
||||
}
|
||||
creds := credentials{UID: out.UID, AccessToken: out.AccessToken, RefreshToken: out.RefreshToken, KeyPassword: keyPassword}
|
||||
if creds.UID == "" || creds.AccessToken == "" || creds.RefreshToken == "" {
|
||||
return errors.New("browser sign-in: no session tokens in response")
|
||||
}
|
||||
s.saveCredentials(creds)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// logout ends the session on the server and removes the credentials
|
||||
// from the session and the store, so neither can be used again.
|
||||
func (s *session) logout(ctx context.Context) error {
|
||||
var err error
|
||||
if s.current().UID != "" {
|
||||
_, err = s.call(ctx, &rest.Opts{Method: "DELETE", Path: "/core/v4/auth"}, nil, nil)
|
||||
if errors.Is(err, errSessionExpired) {
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
// A refresh in flight would otherwise save its new tokens after
|
||||
// these are removed.
|
||||
s.refreshMu.Lock()
|
||||
s.saveCredentials(credentials{})
|
||||
s.refreshMu.Unlock()
|
||||
if err != nil {
|
||||
return fmt.Errorf("sign out: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,596 @@
|
||||
package protondrive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/pquerna/otp/totp"
|
||||
"github.com/rclone/rclone/backend/protondrive/api"
|
||||
"github.com/rclone/rclone/backend/protondrive/protontest"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
testAppVersion = "external-drive-rclone@1.77.0-stable"
|
||||
refreshRoute = "POST /auth/v4/refresh"
|
||||
)
|
||||
|
||||
// appVersionGrammar is Proton's x-pm-appversion grammar with the name
|
||||
// fixed to rclone and the channel required.
|
||||
var appVersionGrammar = regexp.MustCompile(`^external-drive-rclone@[0-9]+\.[0-9]+\.[0-9]+-(stable|beta|alpha)(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$`)
|
||||
|
||||
func TestAppVersionFromRcloneVersion(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"v1.77.0", "external-drive-rclone@1.77.0-stable"},
|
||||
{"1.77.0", "external-drive-rclone@1.77.0-stable"},
|
||||
{"v1.77.0-DEV", "external-drive-rclone@1.77.0-alpha"},
|
||||
{"v1.77.0-beta.10754.448d7361c", "external-drive-rclone@1.77.0-beta+10754.448d7361c"},
|
||||
{"v1.77.0-beta.10754.448d7361c.fix/proton_drive", "external-drive-rclone@1.77.0-beta+10754.448d7361c.fix-proton-drive"},
|
||||
{"v1.77.0-beta", "external-drive-rclone@1.77.0-beta"},
|
||||
{"v1.77.0-rc1", "external-drive-rclone@1.77.0-alpha"},
|
||||
{"garbage", "external-drive-rclone@1.0.0-stable"},
|
||||
{"", "external-drive-rclone@1.0.0-stable"},
|
||||
{"v1.77", "external-drive-rclone@1.0.0-stable"},
|
||||
} {
|
||||
got := appVersionFromRcloneVersion(test.in)
|
||||
assert.Equal(t, test.want, got, "input %q", test.in)
|
||||
assert.Regexp(t, appVersionGrammar, got, "input %q", test.in)
|
||||
}
|
||||
}
|
||||
|
||||
// headerRecorder records the headers of every request it sends.
|
||||
type headerRecorder struct {
|
||||
mu sync.Mutex
|
||||
headers []http.Header
|
||||
next http.RoundTripper
|
||||
}
|
||||
|
||||
func (h *headerRecorder) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
h.mu.Lock()
|
||||
h.headers = append(h.headers, req.Header.Clone())
|
||||
h.mu.Unlock()
|
||||
return h.next.RoundTrip(req)
|
||||
}
|
||||
|
||||
// newTestSession signs a new user in to srv and returns a session using
|
||||
// a store holding its credentials.
|
||||
func newTestSession(t *testing.T, srv *protontest.Server) (*session, *memoryStore, protontest.Session) {
|
||||
t.Helper()
|
||||
tokens := srv.NewSession(srv.AddUser("user", "pass"))
|
||||
store := &memoryStore{creds: credentials{
|
||||
UID: tokens.UID,
|
||||
AccessToken: tokens.AccessToken,
|
||||
RefreshToken: tokens.RefreshToken,
|
||||
KeyPassword: "key-pass",
|
||||
}}
|
||||
return newSession(srv.Client(), srv.URL(), testAppVersion, store), store, tokens
|
||||
}
|
||||
|
||||
// authCall makes an authenticated test call.
|
||||
func authCall(ctx context.Context, s *session) error {
|
||||
var out struct {
|
||||
UserID string `json:"UserID"`
|
||||
}
|
||||
_, err := s.call(ctx, &rest.Opts{Method: "GET", Path: "/tests/auth"}, nil, &out)
|
||||
if err == nil && out.UserID == "" {
|
||||
return errors.New("no UserID in response")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func TestSessionHeaders(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
tokens := srv.NewSession(srv.AddUser("user", "pass"))
|
||||
rec := &headerRecorder{next: srv.Client().Transport}
|
||||
client := &http.Client{Transport: rec}
|
||||
store := &memoryStore{creds: credentials{UID: tokens.UID, AccessToken: tokens.AccessToken, RefreshToken: tokens.RefreshToken}}
|
||||
s := newSession(client, srv.URL(), testAppVersion, store)
|
||||
|
||||
require.NoError(t, authCall(context.Background(), s))
|
||||
_, err := s.callUnauthenticated(context.Background(), &rest.Opts{Method: "GET", Path: "/tests/ping"}, nil, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Len(t, rec.headers, 2)
|
||||
auth, unauth := rec.headers[0], rec.headers[1]
|
||||
assert.Equal(t, tokens.UID, auth.Get("x-pm-uid"))
|
||||
assert.Equal(t, "Bearer "+tokens.AccessToken, auth.Get("Authorization"))
|
||||
for _, h := range []http.Header{auth, unauth} {
|
||||
assert.Equal(t, testAppVersion, h.Get("x-pm-appversion"))
|
||||
assert.Equal(t, "application/vnd.protonmail.v1+json", h.Get("Accept"))
|
||||
assert.Empty(t, h.Get("x-pm-drive-sdk-version"))
|
||||
}
|
||||
assert.Empty(t, unauth.Get("x-pm-uid"))
|
||||
assert.Empty(t, unauth.Get("Authorization"))
|
||||
}
|
||||
|
||||
func TestSessionUnauthenticatedNoRefresh(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
s, _, _ := newTestSession(t, srv)
|
||||
_, err := s.callUnauthenticated(context.Background(), &rest.Opts{Method: "GET", Path: "/tests/auth"}, nil, nil)
|
||||
var apiErr *api.Error
|
||||
require.ErrorAs(t, err, &apiErr)
|
||||
assert.Equal(t, http.StatusUnauthorized, apiErr.Status)
|
||||
assert.Zero(t, srv.Requests()[refreshRoute])
|
||||
}
|
||||
|
||||
func TestSessionRefreshOn401(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
s, store, tokens := newTestSession(t, srv)
|
||||
srv.ExpireAccessToken(tokens.UID)
|
||||
|
||||
require.NoError(t, authCall(context.Background(), s))
|
||||
assert.Equal(t, 1, srv.Requests()[refreshRoute])
|
||||
assert.Equal(t, 2, srv.Requests()["GET /tests/auth"])
|
||||
|
||||
saved := store.Load()
|
||||
assert.Equal(t, tokens.UID, saved.UID)
|
||||
assert.NotEqual(t, tokens.AccessToken, saved.AccessToken)
|
||||
assert.NotEqual(t, tokens.RefreshToken, saved.RefreshToken)
|
||||
assert.Equal(t, "key-pass", saved.KeyPassword)
|
||||
|
||||
// The new tokens are used without another refresh.
|
||||
require.NoError(t, authCall(context.Background(), s))
|
||||
assert.Equal(t, 1, srv.Requests()[refreshRoute])
|
||||
}
|
||||
|
||||
func TestSessionRefreshResendsBody(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
s, _, tokens := newTestSession(t, srv)
|
||||
type echo struct {
|
||||
Echo string `json:"Echo"`
|
||||
}
|
||||
opts := &rest.Opts{Method: "POST", Path: "/tests/echo"}
|
||||
|
||||
srv.ExpireAccessToken(tokens.UID)
|
||||
var out echo
|
||||
_, err := s.call(context.Background(), opts, &echo{Echo: "request"}, &out)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "request", out.Echo)
|
||||
|
||||
srv.ExpireAccessToken(tokens.UID)
|
||||
out = echo{}
|
||||
bodyOpts := *opts
|
||||
bodyOpts.Body = bytes.NewReader([]byte(`{"Echo":"body"}`))
|
||||
_, err = s.call(context.Background(), &bodyOpts, nil, &out)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "body", out.Echo)
|
||||
assert.Equal(t, 2, srv.Requests()[refreshRoute])
|
||||
}
|
||||
|
||||
func TestSessionRefreshSingleFlight(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
s, _, tokens := newTestSession(t, srv)
|
||||
srv.ExpireAccessToken(tokens.UID)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errs := make([]error, 20)
|
||||
for i := range errs {
|
||||
wg.Go(func() {
|
||||
errs[i] = authCall(context.Background(), s)
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
for i, err := range errs {
|
||||
assert.NoError(t, err, "call %d", i)
|
||||
}
|
||||
assert.Equal(t, 1, srv.Requests()[refreshRoute])
|
||||
}
|
||||
|
||||
func TestSessionRefreshRereadsStore(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
a, store, tokens := newTestSession(t, srv)
|
||||
b := newSession(srv.Client(), srv.URL(), testAppVersion, store)
|
||||
srv.ExpireAccessToken(tokens.UID)
|
||||
|
||||
require.NoError(t, authCall(context.Background(), a))
|
||||
assert.Equal(t, 1, srv.Requests()[refreshRoute])
|
||||
|
||||
// b still holds the rotated tokens, so its first call gets 401 and
|
||||
// adopts a's tokens from the store instead of reusing its refresh
|
||||
// token.
|
||||
require.NoError(t, authCall(context.Background(), b))
|
||||
assert.Equal(t, 1, srv.Requests()[refreshRoute])
|
||||
assert.Equal(t, 4, srv.Requests()["GET /tests/auth"])
|
||||
}
|
||||
|
||||
func TestSessionRefreshTwoSessionsConcurrent(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
a, store, tokens := newTestSession(t, srv)
|
||||
b := newSession(srv.Client(), srv.URL(), testAppVersion, store)
|
||||
// Holding each refresh makes both sessions post the same refresh
|
||||
// token before either saves the new one.
|
||||
srv.Inject(protontest.Fault{Method: "POST", Pattern: "/auth/v4/refresh", Delay: 200 * time.Millisecond})
|
||||
srv.ExpireAccessToken(tokens.UID)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errs := make([]error, 2)
|
||||
for i, s := range []*session{a, b} {
|
||||
wg.Go(func() {
|
||||
errs[i] = authCall(context.Background(), s)
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
for i, err := range errs {
|
||||
assert.NoError(t, err, "session %d", i)
|
||||
}
|
||||
assert.Equal(t, 2, srv.Requests()[refreshRoute], "both sessions posted the refresh token")
|
||||
|
||||
// The stored credentials work in a fresh session.
|
||||
c := newSession(srv.Client(), srv.URL(), testAppVersion, store)
|
||||
require.NoError(t, authCall(context.Background(), c))
|
||||
assert.Equal(t, 2, srv.Requests()[refreshRoute])
|
||||
}
|
||||
|
||||
func TestSessionRevoked(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
s, store, tokens := newTestSession(t, srv)
|
||||
s.adoptWait = 50 * time.Millisecond
|
||||
s.reconnect = `run "rclone config reconnect myremote:"`
|
||||
before := store.Load()
|
||||
srv.RevokeSession(tokens.UID)
|
||||
|
||||
err := authCall(context.Background(), s)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, errSessionExpired)
|
||||
assert.True(t, fserrors.IsFatalError(err), "error is fatal: %v", err)
|
||||
assert.Contains(t, err.Error(), `rclone config reconnect myremote:`)
|
||||
assert.Equal(t, before, store.Load(), "credentials kept")
|
||||
assert.Equal(t, 1, srv.Requests()[refreshRoute])
|
||||
|
||||
// Further calls fail the same way without retrying the refresh.
|
||||
err = authCall(context.Background(), s)
|
||||
assert.ErrorIs(t, err, errSessionExpired)
|
||||
assert.Equal(t, 1, srv.Requests()[refreshRoute])
|
||||
assert.Equal(t, before, store.Load(), "credentials kept")
|
||||
}
|
||||
|
||||
func TestSessionServerTime(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
srv.SetClockSkew(3 * time.Hour)
|
||||
s, _, _ := newTestSession(t, srv)
|
||||
assert.True(t, s.serverTime().IsZero())
|
||||
|
||||
require.NoError(t, authCall(context.Background(), s))
|
||||
assert.WithinDuration(t, time.Now().Add(3*time.Hour), s.serverTime(), 5*time.Second)
|
||||
}
|
||||
|
||||
func TestSessionAPIError(t *testing.T) {
|
||||
for _, status := range []int{http.StatusUnprocessableEntity, http.StatusOK} {
|
||||
srv := protontest.New(t)
|
||||
s, _, _ := newTestSession(t, srv)
|
||||
srv.Inject(protontest.Fault{Pattern: "/tests/auth", Times: 1, Status: status, Code: api.CodeAlreadyExists})
|
||||
|
||||
err := authCall(context.Background(), s)
|
||||
var apiErr *api.Error
|
||||
require.ErrorAs(t, err, &apiErr, "status %d", status)
|
||||
assert.Equal(t, status, apiErr.Status)
|
||||
assert.Equal(t, api.CodeAlreadyExists, apiErr.Code)
|
||||
assert.Zero(t, srv.Requests()[refreshRoute])
|
||||
}
|
||||
}
|
||||
|
||||
// newLoginSession returns a session for srv with an empty store.
|
||||
func newLoginSession(srv *protontest.Server) (*session, *memoryStore) {
|
||||
store := &memoryStore{}
|
||||
return newSession(srv.Client(), srv.URL(), testAppVersion, store), store
|
||||
}
|
||||
|
||||
// noTwoFA is a twoFA callback for accounts without 2FA.
|
||||
func noTwoFA() (string, error) {
|
||||
return "", errors.New("unexpected 2FA prompt")
|
||||
}
|
||||
|
||||
// requireSignedIn checks store holds complete credentials for a working
|
||||
// session whose key password derives from keyPassword and u's key salt.
|
||||
func requireSignedIn(t *testing.T, s *session, store *memoryStore, u *protontest.User, keyPassword string) {
|
||||
t.Helper()
|
||||
saved := store.Load()
|
||||
require.NotEmpty(t, saved.UID)
|
||||
require.NotEmpty(t, saved.AccessToken)
|
||||
require.NotEmpty(t, saved.RefreshToken)
|
||||
want, err := computeKeyPassword([]byte(keyPassword), u.KeySalt)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, string(want), saved.KeyPassword)
|
||||
assert.Equal(t, saved, s.current())
|
||||
require.NoError(t, authCall(context.Background(), s))
|
||||
}
|
||||
|
||||
func TestLoginSRP(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "correct password")
|
||||
|
||||
s, store := newLoginSession(srv)
|
||||
err := s.loginSRP(context.Background(), "user", "wrong password", "", noTwoFA)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, credentials{}, store.Load(), "nothing stored")
|
||||
assert.Equal(t, credentials{}, s.current())
|
||||
|
||||
require.NoError(t, s.loginSRP(context.Background(), "user", "correct password", "", noTwoFA))
|
||||
requireSignedIn(t, s, store, u, "correct password")
|
||||
}
|
||||
|
||||
func TestLoginSRPSaltOfPrimaryKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
oldSalt := u.KeySalt
|
||||
u.AddUserKey()
|
||||
require.NotEqual(t, oldSalt, u.KeySalt)
|
||||
|
||||
// The salts list the old key first; the primary key's salt is second.
|
||||
s, store := newLoginSession(srv)
|
||||
require.NoError(t, s.loginSRP(context.Background(), "user", "pass", "", noTwoFA))
|
||||
requireSignedIn(t, s, store, u, "pass")
|
||||
assert.Equal(t, u.KeyPassword(), store.Load().KeyPassword)
|
||||
a, err := loadAccount(context.Background(), s, []byte(store.Load().KeyPassword))
|
||||
require.NoError(t, err)
|
||||
a.close()
|
||||
}
|
||||
|
||||
func TestPrimaryKeySalt(t *testing.T) {
|
||||
keys := []api.UserKey{{ID: "old", Primary: 0}, {ID: "new", Primary: 1}}
|
||||
salts := []api.KeySalt{{ID: "old", KeySalt: "old-salt"}, {ID: "new", KeySalt: "new-salt"}}
|
||||
|
||||
salt, ok := primaryKeySalt(keys, salts)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "new-salt", salt)
|
||||
|
||||
salt, ok = primaryKeySalt(keys, []api.KeySalt{{ID: "other", KeySalt: "first"}, {ID: "x", KeySalt: "second"}})
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "first", salt, "falls back to the first salt")
|
||||
|
||||
salt, ok = primaryKeySalt(nil, salts)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "old-salt", salt)
|
||||
|
||||
_, ok = primaryKeySalt(keys, nil)
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestLoginSRPBadServerProof(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
srv.CorruptServerProof(u)
|
||||
|
||||
s, store := newLoginSession(srv)
|
||||
err := s.loginSRP(context.Background(), "user", "pass", "", noTwoFA)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "server proof verification")
|
||||
assert.Equal(t, credentials{}, store.Load(), "nothing stored")
|
||||
assert.Equal(t, credentials{}, s.current())
|
||||
}
|
||||
|
||||
func TestLoginSRPPasswords(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, password := range []string{
|
||||
"with space ",
|
||||
`quo"te'`,
|
||||
"пароль-密码",
|
||||
strings.Repeat("abcdefghij", 10),
|
||||
} {
|
||||
t.Run(password, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", password)
|
||||
s, store := newLoginSession(srv)
|
||||
require.NoError(t, s.loginSRP(context.Background(), "user", password, "", noTwoFA))
|
||||
requireSignedIn(t, s, store, u, password)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginSRPMailboxPassword(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddTwoPasswordUser("user", "login password", "mailbox password")
|
||||
|
||||
s, store := newLoginSession(srv)
|
||||
require.NoError(t, s.loginSRP(context.Background(), "user", "login password", "mailbox password", noTwoFA))
|
||||
requireSignedIn(t, s, store, u, "mailbox password")
|
||||
|
||||
// Without the mailbox password the key password can't be derived.
|
||||
s, store = newLoginSession(srv)
|
||||
err := s.loginSRP(context.Background(), "user", "login password", "", noTwoFA)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "mailbox_password")
|
||||
assert.Equal(t, credentials{}, store.Load())
|
||||
}
|
||||
|
||||
func TestLoginSRPTOTP(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
secret := u.EnableTOTP()
|
||||
|
||||
calls := 0
|
||||
s, store := newLoginSession(srv)
|
||||
require.NoError(t, s.loginSRP(context.Background(), "user", "pass", "", func() (string, error) {
|
||||
calls++
|
||||
return totp.GenerateCode(secret, time.Now())
|
||||
}))
|
||||
assert.Equal(t, 1, calls)
|
||||
requireSignedIn(t, s, store, u, "pass")
|
||||
|
||||
calls = 0
|
||||
s, store = newLoginSession(srv)
|
||||
err := s.loginSRP(context.Background(), "user", "pass", "", func() (string, error) {
|
||||
calls++
|
||||
return "000000x", nil
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, 1, calls)
|
||||
assert.Equal(t, credentials{}, store.Load())
|
||||
}
|
||||
|
||||
func TestLoginSRPCaptcha(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
srv.RequireCaptcha(u)
|
||||
|
||||
s, store := newLoginSession(srv)
|
||||
err := s.loginSRP(context.Background(), "user", "pass", "", noTwoFA)
|
||||
require.ErrorIs(t, err, errCaptcha)
|
||||
assert.Equal(t, credentials{}, store.Load())
|
||||
}
|
||||
|
||||
func TestForkLogin(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
s, store := newLoginSession(srv)
|
||||
|
||||
f, err := s.forkStart(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, f.Selector)
|
||||
assert.Len(t, f.key, 32)
|
||||
prefix := "https://account.proton.me/desktop/login?app=drive&pv=3#payload="
|
||||
require.True(t, strings.HasPrefix(f.URL, prefix), f.URL)
|
||||
payload, err := url.QueryUnescape(strings.TrimPrefix(f.URL, prefix))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "0:"+f.UserCode+":"+base64.StdEncoding.EncodeToString(f.key)+":external-drive", payload)
|
||||
|
||||
go func() {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
assert.NoError(t, srv.ApproveFork(f.URL, u))
|
||||
}()
|
||||
require.NoError(t, s.forkPoll(context.Background(), f, 10*time.Millisecond, 10*time.Second))
|
||||
requireSignedIn(t, s, store, u, "pass")
|
||||
}
|
||||
|
||||
func TestForkLoginTimeout(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
s, store := newLoginSession(srv)
|
||||
f, err := s.forkStart(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
start := time.Now()
|
||||
err = s.forkPoll(context.Background(), f, 10*time.Millisecond, 100*time.Millisecond)
|
||||
require.Error(t, err)
|
||||
assert.GreaterOrEqual(t, time.Since(start), 100*time.Millisecond)
|
||||
assert.Less(t, time.Since(start), 5*time.Second)
|
||||
assert.Greater(t, srv.Requests()["GET /auth/v4/sessions/forks/{selector}"], 1)
|
||||
assert.Equal(t, credentials{}, store.Load())
|
||||
}
|
||||
|
||||
func TestForkLoginCancel(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
s, store := newLoginSession(srv)
|
||||
f, err := s.forkStart(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
time.AfterFunc(50*time.Millisecond, cancel)
|
||||
start := time.Now()
|
||||
err = s.forkPoll(ctx, f, 10*time.Millisecond, 10*time.Minute)
|
||||
assert.ErrorIs(t, err, context.Canceled)
|
||||
assert.Less(t, time.Since(start), 5*time.Second)
|
||||
assert.Equal(t, credentials{}, store.Load())
|
||||
}
|
||||
|
||||
func TestLogout(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
srv.AddUser("user", "pass")
|
||||
s, store := newLoginSession(srv)
|
||||
require.NoError(t, s.loginSRP(context.Background(), "user", "pass", "", noTwoFA))
|
||||
before := store.Load()
|
||||
|
||||
require.NoError(t, s.logout(context.Background()))
|
||||
assert.Equal(t, credentials{}, store.Load())
|
||||
srv.ResetRequests()
|
||||
err := authCall(context.Background(), s)
|
||||
assert.ErrorIs(t, err, errNotSignedIn)
|
||||
assert.True(t, fserrors.IsFatalError(err))
|
||||
assert.Empty(t, srv.Requests(), "no request is sent without a session")
|
||||
|
||||
// The server ended the session too.
|
||||
old := newSession(srv.Client(), srv.URL(), testAppVersion, &memoryStore{creds: before})
|
||||
old.adoptWait = 0
|
||||
assert.ErrorIs(t, authCall(context.Background(), old), errSessionExpired)
|
||||
}
|
||||
|
||||
func TestCallDuringLogout(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
srv := protontest.New(t)
|
||||
s, _, _ := newTestSession(t, srv)
|
||||
// The call is rejected only after the session has signed out.
|
||||
srv.Inject(protontest.Fault{Method: "GET", Pattern: "/tests/auth", Times: 1, Status: http.StatusUnauthorized, Delay: 500 * time.Millisecond})
|
||||
errc := make(chan error, 1)
|
||||
go func() { errc <- authCall(ctx, s) }()
|
||||
require.Eventually(t, func() bool { return srv.Requests()["GET /tests/auth"] == 1 }, 5*time.Second, 5*time.Millisecond)
|
||||
require.NoError(t, s.logout(ctx))
|
||||
|
||||
err := <-errc
|
||||
assert.ErrorIs(t, err, errNotSignedIn)
|
||||
assert.Equal(t, 1, srv.Requests()["GET /tests/auth"], "retried without credentials")
|
||||
assert.Zero(t, srv.Requests()[refreshRoute])
|
||||
}
|
||||
|
||||
func TestRefreshDuringLogout(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
srv := protontest.New(t)
|
||||
tokens := srv.NewSession(srv.AddUser("user", "pass"))
|
||||
refreshing, release := make(chan struct{}), make(chan struct{})
|
||||
// The refresh succeeds, but its answer arrives only after the
|
||||
// session has signed out.
|
||||
client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if r.URL.Path != "/auth/v4/refresh" {
|
||||
return srv.Client().Transport.RoundTrip(r)
|
||||
}
|
||||
close(refreshing)
|
||||
<-release
|
||||
body := `{"Code":1000,"UID":"` + tokens.UID + `","AccessToken":"new-access","RefreshToken":"new-refresh"}`
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"application/json"}}, Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
|
||||
})}
|
||||
store := &memoryStore{creds: credentials{UID: tokens.UID, AccessToken: tokens.AccessToken, RefreshToken: tokens.RefreshToken, KeyPassword: "key-pass"}}
|
||||
s := newSession(client, srv.URL(), testAppVersion, store)
|
||||
srv.Inject(protontest.Fault{Method: "GET", Pattern: "/tests/auth", Times: 1, Status: http.StatusUnauthorized})
|
||||
|
||||
errc := make(chan error, 1)
|
||||
go func() { errc <- authCall(ctx, s) }()
|
||||
<-refreshing
|
||||
logoutErr := make(chan error, 1)
|
||||
go func() { logoutErr <- s.logout(ctx) }()
|
||||
require.Eventually(t, func() bool { return srv.Requests()["DELETE /core/v4/auth"] == 1 }, 5*time.Second, 5*time.Millisecond)
|
||||
close(release)
|
||||
require.NoError(t, <-logoutErr)
|
||||
assert.Error(t, <-errc, "the call succeeded with tokens refreshed after signing out")
|
||||
|
||||
assert.Equal(t, credentials{}, store.Load(), "tokens saved after signing out")
|
||||
assert.Equal(t, credentials{}, s.current())
|
||||
}
|
||||
|
||||
func TestRefreshTransientErrorRetried(t *testing.T) {
|
||||
for _, status := range []int{http.StatusServiceUnavailable, http.StatusRequestTimeout} {
|
||||
t.Run(http.StatusText(status), func(t *testing.T) {
|
||||
c, srv, _ := newTestClient(t)
|
||||
ctx := context.Background()
|
||||
srv.ExpireAccessToken(c.s.current().UID)
|
||||
srv.Inject(protontest.Fault{Method: "POST", Pattern: "/auth/v4/refresh", Status: status, Times: 1})
|
||||
_, _, err := c.about(ctx)
|
||||
require.NoError(t, err, "a %d on the refresh wasn't retried", status)
|
||||
assert.Equal(t, 2, srv.Requests()["POST /auth/v4/refresh"])
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"Comment": "Keys made by TestGoldenCapture for the tests alone.",
|
||||
"Email": "golden@example.invalid",
|
||||
"Keys": {
|
||||
"AddressKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavQ+PBYJKwYBBAHaRw8BAQdAtg+4mrPbyi6Tjo0WFynmLyYdmwCE243WA+Ak\nQ7yhPRL+BwMIOyH0WSccjL3gS/40gwjnjBy7+HjOp7WJxPc2Eh8FDjOHUkrWi0Qi\nj6R4hVeNOStSLVvewHyPphKoSdQxVM4Q5/mvrkQowER6fdEGmpYAjc0fZ29sZGVu\nIDxnb2xkZW5AZXhhbXBsZS5pbnZhbGlkPsK/BBMWCABxBYJq9D48AwsJBwkQdw3k\nqPWekNs1FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ2VOFH8X\nh5yxg5eroXQ18qcCFQgDFgACAhkBApsDAh4BFiEEK2cnqpmZRXt67JN/dw3kqPWe\nkNsAAG6fAQDV8ehsHsFOKQzpeYLsixmXs/h0/DAH2sHWoj49Z9CqeQD+Oa+nLFB7\nqUQUr66n59p4R6RQ8yv5H3HSiP222s9fmAXHiwRq9D48EgorBgEEAZdVAQUBAQdA\n6FutOZ0NLbRiCAqLSd2txpkq/hfv0HgedU5u29B4mkYDAQoJ/gcDCDsh9FknHIy9\n4NWExuQW8SLjmpFYDBp0NvnfM8CdqvOnOmpuE3Fefei8GtlgkYmQferCHqw38bBv\nrKXVMg3lxzsl4PhnykpmmcYP7niNnCzCrgQYFggAYAWCavQ+PAkQdw3kqPWekNs1\nFAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ34i6EDHnkQF/krz\nlrpZWpICmwwWIQQrZyeqmZlFe3rsk393DeSo9Z6Q2wAA4ocA/Rr/qWSSPj+q+uSN\n5imo6ET2IhnLJvcfhXQaMjN9aZvRAP9RWbdgdexbkbyoPI1I6zT1Yfb3Lg/9Vk/h\n0WiqPzmhAQ==\n=13w4\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"AddressKeyPassphrase": "qoww/oDsj2wClAHEcf1w6KRmiiIegYZN8gnIEYiR+yo=",
|
||||
"RootKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavRU8BYJKwYBBAHaRw8BAQdAyYx5cRV31ovNSetf7tN542QIjXofDRwb5REq\nbgAUfTL+BwMIMMJR+/aYJWvg8RWLH459XM6tdGMymPBJf6uYIfNkCZPJuH/xgQlP\nUgnfauP/HE3mB4raN0Ewfdd5t3YHFH3LkKi9PTeyBIOxoSq+03v9ec0fZ29sZGVu\nIDxnb2xkZW5AZXhhbXBsZS5pbnZhbGlkPsK/BBMWCABxBYJq9FTwAwsJBwkQcokY\nuWqSnJY1FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZwCnRKL/\nEVKgVnzorTxO7x0CFQgDFgACAhkBApsDAh4BFiEElLi748AC52UTTWtJcokYuWqS\nnJYAACRRAPwN9DnHokHSakHFOxhk4uQ7Vr782ISOzHngJ/NgOYtmygD/R/1ctxo5\nY32q3O+N1memq7Dij6DfU4GN6WbTA6ypaArHiwRq9FTwEgorBgEEAZdVAQUBAQdA\ncVbkQob/gu0TMrl+8sxRZKfmMKrnRC8QtN63i88MFE8DAQoJ/gcDCDDCUfv2mCVr\n4OixHp4Iy/zOVSPhV3eQSvPgFIb7lBp0m3nSGBgs6BGHRWLOys+Od7+AlLA0HjGa\nQf4cLnRKfvmiHV69WS0flFzsqxwfkdrCrgQYFggAYAWCavRU8AkQcokYuWqSnJY1\nFAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ4m//YFqqVLRVFiw\njVvAD08CmwwWIQSUuLvjwALnZRNNa0lyiRi5apKclgAAAbYA/jDBwlC+AccJp5hS\n1nHztq3bhQqs88y8eS7fCucXwDA7AQDkXzeePakq4JUtoPCiTe/Zf5wzME0BJ6Hk\nEu0vBJkWCg==\n=oZX1\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"RootKeyPassphrase": "a2W6imzT596kxrkFLloSQicvaE0k4CcUFRAmBkdDp3s=",
|
||||
"RootHashKey": "-----BEGIN PGP MESSAGE-----\n\nwV4DC8CXHujYN4ASAQdAAGQNjBfh2oFIyAgUxqF6Nk9KA70Yg/S4IuThF/nFnAMw\nTjLXIcTbQmZsjmIwJi2xSDFAKvaGADIE7f2J9s65RPHkzcea2btnJOeifP0cJJSy\n0sBZAT+gElIVu/UWLXbOl4r84L9SyJCrb5dAOxrAOmTjdpEif28K3oOo3AanKCc8\nkBVYnFz5tUWq4wjYLYKOG/UZJFY+X6ue6gBBxR7tbXWQ4BGcrF7EeeaSE2XyfX+A\naSsZ7EuLKD6Ow3c0lxFx77/9hZFDlYGRyfZIDFERZw/ibOfMITxIj9DsXI31Xl8u\nf40ZrdRragq2Wzk8t8sd0h5pvFzw0y99PjWlzeo4pTKHT5lfz+xPpj4Nn09KMqhR\nMW1VWtfTES3rwNEYw3NmLRSrEP480kr1tp10uLt4hNfDs9ymCCW/N5ZdNc0ycKJ6\nt7fH9QPa4fbPAUdgNRBjniYeco6hyUzEH7s4kSayX+IqB2zicdpEQhqdM/Q=\n=DyXT\n-----END PGP MESSAGE-----"
|
||||
}
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+240
@@ -0,0 +1,240 @@
|
||||
{
|
||||
"Comment": "A folder tree cli wrote, captured by TestGoldenCapture: the encrypted data is the client's, its key packets re-wrapped and its signatures re-made with the keys in keys.json.",
|
||||
"Label": "cli",
|
||||
"Nodes": [
|
||||
{
|
||||
"ID": "link-1",
|
||||
"ParentID": "",
|
||||
"Folder": {
|
||||
"Link": {
|
||||
"Name": "-----BEGIN PGP MESSAGE-----\n\nwV4DC8CXHujYN4ASAQdAVCWynZ4Ws578li2WEIKaTFx0a28laac8Q4y480PhQwYw\nrrwUKsATjjjZHNjU60zBAZsyibShEyrfP3G34OU/qWHSilsLAL3UeFmIb290neu6\n0sBXAXvvlSSMnGjcRsD+d+eYvEgcoczww5Zr2RoOiGHMA0jTAuxh3/HgGGjzP9KM\nWdzIEYaLFEc+HsO7QcpChbO6AfFGrzzC6fOXj4XOqmZio7zR3EPv3jHXqk92y6x1\nRdT8MBw7LLqzGRUPk3hKQ3e26vtpPIlYwjQsgmajf+Vwb9T1BtJKYMmR+0kaAGHF\nh+rcq5G9qtyz/Rx1I/pU02AIC/bd57Vp6rSAMPjw+rF64OhJPDmOoG6rjdawOrvz\n/5Wmu04GxOVVhbSlqK2tfOml0RPW39TD0txZWfL2iIobpVHOFXmYjqWjnTVNV9ai\nYE4MmvcYHmb6LK/lMuxVtJSQHRzqbEFg440w1aW+dwqr61G77NQE3UPm\n=CrKu\n-----END PGP MESSAGE-----",
|
||||
"NameHash": "2b4a9b35edf57ac4c20fadffcc0699aa47e35de65a62462dc2f7921efa8c8986",
|
||||
"NodeKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavWdyxYJKwYBBAHaRw8BAQdAulKMYMn3ftjv8EbhL4LPlCghXtsB4upOrikr\ntdSvhm/+BwMImdBBjTbsZdPgCmpRj6uXfVFGrpxeTwmlJIqp/vWcAo4Fkhu3ciMD\nTsvhXlMF4PSN3JB0bs9ave1fv0edpVYfuuxobEyplWsO7/u/+hLgkM0fZ29sZGVu\nIDxnb2xkZW5AZXhhbXBsZS5pbnZhbGlkPsK/BBMWCABxBYJq9Z3LAwsJBwkQhgWK\nhuOqBQk1FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ/N4Ok+M\nMo/7aNeC9+TNU1gCFQgDFgACAhkBApsDAh4BFiEEl5I219jVL4IkLk69hgWKhuOq\nBQkAAP+yAP4ndOJRRMPm8bFr2q30hazPl2ZG/6h51st993O0aiy63gD9ECzejnqI\ni5J90M8MnunYrCno1/pVjhZjQAVYycNmzgfHiwRq9Z3LEgorBgEEAZdVAQUBAQdA\nKc5oTJGdyYBUPgEVw/ZEKJ5AzAR55F6L76XYQ831aAcDAQoJ/gcDCJnQQY027GXT\n4Bh5iomv/s/yVQ9w/Pb6MgGRaQB8udlhAHLEzwTRNboLYm68rXTxraGrvJoOTcAB\nqbidcoSD5/3fxAOWkAEYpAUSfvv6PrLCrgQYFggAYAWCavWdywkQhgWKhuOqBQk1\nFAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZzDwqhidrzqSrO0J\nU/jki3wCmwwWIQSXkjbX2NUvgiQuTr2GBYqG46oFCQAA4nkBAJ+rZmGuzEYO4zHC\n5ZJPxb/GnX7/xHyiizAECkt0tWLQAP4qfpSm+CxmhONe18fKRx3XVjwo6EGx996O\nU1n/AgwCAg==\n=lSeB\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"NodePassphrase": "-----BEGIN PGP MESSAGE-----\n\nwV4DC8CXHujYN4ASAQdAfkWEN2fIMJmG6UzcTe81wFDtNwUAsCZOGDukHWPokRsw\nlgrkSJCZ3HkKw++Z09FztwVcyRtd18m50FJnm494eN5SztcWvoFs4/zRuPREAoO1\n0l0B4cHmcy0t9AG4tdCkoVltnePRrNoCWUDJ0tmm7RFZB1vM4gCZnf7lyo8bFkVy\nNJaTRy2UETGGodSpLFM6cbdjjcJuW5KVNEwzr3FlL2toa/TX3KMslcrovcru5i4=\n=FsDy\n-----END PGP MESSAGE-----",
|
||||
"NodePassphraseSignature": "-----BEGIN PGP SIGNATURE-----\n\nwrsEABYKAG0Fgmr1ngcJEHcN5Kj1npDbRRQAAAAAABwAIHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmek9UDO2r6B60LL1Koime1yh9+qzLuuBFIWEUpKidznmxYh\nBCtnJ6qZmUV7euyTf3cN5Kj1npDbAABFFAD+MgdEGeu7giFRdeCRfs7aIyeyPAdU\nIacUPcMNn6rKSQkA/3Pz/qArb6gqFfsHhcMOHrbZFyhoKxw41EQ1GbabqyIP\n=sOfZ\n-----END PGP SIGNATURE-----",
|
||||
"SignatureEmail": "golden@example.invalid",
|
||||
"NameSignatureEmail": "golden@example.invalid",
|
||||
"MediaType": "",
|
||||
"ContentKeyPacket": "",
|
||||
"ContentKeyPacketSignature": "",
|
||||
"CreateTime": 1794481672,
|
||||
"ModifyTime": 1794481672
|
||||
},
|
||||
"HashKey": "-----BEGIN PGP MESSAGE-----\n\nwV4DMsmw6Vp9F9oSAQdAFdmEPiq1DPeS1/IEcv28qTPvAuIKtK20PP7GR88HImMw\nPZdNCzmu8NtbyW7tOHUlDEY/KpTDwLuD1iuYMsJVYV/taEjWbILw9PdZhjdp5PQG\n0sBpAQUrEiwIl9jnCGfad4fQ8Ctls1IRckFwdxChKaxCaBr2OgyPOa8n+P8oY8Gv\n0uWYR1vURqPqd56eiyCQ3NO22HGeOMQNqyKDYNkzqfwq33dp+twjRjM42bANybqd\nEzHUQVsM6GDA+6lzhqbc/dUyoh6/YxX4Rc7R6pjjCh8b4JUHpefnAEeLIagyhIRa\n0gkvwtkjm+LC1W+RnOvvMhI+/b2ADGzeoTauTT1FCCIP0uxXEFPP3PBxPximgrXo\nhCRdom7UVoVm1uZqI9mN50MvXCn90p2KFXjF3VOi/GyBriBkqtsF8eRTDSC0IgTr\nkD2BfLjN0a8GH4s35ZP/DeSqVvru1nDsHgX7/i7FNuo5EMqru7x5h8mi30FUuc/N\nx5G8xjRrwOzWdVW3\n=+Nga\n-----END PGP MESSAGE-----",
|
||||
"XAttr": ""
|
||||
},
|
||||
"Want": {
|
||||
"Name": "rclone-golden-cli-20261112",
|
||||
"ModTime": "2026-11-12T12:07:52+01:00"
|
||||
},
|
||||
"Rekeyed": [
|
||||
"NodeKey: new key of the same algorithm, version and creation time, user ID golden \u003cgolden@example.invalid\u003e",
|
||||
"NameHash: made again with the new parent's hash key",
|
||||
"NodePassphrase: new content, encrypted with a new session key of the same kind",
|
||||
"NodePassphrase: session key re-wrapped in a v3 PKESK",
|
||||
"NodePassphrase: SEIPDv1 data packet encrypted again with the same parameters",
|
||||
"NodePassphraseSignature: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"Name: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"Name: session key re-wrapped in a v3 PKESK",
|
||||
"Name: SEIPDv1 data packet encrypted again with the same parameters",
|
||||
"SignatureEmail: replaced by golden@example.invalid",
|
||||
"NameSignatureEmail: replaced by golden@example.invalid",
|
||||
"NodeHashKey: v4 signature (type 0, SHA-512) by the node key of link-1 made again",
|
||||
"NodeHashKey: session key re-wrapped in a v3 PKESK",
|
||||
"NodeHashKey: SEIPDv1 data packet encrypted again with the same parameters"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ID": "link-2",
|
||||
"ParentID": "link-1",
|
||||
"File": {
|
||||
"Link": {
|
||||
"Name": "-----BEGIN PGP MESSAGE-----\n\nwV4DMsmw6Vp9F9oSAQdAC8NUD6JZTctrbyKmHE2Wq04F7sv9rfmdzYXRIVlb/Vsw\neP0G3IKAgTqjD1Y3NtBVaxgspQg8yRfdC9PWfdzPjOy+74RURel6yqBkqJeHrjr5\n0sBGARj9PCsxwdb17Sp4SZ9Wrb6dvmI7+j1a378XVh30TyoGfR3CEgMUorVNnHfS\naq2n3YNo9IiDY4RA7tXlPIcLp/3D7VP7c69UG8wJ4XdNnl/3AiBuZV01JGRPR54A\nfTyTo/SSwZ0pgerXc5a9nQg05qSS7vtXwnBqOsX00+RAuW82ORZGBpxwdt9rLWrN\nXTUsf9fW4coX8IaVzySPPnPrBaXVgBn4PJpOZWjn4Eswwj0O/LqyC/NuFKWxQsiO\n7XFm7P++QE/jUjPIR7mQiXXGvZ1xQRVlS+fkAEghPsbRp0JS3lbp3yd5d+CpsbHL\nirjGCxaqxUxb6KQfxm/11tMuCQJWSbGUYA==\n=4r6S\n-----END PGP MESSAGE-----",
|
||||
"NameHash": "72f3641700c6aecfd7ad80e9b96ab4cfb0060616b3e2fc6862ecce513fac0e56",
|
||||
"NodeKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavWdyxYJKwYBBAHaRw8BAQdAdCOugFgFFjUwF/ytzyfBuMFDIs/29rtHOMCZ\nKsvdqi3+BwMI8JBYANoiBO/gEGnJVRMzsuQWIorM8bgLjR7Nm9kzEkeKB3BcoiSp\nx7QpPBOxRoTxB4XQRLHL8/Up2zscGyLvfPovfD6E1DEPcIpmEmxF3s0fZ29sZGVu\nIDxnb2xkZW5AZXhhbXBsZS5pbnZhbGlkPsLACQQTFggAewWCavWdywMLCQcJEC7Q\npZiVfnssNRQAAAAAABwAEHNhbHRAbm90YXRpb25zLm9wZW5wZ3Bqcy5vcmex3hCs\nFunCVj9h8MhMF5/tAhUIAxYAAgIZAQKbAwIeCRYhBPiFzK9h2iWtxo6irC7QpZiV\nfnssCScJAwkCBwMHAgAA8iwBAOV2n+cJrVo2VqZxL53qs3wNfr6yG5A9I3ka9YC/\nlAgPAQCI6iujwnZJn1OCrA8mTpbGEQArrN6WCQPmXD4L9YfBC8eLBGr1ncsSCisG\nAQQBl1UBBQEBB0AJ5ah613aN4KShInv2+nfAqisfju6/IwFGjC2ujH9oYgMBCgn+\nBwMI8JBYANoiBO/gNcRTgHPCNOwQEC4pPqYAu/uikiAG+4xORaEzTMzqYjqSdby2\nKjKWyn5TJd1hu+SyRwK+ARwGl0+JpZ3HI8gqGJAuALS2E8KuBBgWCABgBYJq9Z3L\nCRAu0KWYlX57LDUUAAAAAAAcABBzYWx0QG5vdGF0aW9ucy5vcGVucGdwanMub3Jn\ncATLrFwrSqvvGRd64T/eLwKbDBYhBPiFzK9h2iWtxo6irC7QpZiVfnssAACGWAD9\nE2kjFlVk3Vs4yg+kuYKIUE8kHg7IG4d5UO9p6w8HmMAA/0Xm4LKfArpMYLZZ4quy\n7FQO0wsRYR+PTfZlq0k5hh8M\n=d3I3\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"NodePassphrase": "-----BEGIN PGP MESSAGE-----\n\nwV4DMsmw6Vp9F9oSAQdApoI55bosfVIC4/9wqXnk0rTugDAkNcoyQvZl5p3Imz8w\n7aRBsUUVHf4dFhVikzFS6vNSJo/MOEdrQkh55Hrfjg4SKwOk9g75WSS5bqfmZ1tq\n0l0BOkMCFdTfSjmr441C+tpxsngxizmbS3nUuemz6fULU7hyCM3UiwOE/ptdHNs1\nYge8H8zcyIbMANu2d5fZjKtt+NFjg6EzpN1BiXRK90GDIfXBfNRcDBD/ihCDDJY=\n=yXbX\n-----END PGP MESSAGE-----",
|
||||
"NodePassphraseSignature": "-----BEGIN PGP SIGNATURE-----\n\nwrsEABYKAG0Fgmr1ngcJEHcN5Kj1npDbRRQAAAAAABwAIHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmeJGtOOsHzCHLkeMyuJuZ9MgPR5j92NYzblAfD+vItUXhYh\nBCtnJ6qZmUV7euyTf3cN5Kj1npDbAAAXqQEAyPSvJwmcWmPh3ALvAtG4nmj+HXi0\n0o+Sa2JNXFTARl0BAP++uwOTI4IwDrAiRNnDqLSuT+W/o/i6oX8j2nzObSgC\n=MP/y\n-----END PGP SIGNATURE-----",
|
||||
"SignatureEmail": "golden@example.invalid",
|
||||
"NameSignatureEmail": "golden@example.invalid",
|
||||
"MediaType": "application/octet-stream",
|
||||
"ContentKeyPacket": "wWwGFQSiLbvpxpjyD4IUPRB30fGq9Ser+BIBB0AmevMprhlgl6kqYGJUfrRaNKPAkUpm0+rm7L+BRgO9EDAzLDfgFID6Ypo+rezGc847gzfBmRDEyiZYdvokI8IAoUppqtmiwoFS/Xopd7Hsges=",
|
||||
"ContentKeyPacketSignature": "-----BEGIN PGP SIGNATURE-----\n\nwrsEABYKAG0Fgmr1ngcJEC7QpZiVfnssRRQAAAAAABwAIHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmcNZBbuzvtGk35zqYkNcXBMlSdpeLIckRtI2q+kdxDBFRYh\nBPiFzK9h2iWtxo6irC7QpZiVfnssAAB4RQEAiZW9gbsfpAV77+1JuWP927dUJ11a\nyOEQnkjaybu/jnoA+wfvAz2s6hqbVyAd+ecvjuyllj8JP83SE3E7JqZgBqMC\n=C/L/\n-----END PGP SIGNATURE-----",
|
||||
"CreateTime": 1794481672,
|
||||
"ModifyTime": 1794481672
|
||||
},
|
||||
"Revision": {
|
||||
"ManifestSignature": "-----BEGIN PGP SIGNATURE-----\n\nwrsEABYKAG0Fgmr1ngcJEHcN5Kj1npDbRRQAAAAAABwAIHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmdvM1S08e1SN+ZV//Y5dNQcht1xOdpjBErb2ZgYH/KANhYh\nBCtnJ6qZmUV7euyTf3cN5Kj1npDbAADMigD/VH9yfR4nqu8FvGe6lSOYOypEkfZe\n74CISi7ZHUA1A1QA/REocDnh1/rTqnSmDsMhbwUuj7ZZYQ+mHxVA0ONdhFcE\n=qQIO\n-----END PGP SIGNATURE-----",
|
||||
"XAttr": "-----BEGIN PGP MESSAGE-----\n\nwV4Dd9HxqvUnq/gSAQdAyYIhzyUG5ibTJo901Pulc0VUNTfm8gk+DeWXyOcixyQw\nxvk6zyVnNcXfwwQ+Uydx8N5X6UMaz20im27EMcQAbVOAKor5CpwZ/x2yYwIFCfsF\n0sDqAUFV7q/wEp/VF37RbXhK7cPO4pnT2uqKKm4fxeboTaCvFO9aIVkpqtXqHcKo\nKU040hDdjXq2yQsJWPm3Qn3kHr728SqPf7CZ/zsJuaX+7U+8CFQrQxmPAemNBOPb\nEDNsOlHKYAsGb/Nk1NARiKTT8CLptYy/OJOUm8AkwnLKOcHF+OQ/Nuo7BQDI3FcS\nMHmFtqSV95Pur4g3VI7WRlYcWpUBRUCzIdp6nEPInenTJQy6MhpM519Iv0cA4ahn\nn1x8pGeKeaW2nQjGeGJwtAhSEgUirxaLr+2Hw7fgZgnnqLyeCfUaLKgzYv9t2HeF\nJq45eFYqVs8n0dClDEGTmfNDZVRqVaGI5YNaLaKvD1NtJwm3pRmhy4JuaBoEhfzi\nnWsGeKCjj6cIpUc0D2jtzH/Iwx/WfVRaEl6inl/DGwUwajTjRWce8lLEBx4hvBWV\n5dtTZRUfgZZko7yIO9qvEDfCVLOLKOevqcbPWkQozvQpg4PvTs0/q8Gks/lTHS1V\nv/zYwxOHAiAS1Pms5YtejmEN8sHzYrFmXw+sOsD89+ipro0E5vm37kqZaCAM\n=uMeP\n-----END PGP MESSAGE-----",
|
||||
"SignatureEmail": "golden@example.invalid",
|
||||
"CreateTime": 1794481672,
|
||||
"Blocks": null
|
||||
}
|
||||
},
|
||||
"Want": {
|
||||
"Name": "empty.bin",
|
||||
"ModTime": "2026-11-12T11:07:50.956Z",
|
||||
"MIMEType": "application/octet-stream",
|
||||
"SHA1": "da39a3ee5e6b4b0d3255bfef95601890afd80709"
|
||||
},
|
||||
"Rekeyed": [
|
||||
"NodeKey: new key of the same algorithm, version and creation time, user ID golden \u003cgolden@example.invalid\u003e",
|
||||
"NodePassphrase: new content, encrypted with a new session key of the same kind",
|
||||
"NodePassphrase: session key re-wrapped in a v3 PKESK",
|
||||
"NodePassphrase: SEIPDv1 data packet encrypted again with the same parameters",
|
||||
"NodePassphraseSignature: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"Name: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"Name: session key re-wrapped in a v3 PKESK",
|
||||
"Name: SEIPDv1 data packet encrypted again with the same parameters",
|
||||
"SignatureEmail: replaced by golden@example.invalid",
|
||||
"NameSignatureEmail: replaced by golden@example.invalid",
|
||||
"ContentKeyPacket: session key re-wrapped in a v6 PKESK",
|
||||
"ContentKeyPacketSignature: v4 signature (type 0, SHA-512) by the node key of link-2 made again",
|
||||
"XAttr: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"XAttr: session key re-wrapped in a v3 PKESK",
|
||||
"XAttr: SEIPDv1 data packet encrypted again with the same parameters",
|
||||
"ManifestSignature: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"Revision SignatureEmail: replaced by golden@example.invalid"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ID": "link-3",
|
||||
"ParentID": "link-1",
|
||||
"File": {
|
||||
"Link": {
|
||||
"Name": "-----BEGIN PGP MESSAGE-----\n\nwV4DMsmw6Vp9F9oSAQdA6bpTzVBpo8JalizlpwIsBtOtVZVfcLgnXu/4dWnYhSIw\nCU4GUOWxJ1i+ozntudEXmzeSdJD7shBW7eDFoBnVHo8pYfp8KM1erRcoNq0brVOE\n0sBJAZt3K7D0TpZxLLUiqBzstNWDST5MF9sr02UV2h8JcvGGu99nMC5ZrlqZBJC5\nuy/1BQEOtrGtr6ATuQsLIkW2PWI7jnvbnHu8rBAtSetAbu+Zyb4uUjyqbZaq5FQ3\nRlJFb6IQC8TmeFosaDaZQAVgguP0MrQjwXHH4/78tDGgHR/6/WkX72rdFsotsnQr\nHxvQtyIrTzx6Lk7g0YtUn+ZHA9pAkPiwKVAFrPtZHdp5jBY5Iy4C7BNSbUHrYV4j\nObs7eddNYcguC54rvtVOSOhaLD+xp4IJrcZKpK4IpdbLrNSI/hi29ZX6ONjZFCfJ\npl9E9cSnJD60sHx/ngnQGYZ/JYYVtMBG4JHHmg==\n=To2y\n-----END PGP MESSAGE-----",
|
||||
"NameHash": "e7a31618a78d64677ac080f54168dda356c7cb1f2d231edc8f8cce9b4a4bc253",
|
||||
"NodeKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavWdyxYJKwYBBAHaRw8BAQdAFB1g1dCu6PD/Mn8LoB7njYIleBah27bvBADu\nL5FaMVP+BwMI2Dol/MEZ7c/gljP8eUxe82HBP/XPfBzauKh88+SyOqMu8gbhmObp\nL6pR2xf9r5wMVguLq7QMJaW67lPanYTmCLcEVgoxAOv6RQaoRTHG380fZ29sZGVu\nIDxnb2xkZW5AZXhhbXBsZS5pbnZhbGlkPsLACQQTFggAewWCavWdywMLCQcJEKyb\nb8LzyuTSNRQAAAAAABwAEHNhbHRAbm90YXRpb25zLm9wZW5wZ3Bqcy5vcmeE9LcN\n6ntUZTHo9aGwb9+oAhUIAxYAAgIZAQKbAwIeCRYhBM8/XBQ0SNElXzIXlaybb8Lz\nyuTSCScJAwkCBwMHAgAAUjMA/i7fMPE9nkPvDsRyVKDZi4YBZmPK4L1m8GK5d8yH\n2w4gAP9vOIAeU1L3nusk/A6VDNiwhn/MfnX0m21/k1psk7auBseLBGr1ncsSCisG\nAQQBl1UBBQEBB0BevZ4EN2dnAPCK3H12SBQyc4Vt52RQ8nK6b+jCZMEJHQMBCgn+\nBwMI2Dol/MEZ7c/gLS3pXSeAFeDU98itjLM41L+ZmXIdNcOwu50/lP2B1yl5MLE2\nDSVUhlfj69D0FvRIqO3SmfjwxjUO5scH+2qhRE9q6ReDJcKuBBgWCABgBYJq9Z3L\nCRCsm2/C88rk0jUUAAAAAAAcABBzYWx0QG5vdGF0aW9ucy5vcGVucGdwanMub3Jn\n2+p/IrS9Q9C2af7/1sFBawKbDBYhBM8/XBQ0SNElXzIXlaybb8LzyuTSAACB0AD9\nE1nGtVpLRccSMMVT0iefsAuIS3wauoEjzta3GuOI8MsA/1ADWpHZVWAXoGNVoSal\naD81YwSLx7wMhj73AJhCiIEJ\n=6dAU\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"NodePassphrase": "-----BEGIN PGP MESSAGE-----\n\nwV4DMsmw6Vp9F9oSAQdASGjo4QmMRBPihm7jkfvxYSjXAGnnjLYWSY2Y5PTVL1gw\nnAgbUA6QjorukRNF7V3fxr8v20tKAtpN+PNesG6P2ESzgqz4r7jZiNY2z95jr0O/\n0l0BPnV+gIojUlLF9BZrc3p0gJVU0zkvCEHfw7f+WE8DXjryYc2hnAljDVqrMU1p\nEkFdEAr3sR9qo50SZfMz5E9K5wU0V5XkYPpIBudgFfiXF5pJnU47ookjnBCeGzE=\n=tZhO\n-----END PGP MESSAGE-----",
|
||||
"NodePassphraseSignature": "-----BEGIN PGP SIGNATURE-----\n\nwrsEABYKAG0Fgmr1ngcJEHcN5Kj1npDbRRQAAAAAABwAIHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmcSTfCei5wEKB6h88+ZbCFkKGhOSvFy5o37LdbQ29BkRRYh\nBCtnJ6qZmUV7euyTf3cN5Kj1npDbAABogQEAvWtAIE+LL3QDh64Z8k/M1Ooj9QfU\n4HQEfsW67FQTOocBAM0aHd4I1hCtEC9CDJWsGZsfcr+vXgDAKfC5xqVK7yAA\n=50U7\n-----END PGP SIGNATURE-----",
|
||||
"SignatureEmail": "golden@example.invalid",
|
||||
"NameSignatureEmail": "golden@example.invalid",
|
||||
"MediaType": "application/octet-stream",
|
||||
"ContentKeyPacket": "wWwGFQSFOo6NRa3E19B5MlVXqQGmn/ovHBIBB0A3WV6SsqClzY+tlbSWaHxbaL+X3vbC7NVqbg9PR73GRTDiTzh8r856XrWYU19ifpNzLg6TbrrKpmchEkWmNCsPrscWokAuJbe65UONo+H4CPg=",
|
||||
"ContentKeyPacketSignature": "-----BEGIN PGP SIGNATURE-----\n\nwrsEABYKAG0Fgmr1ngcJEKybb8LzyuTSRRQAAAAAABwAIHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmeiIgRSWLrcezV7ZLN/LBkrzQAWwfpDHLSBsEOS/BlgdhYh\nBM8/XBQ0SNElXzIXlaybb8LzyuTSAADGAwEAngc1us8FCirbV/9riXTyD+hszCpH\nQNP7ZcGJ5N0+UWIA/jFfDLhFYU4sDbhmdhRTkwITwHEmQUR5TycSJ/hnMQcL\n=Tbsv\n-----END PGP SIGNATURE-----",
|
||||
"CreateTime": 1794481672,
|
||||
"ModifyTime": 1794481674
|
||||
},
|
||||
"Revision": {
|
||||
"ManifestSignature": "-----BEGIN PGP SIGNATURE-----\n\nwrsEABYKAG0Fgmr1ngkJEHcN5Kj1npDbRRQAAAAAABwAIHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmeFgdD1dX/pQplrZLHAk2LC/LsMB2n8FGB+zMkWMBfAyRYh\nBCtnJ6qZmUV7euyTf3cN5Kj1npDbAAA2LAEA1V9x6x/nomWNygyD6b2GfbNJJPN8\nxrrBKzADDA/ZD70BALjrIVQfISsDBIlHEqzbyBzkA5rpdNJiFxEHKfgmYnwF\n=r/Me\n-----END PGP SIGNATURE-----",
|
||||
"XAttr": "-----BEGIN PGP MESSAGE-----\n\nwV4DV6kBpp/6LxwSAQdACsb9JyqVMyl3yQ+JSyZ4X0AN1gaxX3WUuEM4OMsCIWcw\nPZAq5x2aqL9DLInFUG4WFeT1BS8fpOz6ZSKkytmZDN7IFHFguGtfD+/jPB2oxyVK\n0sEHASB/55dbWy8bozmUxXvrQDlsLheJesNwIzTLFqTm+2MbW2lG3DGfPMa4gPJi\nAk/2JzwyngOVEjlR2C5VJfeDx6Di0HVjcA8kGu8hctFlESgXIiYBvVPUYi0Y8DSV\nApOIcO52uQupzz63LJrhjw/VQV+ijSvrWBzCvBO06xiRPZkItztuu9IpjTuJo1Zx\nAn3rAH1M7SWPkg91jMlWy8dfCvbXnecEedVGu9XVoS8Uq3wgXh9Yaj6Zp4ZkZFo2\ng/9dKJ4ZeWOShi4dC1wncp4wqa/XaM3hGbdw5kfSEucVSnWWNsqlH5HSYWZzmVpq\nY/IjfqHF3W9TtPx3ZEnTglRffCrkf0uEbBE7qCa6GZOD8cqLZG1LCmc78k76rLbk\nCFymOrvLY8qzfC4ASfbFvTUYqXjlmdMfw4j7eqaeFQeQs/VmpciWD77Us1BjMoV3\nzgRIEME6w0dwxOiqYUIkdwduTuwwhyv6P8icg3vCuW6CIdYshwjmdmW085uIATID\nfeErYQurs6+tLBBu+4d8nGoK/c0eGJKSQ8VJIJERSG/23pW/aOWCOSttKrOtkdtk\nIkhsArFrEbwMfHlfIcqzfmDUPSjn9NOOtFI=\n=hJwq\n-----END PGP MESSAGE-----",
|
||||
"SignatureEmail": "golden@example.invalid",
|
||||
"CreateTime": 1794481672,
|
||||
"Blocks": [
|
||||
{
|
||||
"Hash": "ctjxzA1hPKzU7YVJf8njbHChlhSnTb53l6BNXj47cko=",
|
||||
"EncSignature": "",
|
||||
"SignatureEmail": ""
|
||||
},
|
||||
{
|
||||
"Hash": "kW91o7X09+8AyzU9Yoiy4239epE1/1na8HOQt+OWL0c=",
|
||||
"EncSignature": "",
|
||||
"SignatureEmail": ""
|
||||
},
|
||||
{
|
||||
"Hash": "7xepALxxWiEfdwpYhj0MqoLTx/x2ciRgBbE1hNIDkUU=",
|
||||
"EncSignature": "",
|
||||
"SignatureEmail": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"Want": {
|
||||
"Name": "nine-mib.bin",
|
||||
"ModTime": "2026-11-12T11:07:50.956Z",
|
||||
"MIMEType": "application/octet-stream",
|
||||
"Size": 9437191,
|
||||
"SHA1": "39c27cf5bba5da8ccb5ab470efb380c9341c8cdc",
|
||||
"BlockSizes": [
|
||||
4194304,
|
||||
4194304,
|
||||
1048583
|
||||
],
|
||||
"BlockSizesRecorded": true
|
||||
},
|
||||
"Rekeyed": [
|
||||
"NodeKey: new key of the same algorithm, version and creation time, user ID golden \u003cgolden@example.invalid\u003e",
|
||||
"NodePassphrase: new content, encrypted with a new session key of the same kind",
|
||||
"NodePassphrase: session key re-wrapped in a v3 PKESK",
|
||||
"NodePassphrase: SEIPDv1 data packet encrypted again with the same parameters",
|
||||
"NodePassphraseSignature: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"Name: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"Name: session key re-wrapped in a v3 PKESK",
|
||||
"Name: SEIPDv1 data packet encrypted again with the same parameters",
|
||||
"SignatureEmail: replaced by golden@example.invalid",
|
||||
"NameSignatureEmail: replaced by golden@example.invalid",
|
||||
"blocks: 3 kept byte for byte; their signatures, which the backend doesn't read, left out",
|
||||
"ContentKeyPacket: session key re-wrapped in a v6 PKESK",
|
||||
"ContentKeyPacketSignature: v4 signature (type 0, SHA-512) by the node key of link-3 made again",
|
||||
"XAttr: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"XAttr: session key re-wrapped in a v3 PKESK",
|
||||
"XAttr: SEIPDv1 data packet encrypted again with the same parameters",
|
||||
"ManifestSignature: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"Revision SignatureEmail: replaced by golden@example.invalid"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ID": "link-4",
|
||||
"ParentID": "link-1",
|
||||
"File": {
|
||||
"Link": {
|
||||
"Name": "-----BEGIN PGP MESSAGE-----\n\nwV4DMsmw6Vp9F9oSAQdAMatEYS7L8qGawYLZwFMD8H+KfbmuaDr02rCfJTyx5V0w\nVmugXUagwAqbjry7GWzUEAvosBwVZo+YBGtECBRA6lmsGVZJ9U3EPbnZ1zffOBx7\n0sBGAQIe2DEfIoKGkx7UUIXk2Fs0m0yvFWfofIFfw0juXOOAdUCOCt2kX0tfxKjD\nfdN140hLRwJIiJjqqEDogjYyBvM6Kk4cweFOCCaf0Msu62NS5y9lex3bN3sEDkSS\nErIIpKV4pZzFmfS67io8eJs8SlkBZhQvzxYhq1tF8P930mzV6uTw5tRJUD3vDTbt\niWfUoufLWULn71u7qHSY5FYnrUjsPiogyth4L8NGbz4UpzoQtxHMACTKm2gne0fX\nRWUbosbH9ksi8o9nc92LzyfbbqRJ4OMl4/51D+wqTdoURUL5zMbRVmJtrdWH2cOT\ncsVHsw7VgLnxXQftY8x/WE36GKzr/5LMCw==\n=+FHb\n-----END PGP MESSAGE-----",
|
||||
"NameHash": "772cc26af9940f978b18ec02b07a9c7da7f73ac84439891b163a9fcbcfb7d2e3",
|
||||
"NodeKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavWdzBYJKwYBBAHaRw8BAQdAU8+czT99s/PiAmGDbdOh4RFs3KJOIbgQl2tW\nQRLk34D+BwMIkIekrECs5ejgDdTzu42p1A1BXxnKWpLZ7hfynmTIyqRZoU89tX+F\nJMY0Vp5rWJ84KZz5n9HG6BGu2mFP1GEXXpAzwS6ktCzQzB01z9d+ns0fZ29sZGVu\nIDxnb2xkZW5AZXhhbXBsZS5pbnZhbGlkPsLACQQTFggAewWCavWdzAMLCQcJEPyh\n46k0I/1ZNRQAAAAAABwAEHNhbHRAbm90YXRpb25zLm9wZW5wZ3Bqcy5vcmd+aRLq\n6r5IG60Q9cNERKUIAhUIAxYAAgIZAQKbAwIeCRYhBA3/cBz48qmSx92IIvyh46k0\nI/1ZCScJAwkCBwMHAgAAhTsBANPEJukaVTA1AZ5O5jQsYctfASgtxBbEbH1UidKh\n4UNqAP4jmCZVfDIfYo9uWgrycf7KplnR/FYEYq6RNjaU8Pn7C8eLBGr1ncwSCisG\nAQQBl1UBBQEBB0B4QMQnvDn/A1SFeHHYj3Gw1edvBiNvTEvEdZCCOLT7fgMBCgn+\nBwMIkIekrECs5ejglJAviEzcqBj5oi/aqYe+r9or0FKs0ujXHpGHRXe8gKcHR1bc\ncTWx2+4vUrcnkkBKLtvy/1Ez04LNAew9szh2zT6SGYR/pMKuBBgWCABgBYJq9Z3M\nCRD8oeOpNCP9WTUUAAAAAAAcABBzYWx0QG5vdGF0aW9ucy5vcGVucGdwanMub3Jn\ngC6tjIoMHKaTSbAsI8uZ6QKbDBYhBA3/cBz48qmSx92IIvyh46k0I/1ZAADZGAEA\n6d4TRAYkTg1t7hhrhXzZjsIPmwAfuI9Z5qdzv+7Nlj0BAJYvIK160ezWzkztIWcD\nwbaBgnUJFv4WECLUbHzec1YL\n=zexJ\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"NodePassphrase": "-----BEGIN PGP MESSAGE-----\n\nwV4DMsmw6Vp9F9oSAQdAsqsxYzkqdFBtM4W3SWv35+OZhYLWUQ+Vk9Ep7rc0MWow\nDHToM4ACfia0j+uUtDrCyLrEXAvGb2xsnacLTVAOsraRvhL6YqitioRScep9e8BX\n0l0Bs0ORRXGA7TMFM0Iq/D7oyrXyq4pyQz9qHzs7kcvn5BIbYZCjH4caEY5w/ZI7\nj4UK5He8yXzb7hRRxXc25eXWdj2W0rrC+Qoa3HCc02/+kzkrAcionOeZ/+n8yFU=\n=0yXl\n-----END PGP MESSAGE-----",
|
||||
"NodePassphraseSignature": "-----BEGIN PGP SIGNATURE-----\n\nwrsEABYKAG0Fgmr1nggJEHcN5Kj1npDbRRQAAAAAABwAIHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmfmbJhr/w7RyB4hQQd+HoJstxGtWBh8mqSrIM1GxjYBzxYh\nBCtnJ6qZmUV7euyTf3cN5Kj1npDbAAApKgEA/zZjImgQoaRzrux89tYdB7OfHbWt\n3BWrNYiNqUSdQhgBALNYI4DeNsuTj34UzMQvIycsQjWSlu9v1kdBQlPvgC0A\n=bSjE\n-----END PGP SIGNATURE-----",
|
||||
"SignatureEmail": "golden@example.invalid",
|
||||
"NameSignatureEmail": "golden@example.invalid",
|
||||
"MediaType": "text/plain;charset=utf-8",
|
||||
"ContentKeyPacket": "wWwGFQQYYwxWuRINPXIAcr+F5z029VeY8hIBB0B+pOsmx0uPlIxhKXGTqhhPLT0bZ4//C5/V27Ce7p0UYTCL48dlmkLSV6woofxIJChzaop1FSWkewF1Bw/K7H1C1zsMSdzD4dWqAjdssFG8nDU=",
|
||||
"ContentKeyPacketSignature": "-----BEGIN PGP SIGNATURE-----\n\nwrsEABYKAG0Fgmr1nggJEPyh46k0I/1ZRRQAAAAAABwAIHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmfE3hfEg2GB0fFrL3aLDLUGREWbmth7hdWwqmoF1D1jiRYh\nBA3/cBz48qmSx92IIvyh46k0I/1ZAAB/pgEA8dS8Vp8SBLjxDL1/XB9jPmeVaLNh\n6kcK8uU2UecAeUQBALYgscqL8cgwc8BHsVtAlb2+NFqVifHjc7zfDJeXGjgB\n=CeY5\n-----END PGP SIGNATURE-----",
|
||||
"CreateTime": 1794481672,
|
||||
"ModifyTime": 1794481672
|
||||
},
|
||||
"Revision": {
|
||||
"ManifestSignature": "-----BEGIN PGP SIGNATURE-----\n\nwrsEABYKAG0Fgmr1nggJEHcN5Kj1npDbRRQAAAAAABwAIHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmecmke39tT3ONOthgxo96qDGxiHb3vkBRPlxtHN5iOy2RYh\nBCtnJ6qZmUV7euyTf3cN5Kj1npDbAAAG/wD/W8IIW27DjBGUoLthKiJO03pd1i07\nKWUZnYtOIhGnLrMA/jCmr9E672YvBAt+t9244IeMl0ZyerrFLJT9eyj0Q0sM\n=bSpp\n-----END PGP SIGNATURE-----",
|
||||
"XAttr": "-----BEGIN PGP MESSAGE-----\n\nwV4Dhec9NvVXmPISAQdAnJzy+Z4EYv/zgHK2S94AMFhN1clER60mR1TzSdISpwYw\n7UAb9yqA98CN6PRHZFMGQ8yZrhNJn0jXCCwQSKGktFGT3mZkt9nM9RCPs1bBjbDF\n0sDtAauvyEydUBOmX28hYzec9wxVTyhLvanmYhybdEkrmlYBxxWWHa1Ti8jxpvFt\nneGccgk0jonYZA6qrUMTSCFq+TngnMBWGEW3SHJl2xCDFsMzNrVkJED5mtljvxh5\nHy2ic9UW8ww6HAgQ2Hcmw49u5jqyjQQZJzqxFeWRp4+Yk/NExy3Y/zQYPj9irSb0\nkaNfun9jvJcMawOV+OrjFzj+aMwLZXHk9MDhZ8XRrHRnr9yOMGJE0S6K2akfvWXv\nGZM9NZqo/pjrzQzBDm5k413Hnau0QxK/jsO1SnooWIX9sMc3DieWfHeIxSPtfK9M\nsOkkYWX2WcLzs/+UdwmTnX9KOqUfTormRpC+lv75UptkFFrTx7HnniG5/7c4TN8k\n5NQfc1V97V8/ZAn1ByzNuLL9I8edIXitu/MDHFoPu/lbXBJhnf+/lMV2E/Tfzacp\nzSqetfzbJGvNxFlTCcPqdhPy8kALHkSNdsXuo/8YacfiAiiGk6VTczjLDeg/pi4A\nvHTDnPATn1w9NDE7jd2Jnt8NSVwlGc0fMUECqyJ5r7/X8GJl8OPIVts3TND567U6\n=xB9z\n-----END PGP MESSAGE-----",
|
||||
"SignatureEmail": "golden@example.invalid",
|
||||
"CreateTime": 1794481672,
|
||||
"Blocks": [
|
||||
{
|
||||
"Hash": "bk2FU1Wwg2aV4sRH3Ziam+HJZ/lTXVE4Z2FGh4MJIGA=",
|
||||
"EncSignature": "",
|
||||
"SignatureEmail": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"Want": {
|
||||
"Name": "small.txt",
|
||||
"ModTime": "2026-11-12T11:07:50.956Z",
|
||||
"MIMEType": "text/plain;charset=utf-8",
|
||||
"Size": 71,
|
||||
"SHA1": "4f488768897aa1fb85991dfa584e8818bdcd812f",
|
||||
"BlockSizes": [
|
||||
71
|
||||
],
|
||||
"BlockSizesRecorded": true
|
||||
},
|
||||
"Rekeyed": [
|
||||
"NodeKey: new key of the same algorithm, version and creation time, user ID golden \u003cgolden@example.invalid\u003e",
|
||||
"NodePassphrase: new content, encrypted with a new session key of the same kind",
|
||||
"NodePassphrase: session key re-wrapped in a v3 PKESK",
|
||||
"NodePassphrase: SEIPDv1 data packet encrypted again with the same parameters",
|
||||
"NodePassphraseSignature: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"Name: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"Name: session key re-wrapped in a v3 PKESK",
|
||||
"Name: SEIPDv1 data packet encrypted again with the same parameters",
|
||||
"SignatureEmail: replaced by golden@example.invalid",
|
||||
"NameSignatureEmail: replaced by golden@example.invalid",
|
||||
"blocks: 1 kept byte for byte; their signatures, which the backend doesn't read, left out",
|
||||
"ContentKeyPacket: session key re-wrapped in a v6 PKESK",
|
||||
"ContentKeyPacketSignature: v4 signature (type 0, SHA-512) by the node key of link-4 made again",
|
||||
"XAttr: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"XAttr: session key re-wrapped in a v3 PKESK",
|
||||
"XAttr: SEIPDv1 data packet encrypted again with the same parameters",
|
||||
"ManifestSignature: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"Revision SignatureEmail: replaced by golden@example.invalid"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"Comment": "Keys made by TestGoldenCapture for the tests alone.",
|
||||
"Email": "golden@example.invalid",
|
||||
"Keys": {
|
||||
"AddressKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavQ+PBYJKwYBBAHaRw8BAQdAsOznLcXFVFSRVUaH3qo5rys3OiRQwbtMZdLS\nafRdETH+BwMIFGmRbNl/t5Dgc7IAKj9uR/Qp+0d6UvLObm2j7uFmY/JiT0TmzGcy\n0z28V5VeJXxPKS2mXIgHGcHFthM/6oir0rD2eOaAjEqvCEH1NL7nbc0fZ29sZGVu\nIDxnb2xkZW5AZXhhbXBsZS5pbnZhbGlkPsK/BBMWCABxBYJq9D48AwsJBwkQX5SX\niFgESiA1FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ8LZnPyP\n6B8T/JtLAxTd1MgCFQgDFgACAhkBApsDAh4BFiEEj06tdhNZBtSdxQPxX5SXiFgE\nSiAAADRPAQCSqkVAJQCYqqDwUcve+Ic2JDY6FqI5NhTYa9oNwuvJfAEA8/U4+QSf\n3L7m17fNVodJuqXyLtGAXj4R2s8NUrZL0Q7HiwRq9D48EgorBgEEAZdVAQUBAQdA\nqwqDKwlcjpTOMzdB8H2D4D7VVfLYXvr9CAZEW024DlUDAQoJ/gcDCBRpkWzZf7eQ\n4Dm5B4j9mEhDfCeUTa7FcfESDm9gQmL/bcqO2a6tYS/nFfnSYMnAziUMrotnPeFV\n58WTjD91PYdWnfuXirjFmtjXNnTEW2XCrgQYFggAYAWCavQ+PAkQX5SXiFgESiA1\nFAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ5GqVSI3uM+AyGGY\nFO72yw0CmwwWIQSPTq12E1kG1J3FA/FflJeIWARKIAAAvjcBAIwoz2rGW5meAhUa\n+ysq/MqVPYtygk6z9tlIGvhrJxCNAP4vrKesIguDbL5zHf5/siiv2UQh41tt5Nqk\nWrqKioV5Cw==\n=xp0s\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"AddressKeyPassphrase": "nvWWe/o5p5ieeEByEqGfuRaJNXXfteZrJr22WkqvHko=",
|
||||
"RootKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavRo/BYJKwYBBAHaRw8BAQdARj1hfxg3SL9faiG9GeT82C/3WDbFk3yY2Jv9\n/zo57v7+BwMIcIdQScQjSPLgwNxM8qtWy98Iz8EXDAbvYf8+3jI1ZR4E8jwAYUJd\nJwTC4EUWdvlJ/k3X0liLYJHC5GaGpJgYzyZYs0P/6e9OhZ4RT21UbM0fZ29sZGVu\nIDxnb2xkZW5AZXhhbXBsZS5pbnZhbGlkPsK/BBMWCABxBYJq9Gj8AwsJBwkQ5PnT\nllRvlt01FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ7G9xJ/j\nKCxEgBDA5b6TE50CFQgDFgACAhkBApsDAh4BFiEEHenU9PEZupJFwO4z5PnTllRv\nlt0AAEYWAPwM8n4hqlRVGzr6ho8dDBwmMe3gsbiHoXKHY+Tp6iDPvwD+P7YG7dCi\n3ImNbPWSFdXCFqCaf9Rfuko5koMaKDuFzATHiwRq9Gj8EgorBgEEAZdVAQUBAQdA\nzNzrB5jKURbG8jFlkAST0Ywn+ihk4Qb5nAGc6WIYPE4DAQoJ/gcDCHCHUEnEI0jy\n4Pe93bTMUt0jLmP66nC1c/ac4da9lNq6WZq1wdgrLtnqeLvbdGUyD/Ysojm21nj2\naiiGU3WY1pMs2EoZ68CYtEibR4uP7N/CrgQYFggAYAWCavRo/AkQ5PnTllRvlt01\nFAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ2ORKI/eh/s517M+\nvIf9MBQCmwwWIQQd6dT08Rm6kkXA7jPk+dOWVG+W3QAAe2sBAMQUH2VxtEvuQe/5\n/YIbsUn0GzsEQFQ7rAZg1ksrn5B3AP429RYv8Q4eUuta61QO67hWU8kERgP2PcPF\nGOvTBjMGCA==\n=vu31\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"RootKeyPassphrase": "kmPagbeact/lbELQyb2f/FUKXWTEdulQkKuOhBCit4Y=",
|
||||
"RootHashKey": "-----BEGIN PGP MESSAGE-----\n\nwV4DGPXs2ttF5mkSAQdAGe77n4ZD663wlp6uqrVAJFT3FSJdA+zwVlbhr3XFsyww\noDJkCuU7wtIi/0a4KM+TVAnr3/JY62FzZIkKmx/rQ5TADRyqBFPt0eDir3IVny5B\n0sBZAbY2xnC/zccUXQYVLDWYdlP7lKkRTMyNmlXtecKiFIwYfCS6ygYTkKFhMkSr\nL4relY6ZhEa1pKMRliLOi88G8NctEi9vtLdNAHdhuElnrenBHaOz6lHE59I+0t3n\nuM9StK1FpAZVG4k+s9cxq16ywqQt3lVGrbv1Gjsp646IgbN+q/ftYGeRgbbbv8CJ\nz7IxWGJRYZvL8AOo4YfSerFVM7qCDy2XXLgPj1Vr71xVFrOi6p6mjJll/h+g/gOH\nmhOEAE+LFr9zRG19lJHFK9j8POS0W64FyPX/B1GjewlFYskw8bwI78BBJfaxkLHk\nlEUV1BYumWS/KI39bhEUYAYy7FgLVKEdSbwXIpCjv+xVSFzQRkaPBbrykrM=\n=Wj5Y\n-----END PGP MESSAGE-----"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
�� c�ζζF
|
||||
k ¤σρλRp�Θ¨Άπ*Ϊαϋπ`η»:y*ρ¬HξΗ;η\ΝΏώ‡¬
z•Rx9ΑΊ‘5fx~Ρnϊn$—�PΔΫΦ›ayΩ›q1tyΉ*R΄Κρ#ΑzΪ½h`NfΣ)<m―t ΅eu"Θ…H�TΈ–β΅bVΙSΊQ› Ό™'#
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
{
|
||||
"Comment": "A folder tree web wrote, captured by TestGoldenCapture: the encrypted data is the client's, its key packets re-wrapped and its signatures re-made with the keys in keys.json.",
|
||||
"Label": "web",
|
||||
"Nodes": [
|
||||
{
|
||||
"ID": "link-1",
|
||||
"ParentID": "",
|
||||
"Folder": {
|
||||
"Link": {
|
||||
"Name": "-----BEGIN PGP MESSAGE-----\n\nwV4DGPXs2ttF5mkSAQdAjv1Py6YKv0xa5ntdhDuvyA56877CUfUHPdoeMwVqCgww\n9UB4yaN2RAjz1vD1ESa1prPpSn9LSjLz7BUrlBb12Mzzoc8paxu5umDyZSWNc77W\n0sBNAcwBFLQ/IFxgy6PFCKmFKPYawniODwUxsqCQRtlUF7lqqE09AE8bmYFIzXYH\nkAvm9Vnnd6CixTGgnpXVCWPxC/fO5CCBdIIiF0iqp6ZYbKSBNBy/jmxLwLTchvT3\ncyXB6FJrmUXDwnjgwB8lLWIc1aObtHGr/NyYb5bpOeb+b9LNYC9pCkOsXv4vJCO4\nQ/FdjRM0IM0sv1e93VtLKghsZHrceFEgQIBV5ayXP4tC9BRlxGCc4tEJN2M/PIpj\nXGyZKTafT8FF+O3MpFpN1/kfSxubc5B5zYHBxxXmeVzaBXxPWw9DA130jwE6wADO\naUUdFNXqn5Xv6ThDk5KoNsvGTC6o/Tf/om6vsKfvOpA=\n=1iDU\n-----END PGP MESSAGE-----",
|
||||
"NameHash": "1aec699a25b29cd854349f4db51025cdf07194ccda6c9414f7427f13cd6e1de0",
|
||||
"NodeKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavRqERYJKwYBBAHaRw8BAQdApOz9j+Vp6dGa1yghnTOtN1ms3D6GLwLd8Zog\noTuYsKL+BwMINkkhYmrV4DPgjtWLpEWq1gff2SZPpRew+I14+0Zh7xjKJ0TmViWj\nXoV0HiYrL6LvuFSO9i95AkbXFMWHzTskfm7LBBUyDldbz73ODyZUOM0fZ29sZGVu\nIDxnb2xkZW5AZXhhbXBsZS5pbnZhbGlkPsK/BBMWCABxBYJq9GoRAwsJBwkQMc+V\nVZR3yyU1FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ6z28b+s\n05/5BfqzTmK+hFcCFQgDFgACAhkBApsDAh4BFiEEnCbcE//QVcCZRXlkMc+VVZR3\nyyUAABJ9AQDeUi4H3RnjAc2V7tKzmCmP5bnIVqHMporcfapOMdppHAEAk83Y/000\njeArOoHiw/yLWuOnIchbQBap8Du2BU8OpQPHiwRq9GoREgorBgEEAZdVAQUBAQdA\nqk0yrljf9PLOdC4AsbRf6wwsE5N6uyOaIa11kVsruUYDAQoJ/gcDCDZJIWJq1eAz\n4B0P5nOHDqrAzZlNlCnlW5Hmz8DO1iQ5wvIODoOHPEWSMJ70ncJbsoVBJmajpF2z\nGBDMOfCRT3JICwcXNL6OhwD/P4z1pkbCrgQYFggAYAWCavRqEQkQMc+VVZR3yyU1\nFAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZy30fllhx7yxyeRo\ndJqI77ECmwwWIQScJtwT/9BVwJlFeWQxz5VVlHfLJQAAXjgA/0eHPltWW547814o\nTx9CTdbJsFeDNZbStEfm0oHTnoeCAQCkE1Pr1iPaxP9zw1S/56/8/ydPcyUPp9xi\noUz7K7yqBw==\n=MB90\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"NodePassphrase": "-----BEGIN PGP MESSAGE-----\n\nwV4DGPXs2ttF5mkSAQdA0kxuDcRRLu8Bg3XLntuMzUncod5R9xhJjeJQlspp5jww\nE/6creXAEmBzKRvZ0seescMLBrXpmFm1S13Yi1iWYmTDzDg3S75/akYFSfMha7Ga\n0l0Bbwyqclv2qR7M0VrNp5iKoVMmEmKeSVSyq7ALpzfxlAZHlmJNjkZrlARfN3J7\nsEOSNVWMHdCGKv9yqxFZ3KFRkLS3pCpXcXa4lWxNK/fosPWBqQritq4lzDVSM98=\n=F187\n-----END PGP MESSAGE-----",
|
||||
"NodePassphraseSignature": "-----BEGIN PGP SIGNATURE-----\n\nwrsEABYKAG0Fgmr0ak0JEF+Ul4hYBEogRRQAAAAAABwAIHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmeznVHeSr/8EIU9NVqF2GqTqO86HqAISug+35Ab7EghLRYh\nBI9OrXYTWQbUncUD8V+Ul4hYBEogAACmnAEAzq9+/mf78Tu5V7MjlLIe9QpKqPGV\nrm0yD6L1ekPPuXgA/jDlNFWAFtwjpunPlf0lfrJthuufPWw0B32CvjWiRo8E\n=TObr\n-----END PGP SIGNATURE-----",
|
||||
"SignatureEmail": "golden@example.invalid",
|
||||
"NameSignatureEmail": "golden@example.invalid",
|
||||
"MediaType": "",
|
||||
"ContentKeyPacket": "",
|
||||
"ContentKeyPacketSignature": "",
|
||||
"CreateTime": 1794402893,
|
||||
"ModifyTime": 1794402893
|
||||
},
|
||||
"HashKey": "-----BEGIN PGP MESSAGE-----\n\nwV4DjeI2jIiMS9YSAQdACQiO66XunmdiAbV7WNlJEKdP6M4YgtNyc0By/PCp7x0w\ntGhMU+1XIOClM/HKrf8l81iXi8QieBQSygwlzgdjnclMROKMWmjXSrti0+qWWsbu\n0sBpAUK5+31iO0JfNo6p69pEc3qaRxB+yws1fxq63MaCDxHFyExVjJdfWdaPeyov\n8xmcUXX8Mi0aRLB2aBkqdVfuqQjaKzrG7RRnbf4J9r+gSehSW7rho2YmykRfmP7z\nk4F/gEXdIZKhAvHznWIgMgkRZeYQSnF2/HEUxW7c2RK9XOhYhPEGK4RIKMf9O2qo\nJl614p+WXlTjAN+hH7aB02/ObPPosXc80vH9GGkaXWnEtN1qkryHXOtODsoQwl47\nv8+xF823U3kWx4kM0Ohhgi7DpMRDW/ERe0phbgpDhX46/jbzkf9Ikb80Xtbsk3MW\nRoHsQIL3KJKrzfAdzSNAptmcIFPDaWY91i33YqN8BctqwJbVLaCywXjC+Rmrirb+\n+S6hbNthHRSNyeBU\n=kXPH\n-----END PGP MESSAGE-----",
|
||||
"XAttr": ""
|
||||
},
|
||||
"Want": {
|
||||
"Name": "my-little-folder",
|
||||
"ModTime": "2026-11-11T14:14:53+01:00"
|
||||
},
|
||||
"Rekeyed": [
|
||||
"NodeKey: new key of the same algorithm, version and creation time, user ID golden \u003cgolden@example.invalid\u003e",
|
||||
"NameHash: made again with the new parent's hash key",
|
||||
"NodePassphrase: new content, encrypted with a new session key of the same kind",
|
||||
"NodePassphrase: session key re-wrapped in a v3 PKESK",
|
||||
"NodePassphrase: SEIPDv1 data packet encrypted again with the same parameters",
|
||||
"NodePassphraseSignature: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"Name: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"Name: session key re-wrapped in a v3 PKESK",
|
||||
"Name: SEIPDv1 data packet encrypted again with the same parameters",
|
||||
"SignatureEmail: replaced by golden@example.invalid",
|
||||
"NameSignatureEmail: replaced by golden@example.invalid",
|
||||
"NodeHashKey: v4 signature (type 0, SHA-512) by the node key of link-1 made again",
|
||||
"NodeHashKey: session key re-wrapped in a v3 PKESK",
|
||||
"NodeHashKey: SEIPDv1 data packet encrypted again with the same parameters"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ID": "link-2",
|
||||
"ParentID": "link-1",
|
||||
"File": {
|
||||
"Link": {
|
||||
"Name": "-----BEGIN PGP MESSAGE-----\n\nwV4DjeI2jIiMS9YSAQdAy1RT5JmyoKbu4zexRT/rd2BYJGGx20ua4lrLmwWKEiQw\nAuaD9uF446+/EeHj312E63yjCnpu+K2Vg7scA3tb0PKMbhxIe3YVBtCXsdG8S17G\n0sBHAelTk0SGOENvXeE0XIFwi0j0ys1ao6GZvtmFZnUddAiEMhPNiiqYySzTqanO\n8kOBGOjTz9ZtcZFFKepn/smOlF6+w55SM/j/wZzJCD0FMfhESOx7ld2eJpDPBU3q\nTHwLAPdtBWQvSygmR4IITQvgbGzTAFkazgWvjfb+b6AIIjrSFOZZ6MVPMetuemBY\n6zd1JSDwWgunoTeLaNV1aCq2fiU0L8u5g5gmXIfV8UnIhbra7WlmBuIe5jirAqYK\nZ2/FXKPGcLcHYFAu/A5/Auz0uVB8DEtXnhOWK+DScHQkMrDjFDoeXkhWVUjZ84xG\nNHXvEXtA+Q0D4axWSNsrhVDQYjKLlg0/UuE=\n=cK1p\n-----END PGP MESSAGE-----",
|
||||
"NameHash": "ee2f7816bd5d57c533c1aa90f3615aaa3726c5b1cd110b690cddfacaacc4f225",
|
||||
"NodeKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavRqQRYJKwYBBAHaRw8BAQdAaI8f/bJhfDpJk3J0qbrCcD8wYykylO+m7a5H\nbK3RDwv+BwMIye2S3scpSijgQSXYF6wfIVPIgPPSGYdPnZ8p3lUUsBNerdSHJ3P9\nUgzjlRHjiczay4LSk2PIR4gKRgVWfhobzqGBfFHFUjohY1PNxTrd680fZ29sZGVu\nIDxnb2xkZW5AZXhhbXBsZS5pbnZhbGlkPsLACQQTFggAewWCavRqQQMLCQcJEOB8\n/myMSsIPNRQAAAAAABwAEHNhbHRAbm90YXRpb25zLm9wZW5wZ3Bqcy5vcmf0Pg5o\necNTvKUK8/ZbQvB8AhUIAxYAAgIZAQKbAwIeCRYhBM3CCbHOgC83pg0vYeB8/myM\nSsIPCScJAwkCBwMHAgAAC1cA/3TzPE0bdWk4US0Ha7EsWykVDunT3uUHYjXUtpB+\nWqHFAQC3ZGml+y2s/+RD2Qc8mWt4DHgvf2Gw4XNwJnv4SCK3CseLBGr0akESCisG\nAQQBl1UBBQEBB0D9eG5JZddZAPxajMLZOTOlNI0rm8b+zsVVLwkQ0YrtdAMBCgn+\nBwMIye2S3scpSijg7hO/JcviyPE3zBM/Pu2HGw/KrfLgBNBAmjHG0aHtw/MhfgTD\nUv6Co/lIP9adULX5cY8Z17mWncFMWq4yQ/5JQvy7CFSDb8KuBBgWCABgBYJq9GpB\nCRDgfP5sjErCDzUUAAAAAAAcABBzYWx0QG5vdGF0aW9ucy5vcGVucGdwanMub3Jn\nmBRN5YKg7s4iFcbEuJF/gAKbDBYhBM3CCbHOgC83pg0vYeB8/myMSsIPAACGXQEA\noI1IZQhRj9Wh2SzkZjJEtJGC5xqj3uInHlVVFWYxJ2YA/j7ro/WmoEWYnlyE9qLF\nJyQTBATXKJ3aX646ZvgqpAEE\n=JQ9D\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"NodePassphrase": "-----BEGIN PGP MESSAGE-----\n\nwV4DjeI2jIiMS9YSAQdAmZFBcoGy1Uzu5EmfYuCxatQcokQJqynRhNLFBN0/Tgkw\nwcOiKRAjqagoG3E29eMEY3vZLXdNUk1eelCeJiMkimqH6lLZY6o+I0G7Y2Le/dLI\n0l0BkWJMed4QEQYv22v13gcq3vthwHeIaJsTQZspbGBiFaVGLO5C5BD65PgHQ4z1\npVXMDrPIu3gaZPo9VVLUDtXVfxMX2Vtyp+D72dzC1g0IvVdkaLN8voC9AwRTHJA=\n=cYbS\n-----END PGP MESSAGE-----",
|
||||
"NodePassphraseSignature": "-----BEGIN PGP SIGNATURE-----\n\nwrsEABYKAG0Fgmr0an0JEF+Ul4hYBEogRRQAAAAAABwAIHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmcicLjwV6D2L3nubZLhNwumrSX361A7wRc+UsACfqsHExYh\nBI9OrXYTWQbUncUD8V+Ul4hYBEogAAB5SAD/a4BS8ZvfVyAOz73LsImMuXh6PU5w\nqpdypL3wv9wYcXsBAMPhamPNlQ+CJXj7ZKdBKkoyYxLHNZYxc0ZNb15YLXAE\n=UrGq\n-----END PGP SIGNATURE-----",
|
||||
"SignatureEmail": "golden@example.invalid",
|
||||
"NameSignatureEmail": "golden@example.invalid",
|
||||
"MediaType": "text/plain",
|
||||
"ContentKeyPacket": "wWwGFQSnLIQTXgUA95Q6F2xR3klEr6+aMRIBB0DLVgdDCcF6Ddum3oqXgZwJ4p8s5jspwe6NRAjmV242ejDFByn2QUAxrBm+Y8vXVUOdVHjpFxxgopgmw1lblDYI7vc7EyZtn7t7KiqIovyGwrc=",
|
||||
"ContentKeyPacketSignature": "-----BEGIN PGP SIGNATURE-----\n\nwrsEABYKAG0Fgmr0an0JEOB8/myMSsIPRRQAAAAAABwAIHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmct5vykzPe19AeIij8SC7dccQgTXdnD7TqVwj+xE0i3bRYh\nBM3CCbHOgC83pg0vYeB8/myMSsIPAACUbgEAyagfoI1Q1aAki2aLeT0rGJ7oNi0G\nUk5aDoT8ZFcwiEoA/RPVX+Gm9I6EPb2s2hKDgDLcwyXlU0hr0ai5SlwA4hQA\n=Tyi4\n-----END PGP SIGNATURE-----",
|
||||
"CreateTime": 1794402952,
|
||||
"ModifyTime": 1794402952
|
||||
},
|
||||
"Revision": {
|
||||
"ManifestSignature": "-----BEGIN PGP SIGNATURE-----\n\nwrsEABYKAG0Fgmr0an0JEF+Ul4hYBEogRRQAAAAAABwAIHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmcsYbIW3DX4VBgYltfHcjBIgN3fhkqhKU1qr7YtGTqiMBYh\nBI9OrXYTWQbUncUD8V+Ul4hYBEogAAAyCwEA+EBCXpCtPqEHJp0Jlbx8PCa2Uh28\ntkXfiyCU0wu+45oBANSQUFHucAq0UqspD6wHFcCBvE0jeZXbOQPcopr1iC0H\n=IYeK\n-----END PGP SIGNATURE-----",
|
||||
"XAttr": "-----BEGIN PGP MESSAGE-----\n\nwV4DUd5JRK+vmjESAQdArd0mXTdoTVeffGXSbD/J/VmeZalNaabquXle71t/yhIw\nLZg6EOZu5AZ8PbzzrhgikBj76R+c0K5nycnOWhpm+1GTf9T3TkiyXxBrVPAExgGQ\n0sDtAbB2WXLbUL9+cuXcUc5E06J8ALH0R0x9U1pfALC155vnv23NpXJ4+ClD+YyF\n+pN9clIEnfqbgjrD5mqQ8dTmQDfhOMUI7KfaZVAbOOUAaAytQxOXmo7iHB5mmMuI\nf95ckYXL5EIBDVJzLkbcDJvPj3paw41++DYTd/ol9a2gvWdLd8MGpCH8saps2lAr\noX5ljZYoZk3z3wmItgGRW/Rid3/Z9yaKJm5ITbpEbnUGrAAaBVTUxIHTqH+5MqUx\nsdb/OJ6EYYpLtKuFwIRp0JFWolu/8Pvsz3gSt8IHd/pDL2S123eAe1MCtr65kC6H\nHwJ7Hc+ejgUuqhFvyIhuyQuyEC40bFcGHmzrb+/zI2lTGUu9pjtcqp373QTZ/dJ+\nlvnV8CyCu7h124Bwnn63FdTrV3mXyFebGmKMEBuwEXF+0uJb7GnN018Z8OYKCDOA\nBGXpMVYzu8eYjgtBijVP4bO2uI4Os/rDNITzexi0GV3TcBELZ87zq/qDRYx2D04p\nM5lnOIMXEDN/zWyucJsnbfwTuYiPvNUmbd114TJk0dAXylo7jvqysWX9sORBhW2s\n=9Hrt\n-----END PGP MESSAGE-----",
|
||||
"SignatureEmail": "golden@example.invalid",
|
||||
"CreateTime": 1794402952,
|
||||
"Blocks": [
|
||||
{
|
||||
"Hash": "i2qnVyJAs9VK8ghYx//9yU1cnrfpJ+g8/Wo8MkxmZNg=",
|
||||
"EncSignature": "",
|
||||
"SignatureEmail": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"Want": {
|
||||
"Name": "Readme.txt",
|
||||
"ModTime": "2026-11-11T13:15:53.571Z",
|
||||
"MIMEType": "text/plain",
|
||||
"Size": 66,
|
||||
"SHA1": "a719cb0b84f635dfbd6e4ebb9ab0430a337e1fe4",
|
||||
"BlockSizes": [
|
||||
66
|
||||
],
|
||||
"BlockSizesRecorded": true
|
||||
},
|
||||
"Rekeyed": [
|
||||
"NodeKey: new key of the same algorithm, version and creation time, user ID golden \u003cgolden@example.invalid\u003e",
|
||||
"NodePassphrase: new content, encrypted with a new session key of the same kind",
|
||||
"NodePassphrase: session key re-wrapped in a v3 PKESK",
|
||||
"NodePassphrase: SEIPDv1 data packet encrypted again with the same parameters",
|
||||
"NodePassphraseSignature: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"Name: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"Name: session key re-wrapped in a v3 PKESK",
|
||||
"Name: SEIPDv1 data packet encrypted again with the same parameters",
|
||||
"SignatureEmail: replaced by golden@example.invalid",
|
||||
"NameSignatureEmail: replaced by golden@example.invalid",
|
||||
"blocks: 1 kept byte for byte; their signatures, which the backend doesn't read, left out",
|
||||
"ContentKeyPacket: session key re-wrapped in a v6 PKESK",
|
||||
"ContentKeyPacketSignature: v4 signature (type 0, SHA-512) by the node key of link-2 made again",
|
||||
"XAttr: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"XAttr: session key re-wrapped in a v3 PKESK",
|
||||
"XAttr: SEIPDv1 data packet encrypted again with the same parameters",
|
||||
"ManifestSignature: v4 signature (type 0, SHA-512) by the address key made again",
|
||||
"Revision SignatureEmail: replaced by golden@example.invalid"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"Comment": "File written by go-proton-api v1.0.4 and Proton-API-Bridge v1.0.5 with test-only keys.",
|
||||
"Type": "file",
|
||||
"ParentKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavRGjxYJKwYBBAHaRw8BAQdAcxECZNQ4lVFMtNsKOynFlDJucZF6T2fiG1Xg\nAcPbsBj+BwMIW7QxRJI04C/gqBs53KUbuGjmEhYxMTOnCoFiPLLNk8dFZMWeIuHY\nj/c6kqk8USn/PTta1KGFW2wVoT1t0rGehRoZxCmh/EOKi26tzkMEls0iRHJpdmUg\na2V5IDxub3JlcGx5QHByb3Rvbm1haWwuY29tPsK/BBMWCABxBYJq9EaPAwsJBwkQ\nOW3ol9vsq381FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ7sH\nDVufUE3ZEidEYWxokZACFQgDFgACAhkBApsDAh4BFiEEFieXN9ibAooUurRPOW3o\nl9vsq38AABPPAP9AUPozVu96istERmpB0nxNl6jZNSjQqYhFrFY7RuijpwD+I43/\nUyAYxn4yjdERbSpbJ1UIFmQ0eHxZARZRXBNJ4QHHiwRq9EaPEgorBgEEAZdVAQUB\nAQdAYmEyw9gdVGFAcbRmIbIZDZk/3L4MdouVywQj6ZWIxVUDAQoJ/gcDCFu0MUSS\nNOAv4A0vXnvSdZXU1vnnSMmDY7ckUn4M3xC3OacK59ho4XwGEZGYBHXE6DkhHxlD\ntH7Y1KPHqa8p4L/fZFLEIqgY0kKWRZs12z/CrgQYFggAYAWCavRGjwkQOW3ol9vs\nq381FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ/f359je70Fd\nlw21Vr7EIuMCmwwWIQQWJ5c32JsCihS6tE85beiX2+yrfwAA+ewA/j2udkH8FHHm\n7FobVhNzMs4cuvJq8iLcLyb4He5kABMNAP99hX2S4NwPqTKcKALsifcI6U6BtpKO\nsPJKhapNINbNBw==\n=HkFm\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"ParentKeyPassphrase": "Ec2hLKXDjN5pVFnbZliDIgC5W0z3oCUFDrkPY/+sZNE=",
|
||||
"AddressKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavRGjxYJKwYBBAHaRw8BAQdARWcSouC7sGNRuyVZoKs3SWWFYihHeq9P1Byv\nGkcqbtv+BwMIFdF5s0sA7VXg0qHXJ73pLNAbvFf1GrrtQYmrM8fZAbsfGaNgMczI\nK8onNODe4Ti5QaScmTEccWWcNWWmT/pHimMzeLqbK0+WwAKL3xGmWc0XdGVzdCA8\ndGVzdEBwcm90b24udGVzdD7CvwQTFggAcQWCavRGjwMLCQcJEAyZoc4ZKeB6NRQA\nAAAAABwAEHNhbHRAbm90YXRpb25zLm9wZW5wZ3Bqcy5vcmfqIiUAbBuc+hLJF/eV\n8LwvAhUIAxYAAgIZAQKbAwIeARYhBI0BQxJ3GY4DEE67cwyZoc4ZKeB6AABP8AEA\n/4zyw5U3FwgB/osSM8vswD0ooIxsXpAjxsWm4Wmzwp0BANkBBZEpQqXwinPJNFgF\njKdR3nQ6TaUdgbuLs7juGx4Ox4sEavRGjxIKKwYBBAGXVQEFAQEHQBa3CAAgulYq\nnJziefREc2nW0gRQPB8JZZ1TLAFD0rxeAwEKCf4HAwgV0XmzSwDtVeAXJYeuTIWA\nQYtMQn4+Kl5vu26nR8lUqYEQDu+h28XbyfnFTORCteFPEzs+fwmaPW3MjghvpY7H\nFMdUb5l9W2VhVXvApqpKwq4EGBYIAGAFgmr0Ro8JEAyZoc4ZKeB6NRQAAAAAABwA\nEHNhbHRAbm90YXRpb25zLm9wZW5wZ3Bqcy5vcmfl4YcfuQTTdndcPRHWUQKtApsM\nFiEEjQFDEncZjgMQTrtzDJmhzhkp4HoAANjzAPsEUwd1MSBwMYQlTQk57jHtDDJu\nHXam/ulN7MKrmqrteQEA/7u8NOc+UNuo9/DHp7so6thEdwSDIKihHeHsORQlxwc=\n=Tu28\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"AddressKeyPassphrase": "9sHbnSDCspkMZyKHdGAQ9B6zrYgbLHkWQqsECWvzmJY=",
|
||||
"NodeKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavRGjxYJKwYBBAHaRw8BAQdAMdQ8eoAyMsrtLZG+Ja6KEJ4MMh2AUHQQr1TK\nLjR83JD+BwMIWJ4sfdaGNG/gtXp7upM3K9GZJi/YCLG0R0crgfRD3dB1SAK0srAR\nuGV1rjVmX+RdhLD0388bJGwe1b8cT78HrVgoyEl3rYbGh3KVofWC8M0iRHJpdmUg\na2V5IDxub3JlcGx5QHByb3Rvbm1haWwuY29tPsK/BBMWCABxBYJq9EaPAwsJBwkQ\n3RlbDelLl201FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ50x\nFfdZqPwGy/8zDhp14l4CFQgDFgACAhkBApsDAh4BFiEE6Ll4lgslPayNe/e33Rlb\nDelLl20AACoFAQCkOD1psuolcg/OHsHgXOeFp8k+DEWhgyF0DsT+4W2WZQD/cGaR\nI94gxrk0d+/GVMEJ8K8cGgVUKk8bVokguOlZTAnHiwRq9EaPEgorBgEEAZdVAQUB\nAQdAgSEFuMxdPzSo81cQj65bZ0Byfzv8MZcCGkVlUmBhM1UDAQoJ/gcDCFieLH3W\nhjRv4C7ZRdYc1yUz0wptMrnqB+46s4MnlhxevozWeNnfm3wfrvF6/9UVpTA1WTXl\nhuySGG6+wQbOKiLVeUfG4ylNvm/CmrhwZvTCrgQYFggAYAWCavRGjwkQ3RlbDelL\nl201FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ2XJ6f3YWAv/\nAQ+PhALqTCECmwwWIQTouXiWCyU9rI1797fdGVsN6UuXbQAAPCsBAKSfTYaoZyjh\nffeSWz9nLbjk8WGkcyjlrwp6kBcBwCWbAQC8nMN99x12EEplbWSCIZW7HpwqeZPw\nBXrKSDGi8oQ5Cg==\n=CTFh\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"NodePassphrase": "-----BEGIN PGP MESSAGE-----\n\nwV4DG/kl44TpWE8SAQdATdhH9ktXSlE5lASZIDs4WQhYeh5PYNIZeElref9f9Fgw\nakiTnteDGk5WsFbmLqqRJVKzlCLJmwzlK2LZYl0K/enDs+amgSiLmAmHUWQ0tJ06\n0l0Bjuc9mLeUqLC7HLpYxVaq4Sc3yti09iJ5nbKeRfwMkrGgF6kx7RXaS6yPWIuJ\nO2HKKtrfZPE31q+557sJroNzd0qxmrtkLGW9TU7l9mQAAL+vp9ZEcXplDUTkT5Q=\n=uYiD\n-----END PGP MESSAGE-----",
|
||||
"NodePassphraseSignature": "-----BEGIN PGP SIGNATURE-----\n\nwqsEABYIAF0Fgmr0Ro8JEAyZoc4ZKeB6NRQAAAAAABwAEHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmfrNxwp4eaCYF7zWtW5log+FiEEjQFDEncZjgMQTrtzDJmh\nzhkp4HoAADOtAP9TOTKJycueysm+xuWWGKFTl6JITkgjixzCb0DdAlKhYQEAmXb3\nRU/ppZKbBq1CzdVSlrZD8cYYE8X/2VhveyYV+As=\n=3ybK\n-----END PGP SIGNATURE-----",
|
||||
"NodePassphrasePlain": "2b2rA5XFrudVun9TZmVWlzBivMS2asUFUuPiZEB6AxY=",
|
||||
"Name": "-----BEGIN PGP MESSAGE-----\n\nwV4DG/kl44TpWE8SAQdAJ2WiRvBRE3rxC3caxYNHgV9nQMbWVxp3HT4NAca7+yAw\nh/70OPPlroAvnXkL4QBheuNZ/W0iYNzi5hhRH3WUoDCj/R2PTOCWUtihv3zCPrSS\n0sBDAZ9HgQU1SpdB4IrUcl3jSFfd1mGtEfFwI96lqqqXaquW5yDrwoTegAzzNJY4\nBM2rYt4vwsvaVbYenz24c0LBGQMB1CW3mCfhcBFLGoDxwFSJmlq89KUhO2tOaebO\n78mNPVa3B+KQVDPC9MqO3ZD/OKGu9KC/IgCOgB+hzM/IoAmq0a+qGFWeX6wU+eEQ\n1bzczNTcZjCJPGphLfb5UNhOwlLifxlHOm2zGA8hyNOB4sKVTHNe179fnlLiA9w9\nJnN9kbzjla/aGH4wVkY5yFhyGGJm/99qxT+84JpKy1i0DWiuvLl37civmghgCp3u\n5MEljn/6L61m7eDSXclYXhwB7Y5YUQ==\n=Zlzt\n-----END PGP MESSAGE-----",
|
||||
"NamePlain": "Résumé – final.pdf",
|
||||
"ContentKeyPacket": "wV4DE2cZOOUo3HsSAQdA+HpQeTNWdV927TQjQEQ33kdJjv3M+P8JAorH4hoehwEwSgTIAKthsuZESyJAekahsn6pMirpgSg17ZCWjrTqJDxoEx1yqpQXTvLaX5G0vEAh",
|
||||
"ContentKeyPacketSignature": "-----BEGIN PGP SIGNATURE-----\n\nwqsEABYIAF0Fgmr0Ro8JEN0ZWw3pS5dtNRQAAAAAABwAEHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmfqeuaugxYe/Ga6r44Dz7UVFiEE6Ll4lgslPayNe/e33Rlb\nDelLl20AAKq0AQDj9Y0j1qVMofDJOQjXCluf3AWimxB6nHifhzSVGK3rugD/W5hj\nP3BZ0GwTUSpI70RSF3qNtlI+4DJTvatFXOQ8Www=\n=pNq3\n-----END PGP SIGNATURE-----",
|
||||
"ContentSessionKey": "ff21ed4de15ad45e6333155fe471a6230e104f69b89fdba4148f4282bc1f1c3a",
|
||||
"Block": "0uoB0Lr9th+xnh0RYvc9L+7htwoLpGNs6lz9HAPTnBcj63IDpa4REH4ShlaCbi8eE8f3dQGBroWTyVgqoA1CnzSU+s3pGZAdA66AsgTGzQyBEKWIwMzBSaXN3Za68dFLV2N+Vf/wV6qgcJQDj/1/6AM6SRTGD+vvC8iGMX3sY0H6YVr6mnwOIUwdMAqpT50baWnZ2uQoAMjooFzF/BgPRoQuG7Sgq1D2qMZKYBsPE1NbEbcG4PmNQsK78P9iFIwLAdHlJSOA6NsgQ2k++w31Zrm+Jl4BqLo02zsw7Oz7yuYUgmYK5d6Mg0CTclp1EWm4osSQjdY8zyG/BKfK+V8ISy9xP99zdGHKc1iF++rrpI+HxcS2G0LCA1AnYIAFthTReCrea4o1VQJkcM9HXoKf1E4vYtD2yFPE8dPCkAUhwRMBVhQvX6Kr8aylzLmngIs56G8f1vSHYtY7mhUTVnLvQu58wvEV5mb++g3tkwMX9HspnZHWGRy8dOgO1OHA+HLnkE/YHGKbHYW2mfYgc9BGMI5GcJo1CVR+rDzhTfAc2UddvsIuq3rulnoeBm56egYBm9SCsyDva9FnsBDrUf6R6M+M6DzYs578SBqRCP/3ojN0J+1jKGa8xXffsTJlEetf6ydfFm6I+X31R6/hgdETWWQQb7OLOxajhzV9HPdYYrjY4Qb0F//CDaBjG4qoDbSqWZopq2qwp8vkpfdxA3bn/rqph6ACaZsiWcKwyzwUxKnRRE5NtN7c33HyZB5auq3gk+kW1g1o47g29g2UQtbVfQndMiR9C1kG5AXe+78pDkXNsx41lAMr/+ZHpgcAyl8RTn5CyaQAcZ06czLESgiDXZ2uziuS0RWeGhkVlTWIZCoLuKPF/oGalQwfKupjCaNU7VACdTJevovafWduBqyIUtHoCNpUAiatfv3xwlrrj40Gt6Zd6MPzEAt6hCOTODIvLmEQqHADH+6BwKUlE9JGmod6ArfQCEBCaH7XqfpVompGstA90iUp5QpSDN0Jc/dm3JiQPUp2Jru9brOWsnL3LZ31p+kMgDQPhOyvSdkKUDGnZC/1Wz9qUQug6cgctH5HhZTDUH6lyxDGt8IniMa9Bvg+Bfq1KlW7BMNUFfTCSOSAvMVhK144rZe2hXgFKNyU5Hbutme0ciEUDsXt11ykWQrosf/64O9/u4A5MBvnw2p/CGwI0J+eGm1KD7vOR1BuWkCIKZvQEpimYW603upJhyPLQRiGE8l2IcZcp4syJvUAYr+lEUtOJKf6aDXbR96wBX6iBlfuI5GVgr+2KNsRM9TNcntDTT0rxxWXUIqxEfFPmNHHlFVm21t2sa9aVohqB9GajaAQNJ2pMb4HG68n/A5XMtK+Qccbnue8wUcxKPOXbbcnveH51qX3ufTgFuGtdATZGXqKAqablyRxhW6Umix3ZVSO",
|
||||
"BlockPlain": "jhkssnVxoBNx+hSsf98+/1wDc1Em3a816t6SaYH0L732514CBIsyY7lgn9uxeiQ2y+crfl2NxSh4faSSqGqh+A8RSOtYwkVj3IwubJP4rNpapz06/ojNLPUnChA+Zph9HMnpyo0SXoM5E0D7WwrQVYWXWZkj+G6rCjeP5j0JkBVILLdAtCGMGgAaB9p6zsniAGJT9iuFSo4HY5mzhOvKY3oICQijcOyEhJxJ5SVXV4XCsGE28UMLjWME1NjMlO0c7JmKb26NER2hyY77Qrecuqqb3yylWlcdq0qGSqa/39cm5e+THkSFMAJYpeiUy2gnBRsy51m7nVxbnmqvccHw1eNGBOeQJ2duMyLzxegHeXepbwRfmw+j5x22CtbLw4RLFuq2ICEv5EUvXlhfS8hQridBUOb6cmWe6PiehoNXaEH2pSuh/YeFa83IxXGXAhUrN0eA3ldU2+MxQt+gv6ECoFvvV2kyZ3m0jHsr3vDrqHGwmzcDBCZNVZzDFnHeE6e2u+fuzQe0BXUL7DTXG8RGJokt9hGW4G8SpyAkUhQInDCHwHJxOGkrxe/OWRURJ8Z4MRBVLc7RwxzF3wTAElSobjFkbMMTtsBkA5qIrDpDptGwiGN4e3VDHMMH40oylVV/9j4hJx4VG6s+YujLVtMw5oTDQarj+2wx0LD/N0dZitAqa/jhM9E3/j1r/t3pdOWQygJGdQOnibZ1Zslz2Nzw7zUcfR6WP/scBFC5jgLKsQ5w5jsde2j7TRVZ1SgVmSMd+aVrq3vil6NIiPpa7URRA1iJ4TUz5Oev9/lhbi772vrpv7N0/5hqpU2dyY4+09NgN38sMLFqtCr830ybEoi2T7o1drK24jxNXyQ5rUmpsHM1JUvsHvYfTT09Rh3FszVjhXCeqGtKH2sIeWRJKX+GUfKZPVI/CZezKx2Wq9F7THJ3rs/MuOu5HV7aEGZEoqnQ/r7rx/ZQalqkoEPgZX5DQD7RJDG/Q/SpkkqO76GaG3S7N7SchIdsvSgv/e9TTlW9+wetAjrIrBg4dfC7JDmTf+N9fiXiN7rDS0Xs9+5z6s1SHfnYfgPFyfRDHJvSvqQqBGd5qJQngEYEYfplKryWT2iJGkPt47IfQpm4fUQ+t4fDe29/TZjwu7uhEIb33QGx35KIOU+NMs9/xqU9CuiFbGuHCHtx3z8nvzNDafis2ZyBo6ZQkOn60uU1e+X3Fykim+CjzRAk1zg8C9L2CCctR9tWkEOqmyIlLNc59KWIMSmUZNrSul6OcoMaA5W9iPzq4Gk/sowF98YfSXJ8YFK84Ff61K7hpZnZTZ2BDzFtCRYOoL5xPYJlBxzf5gNqgHDyP4GXf81n10xWRPlyCm4kMA==",
|
||||
"BlockEncSignature": "-----BEGIN PGP MESSAGE-----\n\nwV4DE2cZOOUo3HsSAQdA3D0mxEsX6sY46Ax/7YxBktkQmWVFp1rhE0kOA8Im3Fcw\n/vQzS+ARcOz1lWRVfNBWU7sN0sTEAwthazkdBpJmfP+fUgMmN+YaU6sgxtiodSsh\n0sAeAQM0RFbe5+HJGyGwlV4tjSPaBsl4W4AZxhcecEXvMEMVcn7fCxSdTDcqVI5d\nAn3aLGsmotd0Wq9xeY355AwP0DgRhlW4p0ygdV++mKxH8RNxV1iYJVYYzBTeXuC3\nNESgJSAtxgPGAENnsgYBMp6+f2gMkZkLCGyd93S/rhfq05/7aCJMXAZM+nz4DtFQ\nTPdBwEkaftGVU/bArJYVysxd99zjOuSLG8Ns90cyMkwCq88W7x0oiEhkP2wjMrKD\njULd3e6C6sEXGq48IU5Q2WQC5QRVjg+rcP5fNTCbnSOY\n=nNd5\n-----END PGP MESSAGE-----",
|
||||
"BlockHash": "m5+liJFgVPWGYIPe2f5BbMTMC+X11GW94W38s69/7ss=",
|
||||
"ManifestSignature": "-----BEGIN PGP SIGNATURE-----\n\nwqsEABYIAF0Fgmr0Ro8JEAyZoc4ZKeB6NRQAAAAAABwAEHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmdyrNWgia1zbMJCpk6+Bv9dFiEEjQFDEncZjgMQTrtzDJmh\nzhkp4HoAAEM3AP9/r2fJp/fmtW8pd46LN7N3rd8VdlnOofL/9F2puiwD3wEA7ySH\n9JEdgvpbRTJuvxV+rVa+v6Qg/BMQhH/IdvZzyAs=\n=tJ4f\n-----END PGP SIGNATURE-----",
|
||||
"XAttr": "-----BEGIN PGP MESSAGE-----\n\nwV4DE2cZOOUo3HsSAQdAY+2NSlDFnAh0+btNiRFb5iCkadXpY7b19X2eSl2L7wQw\nBNSwrZCLPVYLjcvol22EiHnFGDZErj2HDL79ycBGd2wj6xa6ima2VSU+MBPk1PaV\n0sDFAVwSz9YT6Uew4fBSWgZZ6Uq+Sx8r6vP7OLxqO99RGMjnSNhSEHK4DZKU/qzm\nTewFbJQYMe6YwtDyIO7zkm/H+msl2tfUbwXxF/pWSagG06Zt5HKsNc/Nr8HC5Teu\n0SN4qKYr1yNbjtnKvhyCU736hi+H2lXKbeZtgQ0eNyynsF2taAb58Zgpu5rMOwLc\nO6jWtaAtOHHdWnSiWVFFZx9MASGwp7vFbyJ7kiTqVaHDa61bIvDluikIFvwdvADH\nxOng2vnY/jFnxhaPHj39R0V6O3yVhUgV/lmMPt+N5NNM0Oi55zSI3cfsN/9hbheF\nxCqb0Cy9tpNCsEI6bbWz0C0Z1vOSLcLm9CO1FHkICSzcqkhPjpqB2vQJoOeSh9/g\nwiQS6dTri89tVud0HrA8LSqOY7txqUMQyZkJ9HqCW0Df0fsEJkFDv7clbZlNZzWC\nlz+UnaJcrFnsmvL04TJWimHc95wWxI/I/1jTgVR95t6c3uGJLClXwfLM3fS9KF65\nZc/3XYbEcEU=\n=afPz\n-----END PGP MESSAGE-----",
|
||||
"XAttrModificationTime": "2023-11-20T09:30:15+0530",
|
||||
"XAttrSize": 1024,
|
||||
"XAttrBlockSizes": [
|
||||
1024
|
||||
],
|
||||
"XAttrSHA1": "1ce9dffe44e6ef3d42ae7866bab5cb5c53584eb2"
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"Comment": "File written by go-proton-api v1.0.4 and Proton-API-Bridge v1.0.5 with test-only keys, as stored by the server.",
|
||||
"Email": "test@proton.test",
|
||||
"Keys": {
|
||||
"AddressKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavRvDRYJKwYBBAHaRw8BAQdAn9wJYMU3m+C4tvfsvhRU93tzuWRg310WMrdV\nsx2Kl4T+BwMICgcyih3EgXTgjvT1fS6rTeedqa5rUjieViZWwF1DULHxaKeQ/9NC\nip2/VVyd6BpmLjFxRS5lBVbtuNqclWyffNJB6PLi48PVdcRghoWirc0XdGVzdCA8\ndGVzdEBwcm90b24udGVzdD7CvwQTFggAcQWCavRvDQMLCQcJEDYqIcFTgR6HNRQA\nAAAAABwAEHNhbHRAbm90YXRpb25zLm9wZW5wZ3Bqcy5vcmdMNQHu/YNFFXJDfjYH\nttlIAhUIAxYAAgIZAQKbAwIeARYhBGxv8QxbcgWIUDOOgzYqIcFTgR6HAABfEgEA\n097HW3FEUM4p8QBfRygziST5G8Pdnta9qt/8K4FWp7YBAICE+ObxNiCrHaKFKw9N\n8ZtkygUpwLPa/9U/5jHp168Lx4sEavRvDRIKKwYBBAGXVQEFAQEHQCgylrutR9Kf\n40IBXiewlxsthm2s5Bb/Mf0WKds0xIgZAwEKCf4HAwgKBzKKHcSBdODoYR71RzME\nDPMye9wjrTDZvVN9Ce1+tnH35KvtqgTYpzi/65Uy0Sqg7hJO5r//90X6wMJTNnSp\n2bLBFY7MJRkPZpHYg1Hhwq4EGBYIAGAFgmr0bw0JEDYqIcFTgR6HNRQAAAAAABwA\nEHNhbHRAbm90YXRpb25zLm9wZW5wZ3Bqcy5vcmfMW1GevtJUOgpjFl+siG8BApsM\nFiEEbG/xDFtyBYhQM46DNiohwVOBHocAAOA2AP4l88/MhdP1y+MBfEy1H27j0d+x\nhgx159BCo5uDgrp0MwD6AycyNO8rXnRXpH8UXIbx1joU7lDPq2nqrW9m9V1ooAk=\n=vDgd\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"AddressKeyPassphrase": "LzLbQlEPpaeI1rAGLEuCW2Gi35bR7rV9kmOhGO1i1o8=",
|
||||
"RootKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavRvDRYJKwYBBAHaRw8BAQdAkJLDdaqwDy4m5/Y2dacYwIkjkXAQxluj4igu\n68kVdbz+BwMIsmWbW1fOheTguxoV6ylBcmCr/3qb+TuFSTMkoqMwc84cRLv4xXzf\nBEzd5gAWhlixubeV6YBu3P63/3TTGzKxM3+nXLF6Mr/9GX/SsZ60Bs0iRHJpdmUg\na2V5IDxub3JlcGx5QHByb3Rvbm1haWwuY29tPsK/BBMWCABxBYJq9G8NAwsJBwkQ\nO8tFiQrA4nU1FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ/t4\nmbz8DuaNQ1vT/RFIIwYCFQgDFgACAhkBApsDAh4BFiEEYjEiVAYTl9y+qSUaO8tF\niQrA4nUAACGMAQCIefwEX02ipNhmV9eBYi2vq4uzW9qV6C+eFr66Y4HdtAD/ebQy\n0PmEFOk4X0lIq9qweqVOhhD4Hu0UaFmQ9A9qvAzHiwRq9G8NEgorBgEEAZdVAQUB\nAQdANv99N7oVHI9Wpz3c6Zbyem/iLYdVGvMZQyOEFYvu6gADAQoJ/gcDCLJlm1tX\nzoXk4C2C+l3NxlfgX5WLrgOgO+YQGy1kk5Kl2IvQDinoaC0sZT8wfBn/gXufXQcK\n0lqzn1DgCKnr/YSZjSdGMech2WGbTAvxpZzCrgQYFggAYAWCavRvDQkQO8tFiQrA\n4nU1FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ2pgeZSX9qPC\nKIxpuRRxFsICmwwWIQRiMSJUBhOX3L6pJRo7y0WJCsDidQAAkNQA/3p9r/crMHES\nGvfl4RIl9qHh6IcUbAduodC6SEwGCKUrAP0cfD3AKkgmtYBPFIYiW1g6i9cYT8f4\nYOw7KHzrzr/2Cw==\n=iBcU\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"RootKeyPassphrase": "AjtWb9qdq42Nys8cIQy48farxWagZEFrZV1f9H0Cx1w=",
|
||||
"RootHashKey": "-----BEGIN PGP MESSAGE-----\n\nwV4D75G3Biss+HISAQdAWjqo2i4N2XgibdrHT+PAgad2HV0owQ/sKmmcnzEILxcw\nseO8euBxqvlvt1cEx/AtX5S/1Q+nJfq0RxG2nsLVWYt0jyTjyRZ2tNCoLwow9XNh\n0sBlAaTAs3K/ZUm/MUDS1GvLMtfgQbR1eLRS0VER++m+wG2FapA/mL9SM+3E8bIC\nT21SoACKmZxJc9VrCLPC2XvNbiPXVNcvdx2e8DJfs1VVxYe2P2F9a5N8gCSPkG/y\nS26vVlz42DEtt3EE/yqYMuY1AQGGxPohT9NE1uCOvcO/h/4MLqwR2SKtcXE1Tgcf\nQLvKPlLebzb5ZBF78bi+EXn+0dTM2reLCs5a3f5Vw3abL+b/wkSIHa97jjZpD8Jf\nSFJ8GS/bOviY6d6pJQafJby7/iP07zpkYq0kJObknao9PMy3j8POLXh5dmZIhWmo\ndJn/T+6sOuqNatF63ch8K4Q03E0NbyfnPcEgj5pGodKtTumulGPXRGBfvKVYhb06\nM5GtgDVKem4=\n=pGrY\n-----END PGP MESSAGE-----"
|
||||
},
|
||||
"Name": "Résumé – final.txt",
|
||||
"ModTime": "2023-11-20T09:30:15+0530",
|
||||
"Content": "DteMSjInE8AZ4F9UF/gjhwqq9DweL4x3dF4cCGrW8kx7WAa2Clsc2YbM0o+vbk2s6GcUYEMi62HpKPQKtGy3efgelagpzZAMK78/fSSdWXQp91NGrISUl/fjEeyAzQI1y3u4Wk03ntm43yf25CCD/2Hh8uEmHMttM4e7h6A09MrW5ro7XMwOQI362C+NuIlypUqbESZkrBnnu7ONrt2bdkYtQWDVqOKhfKs0vUrCJBaARJlgAlaXdy6vOKK7HL3QvTFYNDSeeuMERJvoXoShoZMpP5aNtDAOc4/Jvjadat1honfBerTZO0V6WsK1wQt4t9pV/US9cBHJeyVuJn0ChCd14GLRmXAUVz0PMB2N7QNiXbXZ6Cf8DFoSlZ3APIitSdTqoec51w+DQ+WphxYM9pX3xEnruGuxKgp3/rWisQju8l6dQe7Z7gzmSPnL6adIo044uuHQGpBwt7p2f6DPak6sdgtnjulwQpRLF5WemSVYpvWemxeBM2OVfyXMIvCymCHVTJB1QNYD8fB8d0Iw6pycqL/tTKUKvrMzXYDmafACsFsdUcsoLqFYn9Qv5JMtcxwTxWzit9fAo3tT9bdggM4jNgKM1Bu9C7M9fOJ5ZfanCOP7VdPcBon5XnqKz7ToVbl+r3C4EiwLpeFyz5sJFr7zmp1u0E35d85JyJoL3mJ6ko9TlkO1F76OoEBr8XHpXwfWXVozUfUlVcQehY5Q1oT4fPTrwcoDVOLg3Otxrcz37B5Dw8YSyoHKr9tE+xZ7SaFx7Oskna0p+Q+ttoy2GCIq7zjxJe2QZCy+pNZtg/RlhgJ/Qh6rw1TW+Yu5Q9UU7HonVt4NXuOVQF6T/ZDZ8m1GqXTn1thBBvqO5Fg3qUSlj7lHKhGCsFpjxZQtLLNzJIFMs/ujqWwq6pLONJEw+sUl4uFTy+PCAvWjDRlUhZ/3nmlQFl/8PX0nXp4Eg70oOWBi4l+f5lx7NJBpOIpimNH6qdsVI8+7MtnkGC36zyNlc3Zskz1HabOsLHwwUGAy0GmQH/jXX9MF35hUdOH13angj37evED+nYkUH5HWHh0vkDlmGDmye6ZkNHClSkuRIYKlhr5IANuPHgISD4H8Qwi1wfUpScM+rA6ecoq7AfILBPtVXN9K9hjR1HSfguEm77rK4hFOkbOb/QTnjjbj6hw7IeOyaMqRBjjLMTtsG8tXdqJPBZJ63CKmmdOJF7MS9zMiIMchZ1XufI4K9S51LyX5g+OM090QVvSzSo2quq4eWs9bj8Nkkz3wA61m9yYi3WZ9/obyJ7P0oOw++87egK5UryUh3y5+maPGpSwuYBXvlo8pqbDJ4stCtyfRtDrk/8aY6GgugygPphb6U/E8HP5MLHL1xeHLRxDw+7lebH/j2S3CWQWykltLBftQB8/FyoWHap97iC1yvz9h+lAxwlzK/KrIS2Qs11wIZFs2vciv4ocOqPO8FZVT+OB/rNh/XvMLVMb2GSaVXGjWaBGioLNd+FdsS3cR24/OjzVjecDwCkhH1EO9ey8+jQlz2Yk+3+xDViscwjqZ60FbU+0J4dgAy7pOTy9+6Gi52NQYu/Flvul5LT5fBDDoa62uIbhuxYpHsKD9vVpgtvenCRDmrYAA7QHBhAb0/hcYCN5RiLajxfChYLobFgGcf/z0t2fX8Zn90bkZ0H6FljeLEMBx0ttJtqBRDaQrK+o/3YpzRi4GPHxKvkFmanUPzZfWfLzwM0Mdou48eTFLjKzruOtzAMXStJ09R+QqaX0CJmBHQefg0as7Iyn96h0vtIbXpanb8pjZ/uxnHZVkV4HP+ztnNx2DFCbfMUkc1Fv6bamhOHyDSEL0c5R9T0LobllYzPNvRYC649LFtrGNwnFeeXMomx92DKPu+Nhlv1kp7NkjhiA0nz98C1+ELsEEB7Wh6ppq3eBLn17cFWbAELuER8JpjOBZ2rH9apC7Q+9/58BIo9rHsYcM6Yf/bpCRFoCbovEtOO296Cf/sMyS8ImKvIZBsTguSCu4nPktE1nKL04Myr0HQjXOK1sf+j3xokRTj8w/6SQ0OZRYS7TnMBuXsAvKVou1x8+jQwv9uAmqT0lZwJNBK8CK8ZHkZfMtFxki5f1YyzhUBAr8J4P8hJ0sv9mRDWCwp/T7K80W6XXAB3IdUOHvkCpWj1BuxbJ+wjWgRL7NCFAUHQkIKGGkthESSOOZSI358M6ZZwYpPr2y+x78BfQ+9tDJDtR+b7j7EdXIqf/ULuThBxvWX4wb9S7dsFWw+ur9tA6Fq0KSdNM4ed9l3Z3gHdGIwJ6o48ji1f9vvsWJXzjhEYJPJvfFb4c1gV9gr6CqNYuiYaoUWe6tNbGe+CPp/6saETXA4f8GzjGc5KsAyh2BFhS5p31xzjUAMyfZjLSidsLF3rtVwiOsBes+8//VsYG9hCBqvHLzN5FORCnax1/q50mHdwAzw4GHmF6ja9Cia5KzIxaBBTFcIlRCJdbExfj4PRzb7+Zdp1FaEVkPmkJPEtUjPU4c4GASwLnmcIkNz79ifE7Pz0ruYEGUyrwYVSjBKaaXJrDELbmxFYa9DbUyf18NZASP6ztRt4OeK2lOAd3PTIeXHYycc98bpZJbBabA/dj/je6EtTqN/1wXaH0OCVBIwqsDSVSgJK9Rjy5YAscUKLC+Z38i05cnsRbsBsRTyKalwgAB+hQ1DaRfw2hAVYD7SKrJQuSvXwL8PQ4wsbKQJn52oD6XuNhjcdGRIoGlRnKTsAjUv67srL5WEUzkPYNRgEYIq+EDjT9B/OfMBKH+gu2p8K04qaeS0BCrVYh/Lzf9kHuExcA/MDncTupkt0eJHibl2jQ+XJVavPAa6aQ9cLFkulm0NmNE7/NHb9C3ipbhTI9p83o+Q74L/ZvWPLW6yC689pf/tRBNJJcSN/YVIq2WbcMjCNxBiJsA2eCty6R4O0ICv18XsrrQU5lZP/PDiB038uAen8DPCH8xIQn/QKi9JD9mHhf8XUVmKL3u3656YuPvJ2faOjSGbZ9WfmY73GoN/1h4er2wbBmjKyVWYHlx81GLeYWx5rPbaLVaGxke0JaSIt2sS/pH7G5gX0g3gG0Il0VkWuFiJXiK8ifqWiLtutGUGH10ijKPijNNTC5VWvsKzZy6+s+G9EeCIEluuAiMklC0cTfdNLrWZ702hvh22P2PXaWp+tlknGI6Vw+ocaatdkaz5SKz9EdHzf0N/QsIDzqlFsmDHmQ7aElAFkK7KeIq43xjVfqU3b/fgPl7bJGuig6hv3B7csvPcgX1ZuuaFaknTErR2j+mVzBspVfkcVdLhCBLf+aCyenh22EGdCWYJf/gXd2fG+wfBQ=="
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"Name": "-----BEGIN PGP MESSAGE-----\n\nwV4D75G3Biss+HISAQdAeTQ04HdHu3aYN20zc8f3lCO/wa8C9lOB73AP9okbESEw\nXEGACuJqMH9Ww87+ZrXvnepYfTgfUgTt1RmgIYOvMn9oLskG/QppvpCa4WIR2z3J\n0sBDAR95/Z8E+5hfeo/AMnGWrGXAXWGhOkfkCAAthgPA/DCQ6JpLsod2Jn5WF1kf\n0oHheYBjMtowHd8XcmafBdQ5+v/mzoEOdorl6i+q+U5gzTkiHVwYA+iNa7zwoEdV\ng+vHtmG4ZLeJeYRzrjLHQ8pfZvRARjikPVbFPN0k8Fa3ISW8ccBeY7hVsNLY98wL\nEX3z46FdxGb1Hn3MY7K0JITWYPK897AoE+RHypqTKfQtze/UJw0rEuFXSTWNFGcC\nh3TiifPxzIot4PSpoNf8UneQR8I5660Iijo3yt1l9E/4yaAazFeXdEXJ0ZCJ4ZvU\nC7yA+RaHQfFp6dF/FgyNYN1UAFUv5g==\n=Uv5h\n-----END PGP MESSAGE-----",
|
||||
"NameHash": "d80859185c47f576afd1f7aa0e1a2dab9127c843b0d741a7602b4564fc37b656",
|
||||
"NodeKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavRvDRYJKwYBBAHaRw8BAQdADN8Yzwsl9W83z2uIQZ4Ux44dKhvc/Qojx8c7\nTEbZ+F7+BwMIikWiUxjYUjbgAd02ez9abEkUMQrYhuycQJAoE0QTKrZEKDzp+okY\ngksWkWVYmjQhMnwB/qbdwV5oYYkRe1nPcsbhAdFiVtvOCypcl9tyDs0iRHJpdmUg\na2V5IDxub3JlcGx5QHByb3Rvbm1haWwuY29tPsK/BBMWCABxBYJq9G8NAwsJBwkQ\nkUBZTYL0eA81FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ+Db\nYMKdZE69sZOTmHSwEi4CFQgDFgACAhkBApsDAh4BFiEERPv80oTZWPDRaMcIkUBZ\nTYL0eA8AAF1XAQCFiD0Z2XTx5XaCvGdq17JGAxCQ+grj5nQmtnDfJjQqdwEAytp3\nbrYVsGfHk0Tu2piZvChMokMHhcjNghFtFmKhywvHiwRq9G8NEgorBgEEAZdVAQUB\nAQdAW4iCSarmRG2nA3sr8gscojT6gfJkhZfQE4r5WUZ+nU8DAQoJ/gcDCIpFolMY\n2FI24H5TLfkSXpV3QP1or5Acny9UVCynWdnG1o6VGJLXrQwcJvSjS6m3sGx54jyz\noWDQylzqxe3EbQKcu1v/7PxFSP8xumCcVWPCrgQYFggAYAWCavRvDQkQkUBZTYL0\neA81FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZy3CYMcltPWm\n/6tvXL3kLcUCmwwWIQRE+/zShNlY8NFoxwiRQFlNgvR4DwAAqwQA/3M0VdF+lqvq\n2IoRd7WSVwe/AlAvuBsEUsngfhWRL1pnAQDk9Puh0cuZnqkomh+y9SlZgDAwWwV1\nfwpjPOLA7xmbDA==\n=4jUT\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"NodePassphrase": "-----BEGIN PGP MESSAGE-----\n\nwV4D75G3Biss+HISAQdAlymniWYnrqy+gLH0izGJYV3OLaVWKMr3aKCcCSU5J3Qw\nFMx6XX1hETVuI1H5Smfql5FXPGyhhQQdJhj/zt0KWvXzxmoqAzQYXmuxqBUYyeB6\n0l0B7gkXd44uLjzPpV6K3BuT0a50rEA+jtM0Yx7DHkGQdV9nIDUCsiny/4hAYUep\neUFWHS7u/APBqkKQ8MG3NnkGkhQE2mnLSIlcKl0rzELsejfAeP5xD04r6RY0rzw=\n=AVcs\n-----END PGP MESSAGE-----",
|
||||
"NodePassphraseSignature": "-----BEGIN PGP SIGNATURE-----\n\nwqsEABYIAF0Fgmr0bw0JEDYqIcFTgR6HNRQAAAAAABwAEHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmfzOVpw3MH8ZeApbtY+kBxrFiEEbG/xDFtyBYhQM46DNioh\nwVOBHocAADUEAQD53J/jV+suvOSZ27uuv99Dr9BT+X9JLGl5EdpIzWieKgEA4vD5\nGzxbMgz0Siz3yuVlnwXFt1rNI6oIEV5x+bxhBgc=\n=NRz6\n-----END PGP SIGNATURE-----",
|
||||
"SignatureEmail": "test@proton.test",
|
||||
"NameSignatureEmail": "test@proton.test",
|
||||
"MediaType": "text/plain",
|
||||
"ContentKeyPacket": "wV4DPENlinHdht0SAQdAq5Hb36qZH+/AGcTBA4dVAIMFaGu92geylFqmbS5LqUYw0yB4qPE7lUnVYZCysz+3bPCnxt/pDxx1Ktzw35ocUceim2ki+cr5b3mzc0ZVXhBt",
|
||||
"ContentKeyPacketSignature": "-----BEGIN PGP SIGNATURE-----\n\nwqsEABYIAF0Fgmr0bw4JEJFAWU2C9HgPNRQAAAAAABwAEHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmdEBzg7W/WHkXZnjiSnKULSFiEERPv80oTZWPDRaMcIkUBZ\nTYL0eA8AAIwUAQD5O16YIXf936L9zUviRububNQZheSxkX12fDGSLCxiDAEA0sqA\nbZ4D7d5X/kg9uVzEzG6nHUYYgXlzXYkPlVuK5wM=\n=WNUY\n-----END PGP SIGNATURE-----"
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"ManifestSignature": "-----BEGIN PGP SIGNATURE-----\n\nwqsEABYIAF0Fgmr0bw4JEDYqIcFTgR6HNRQAAAAAABwAEHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmcWDcsQsoZ9oxLWcCDf/2keFiEEbG/xDFtyBYhQM46DNioh\nwVOBHocAAHOQAQCug7+Y6J4HfxtEhgjzXvJPaU5RSKdJkZTJhJTXodyAugD8DbFv\nY1IDnwP7VOrfCD28G7zbx+7WCXWnixkS9XTZDwI=\n=YQiZ\n-----END PGP SIGNATURE-----",
|
||||
"XAttr": "-----BEGIN PGP MESSAGE-----\n\nwV4DPENlinHdht0SAQdAtLl5oIJ9RSeGAO3n4w1r6iQVzrNj7Y4WNgL3hyWhaTIw\njAOU7a/gztg7Q9yyKLKsrRED5j9UACo9q87qlgJKGaOyW58nfYsLdLF/oM2Bl97P\n0sDFAermVanvd3hEwvHUaZBDp7s7pIzglXER9AEALmmz3sK7yc6axP3ilXYz/+QS\nHDigy3SOLNHVwSwhGNAyAVtR4KOVXWdrfSkoSVkMtVDmL2wcmEoz43clvfjmX8Db\nXquYw/pAvDRzz1CYin8H3kok0IYPvueP7vIoQwPqbwYuJcJtcOP5OUdHcL+GXiBr\nvAEv9cN9bfzD32le+UowmODCFBI6Mbwl0CsrSxSonAcOeun0WP3dWDQ/Rrub3Jgb\nVEEt2qiSKVy9e6dkm8E91g9QB5VcnS+PmOq6ZHWTcOTREWa3sbHjUPMktQIyN6x2\nw5a36TtsIjW5+CrCvQqVTSOOxvIYAZSd0d+t/KCcKZYKPIo8Lyo74RYnZ4WqwHhv\nBrGULEbrxkofFZugRYKN9A9ZGA0FYdU83FfyACW4ACEdkLC5Na5zvd0gEKUGhZDg\ntuM7yYQyYxSuBr4JXQ87qjXwdhRYLKdjSiHwE0qaxc2vuHknHfuP7XHDflh7zu/9\nB5Cmzq0JGRY=\n=IxS/\n-----END PGP MESSAGE-----",
|
||||
"SignatureEmail": "test@proton.test",
|
||||
"Blocks": [
|
||||
{
|
||||
"Hash": "NFq37D8SE2nGxYSHBEYd1VVxjxpjpWcFLh4XYx55x8I=",
|
||||
"EncSignature": "-----BEGIN PGP MESSAGE-----\n\nwV4DPENlinHdht0SAQdAcw5K1cHCktkNtqI3nz+9A6hfL8nXF6wgHX1oPo7H+nUw\nRWZ8e9hYK3k2VqdYXtCVH8vKSHYUw2hqQ7NBWErFu0IuR3iM82ojiCWHSy9iwP9Z\n0sAeAUmlJSuI1pxhxi5KFeSSeyt34m6H+ZSqo1Bm0ammv/j9I2htUAlwD1MxQjWh\ns5qMzuIjmqAXddSl5rOxJ63uOi1oQ7PUXByX3Aa774uzRri/AxktU3L240K69KBY\nt1ZDPtcbFkWOHdpULGKyZx0M4t3U6uSKKoXCiqPg0URdYyLfQPd99QdO+iaT2IJQ\nxFqZFJeREqoMq/JdNXDFuB3YUTiY2U4h5Q3zZlkha1cucI0Vw3CtWuXCjGxOV2+s\nv4aTzgx3xaBVFYye6RvBvQ9H5V23MbLEXjFWSTw9+2QG\n=bKI/\n-----END PGP MESSAGE-----",
|
||||
"SignatureEmail": "test@proton.test"
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"Comment": "Revision written by go-proton-api v1.0.4 and Proton-API-Bridge v1.0.5 with test-only keys, as stored by the server, of a crypto-refresh file: the node key, content key packet and its signature are SDK-style stand-ins made with plain gopenpgp; the old library wrote the node passphrase, name, hash, blocks, block signatures, manifest and extended attributes.",
|
||||
"Email": "test@proton.test",
|
||||
"Keys": {
|
||||
"AddressKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavR2jBYJKwYBBAHaRw8BAQdAYxcvYBaGtyA+Kyk4og3+LZivUVrk7cSS1M9/\n4/RztEv+BwMIAR3oa1tvWYzgG1gdifjVo2uzul/v9tOLq+OHNTuRzFOF2a76fAYr\nQoCYBmBhisymd19Z3GeC9X+sOJR5YtxAxIpXbynpZ6yi4HZHuz2US80XdGVzdCA8\ndGVzdEBwcm90b24udGVzdD7CvwQTFggAcQWCavR2jAMLCQcJEOjFeJjESJCeNRQA\nAAAAABwAEHNhbHRAbm90YXRpb25zLm9wZW5wZ3Bqcy5vcmercZpFiE6lJB7Dnzfo\n7CGFAhUIAxYAAgIZAQKbAwIeARYhBKLw1m3WxlgHlQYU5ujFeJjESJCeAABHvwEA\n+Z4fN4i4gGrvSTLIvWbTN5Ei1otzunUyaEFg4Bi0L/4BAN7Y4RxMfxXIAvXDDRcO\nHUgMaPny3l8BfSsSszRfcZcGx4sEavR2jBIKKwYBBAGXVQEFAQEHQBk/eEB3QpmN\nWzJB41zvXiWkUzsP0HzQaNAAOPjiHy9jAwEKCf4HAwgBHehrW29ZjODxy5OKTsy1\nSBIbMWlzN6H1fjtJUhWho7qXVu7AAixJen+j+FsEnR3ADZ52cpVsKmbHMfSEuesj\nzPqpQwYvFzeKKqS3TwwCwq4EGBYIAGAFgmr0dowJEOjFeJjESJCeNRQAAAAAABwA\nEHNhbHRAbm90YXRpb25zLm9wZW5wZ3Bqcy5vcmfUCpzU0sf+A8/15f86ylhxApsM\nFiEEovDWbdbGWAeVBhTm6MV4mMRIkJ4AAI+7APsHM6jxLF+DZDziuO55F67ZfqQ0\n4K46bhWMa6Zs+Eve3AEAiWgBlXRNJdAzaebECEdpCktkmZ0Lnxcql+PmtStZaA0=\n=kaLF\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"AddressKeyPassphrase": "iwbEuKOt4yaNBKaN7qmznAm8lfzXZhOASJ+5+FQ93Lw=",
|
||||
"RootKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavR2jBYJKwYBBAHaRw8BAQdALR6055bPx21abVlNEHwqHKGxX2uufVwoO3l8\n8bPH+0b+BwMI0LcbSOntz43guIwcXWuS5WRg0WZ2M/AUefYKiA6d1LWMEQhieATu\n/nXRGdhx2ifSFLIEg9ec7O2I1fPOjT+3NqFKueIQaRvkSGWjmdhCmM0iRHJpdmUg\na2V5IDxub3JlcGx5QHByb3Rvbm1haWwuY29tPsK/BBMWCABxBYJq9HaMAwsJBwkQ\nOg1jrc2qPG41FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ3S2\nqQS0vDPnpg2fJU8XmlcCFQgDFgACAhkBApsDAh4BFiEExH2zAwmj+0hpXXQDOg1j\nrc2qPG4AAEWmAP9wE87+vZEeZDGQWfg6zws6x+KMWTTQZkzY6JdcBYgYUAD/WPDY\n7nznixq+QgpKLwgpg7eHjq1JULAuMvSCNexwRA/HiwRq9HaMEgorBgEEAZdVAQUB\nAQdAEBN85n+6p3iumQ1FzRnhD5Ylf9k/U/rOlW00LXAxDzcDAQoJ/gcDCNC3G0jp\n7c+N4Ls4entT1mAbA3Fk5kTIxYnVQ8XIz5jn4K+bn2VqKksSnrbLgnQfhW5tUB0B\nrKkSGiXpWpVYdl+VaTApMtB4SVFhlRXcF7TCrgQYFggAYAWCavR2jAkQOg1jrc2q\nPG41FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ+ioSh0RloIP\nBrzFsLHRiuYCmwwWIQTEfbMDCaP7SGlddAM6DWOtzao8bgAAeeIA/ibQx85gu0Cd\nMkWq5kCZ6+F7oxJbi1oJZx4RwCliOeufAQDq5EiXJbgM+sl7FT08TZJt0lk+J0Hb\nYWfUC8tMhAFeDw==\n=9zK0\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"RootKeyPassphrase": "K2ssV7FaPJYYC1hkv4EpYMXAR5mdpliJdXe3un8fESo=",
|
||||
"RootHashKey": "-----BEGIN PGP MESSAGE-----\n\nwV4DfdbiRzc5UeoSAQdAozxxuTOM/JbY+q/TA4Y+MkF9un0reHW2d2PTlEPOxxIw\nE5hk5xx+Ac0CPFazJIEBqJS4yIMptA3xbGz+b294hbVeLvt/d+Gn1mWzB4/XPkac\n0sBlAaISANVJvSKh7vXBpGWa7hKRJv+3AdhkeEY6I9rAz8kYg4uw/+NkGNeGu8nm\nr81go8CY1ay98Bt09bUYrILvrGTrIIAOMSClGjJXmmgGPEDMe/ESQoZWhi1LVaCf\ntlHHDu8kCtjOESViElaHe8xzsEwWInz8C+sNYzgK7jx86Bos2lsM8drI9SSeXUUa\nZBH33dOnS31vKx8ZpGS+4f48CDqp+R8BJJ5S0ZKJ11fNbLfO3/c/wwvg/fE6wvSz\nFcbadk+VT0bc/c3wcxU4+S1Ifbf6TxJ1Bs7weDctak+BQxpxS/oMN8lGNxkStl5H\n2hkCF3dmB/TynoWiVyiqv+z1v7MP2hDWwpvKtOViSfrPhZZI8dwWMSD1ELEqLOFL\nOMswK1FUmIU=\n=NZ0X\n-----END PGP MESSAGE-----"
|
||||
},
|
||||
"Name": "Résumé – final.txt",
|
||||
"ModTime": "2023-11-20T09:30:15+0530",
|
||||
"Content": "DteMSjInE8AZ4F9UF/gjhwqq9DweL4x3dF4cCGrW8kx7WAa2Clsc2YbM0o+vbk2s6GcUYEMi62HpKPQKtGy3efgelagpzZAMK78/fSSdWXQp91NGrISUl/fjEeyAzQI1y3u4Wk03ntm43yf25CCD/2Hh8uEmHMttM4e7h6A09MrW5ro7XMwOQI362C+NuIlypUqbESZkrBnnu7ONrt2bdkYtQWDVqOKhfKs0vUrCJBaARJlgAlaXdy6vOKK7HL3QvTFYNDSeeuMERJvoXoShoZMpP5aNtDAOc4/Jvjadat1honfBerTZO0V6WsK1wQt4t9pV/US9cBHJeyVuJn0ChCd14GLRmXAUVz0PMB2N7QNiXbXZ6Cf8DFoSlZ3APIitSdTqoec51w+DQ+WphxYM9pX3xEnruGuxKgp3/rWisQju8l6dQe7Z7gzmSPnL6adIo044uuHQGpBwt7p2f6DPak6sdgtnjulwQpRLF5WemSVYpvWemxeBM2OVfyXMIvCymCHVTJB1QNYD8fB8d0Iw6pycqL/tTKUKvrMzXYDmafACsFsdUcsoLqFYn9Qv5JMtcxwTxWzit9fAo3tT9bdggM4jNgKM1Bu9C7M9fOJ5ZfanCOP7VdPcBon5XnqKz7ToVbl+r3C4EiwLpeFyz5sJFr7zmp1u0E35d85JyJoL3mJ6ko9TlkO1F76OoEBr8XHpXwfWXVozUfUlVcQehY5Q1oT4fPTrwcoDVOLg3Otxrcz37B5Dw8YSyoHKr9tE+xZ7SaFx7Oskna0p+Q+ttoy2GCIq7zjxJe2QZCy+pNZtg/RlhgJ/Qh6rw1TW+Yu5Q9UU7HonVt4NXuOVQF6T/ZDZ8m1GqXTn1thBBvqO5Fg3qUSlj7lHKhGCsFpjxZQtLLNzJIFMs/ujqWwq6pLONJEw+sUl4uFTy+PCAvWjDRlUhZ/3nmlQFl/8PX0nXp4Eg70oOWBi4l+f5lx7NJBpOIpimNH6qdsVI8+7MtnkGC36zyNlc3Zskz1HabOsLHwwUGAy0GmQH/jXX9MF35hUdOH13angj37evED+nYkUH5HWHh0vkDlmGDmye6ZkNHClSkuRIYKlhr5IANuPHgISD4H8Qwi1wfUpScM+rA6ecoq7AfILBPtVXN9K9hjR1HSfguEm77rK4hFOkbOb/QTnjjbj6hw7IeOyaMqRBjjLMTtsG8tXdqJPBZJ63CKmmdOJF7MS9zMiIMchZ1XufI4K9S51LyX5g+OM090QVvSzSo2quq4eWs9bj8Nkkz3wA61m9yYi3WZ9/obyJ7P0oOw++87egK5UryUh3y5+maPGpSwuYBXvlo8pqbDJ4stCtyfRtDrk/8aY6GgugygPphb6U/E8HP5MLHL1xeHLRxDw+7lebH/j2S3CWQWykltLBftQB8/FyoWHap97iC1yvz9h+lAxwlzK/KrIS2Qs11wIZFs2vciv4ocOqPO8FZVT+OB/rNh/XvMLVMb2GSaVXGjWaBGioLNd+FdsS3cR24/OjzVjecDwCkhH1EO9ey8+jQlz2Yk+3+xDViscwjqZ60FbU+0J4dgAy7pOTy9+6Gi52NQYu/Flvul5LT5fBDDoa62uIbhuxYpHsKD9vVpgtvenCRDmrYAA7QHBhAb0/hcYCN5RiLajxfChYLobFgGcf/z0t2fX8Zn90bkZ0H6FljeLEMBx0ttJtqBRDaQrK+o/3YpzRi4GPHxKvkFmanUPzZfWfLzwM0Mdou48eTFLjKzruOtzAMXStJ09R+QqaX0CJmBHQefg0as7Iyn96h0vtIbXpanb8pjZ/uxnHZVkV4HP+ztnNx2DFCbfMUkc1Fv6bamhOHyDSEL0c5R9T0LobllYzPNvRYC649LFtrGNwnFeeXMomx92DKPu+Nhlv1kp7NkjhiA0nz98C1+ELsEEB7Wh6ppq3eBLn17cFWbAELuER8JpjOBZ2rH9apC7Q+9/58BIo9rHsYcM6Yf/bpCRFoCbovEtOO296Cf/sMyS8ImKvIZBsTguSCu4nPktE1nKL04Myr0HQjXOK1sf+j3xokRTj8w/6SQ0OZRYS7TnMBuXsAvKVou1x8+jQwv9uAmqT0lZwJNBK8CK8ZHkZfMtFxki5f1YyzhUBAr8J4P8hJ0sv9mRDWCwp/T7K80W6XXAB3IdUOHvkCpWj1BuxbJ+wjWgRL7NCFAUHQkIKGGkthESSOOZSI358M6ZZwYpPr2y+x78BfQ+9tDJDtR+b7j7EdXIqf/ULuThBxvWX4wb9S7dsFWw+ur9tA6Fq0KSdNM4ed9l3Z3gHdGIwJ6o48ji1f9vvsWJXzjhEYJPJvfFb4c1gV9gr6CqNYuiYaoUWe6tNbGe+CPp/6saETXA4f8GzjGc5KsAyh2BFhS5p31xzjUAMyfZjLSidsLF3rtVwiOsBes+8//VsYG9hCBqvHLzN5FORCnax1/q50mHdwAzw4GHmF6ja9Cia5KzIxaBBTFcIlRCJdbExfj4PRzb7+Zdp1FaEVkPmkJPEtUjPU4c4GASwLnmcIkNz79ifE7Pz0ruYEGUyrwYVSjBKaaXJrDELbmxFYa9DbUyf18NZASP6ztRt4OeK2lOAd3PTIeXHYycc98bpZJbBabA/dj/je6EtTqN/1wXaH0OCVBIwqsDSVSgJK9Rjy5YAscUKLC+Z38i05cnsRbsBsRTyKalwgAB+hQ1DaRfw2hAVYD7SKrJQuSvXwL8PQ4wsbKQJn52oD6XuNhjcdGRIoGlRnKTsAjUv67srL5WEUzkPYNRgEYIq+EDjT9B/OfMBKH+gu2p8K04qaeS0BCrVYh/Lzf9kHuExcA/MDncTupkt0eJHibl2jQ+XJVavPAa6aQ9cLFkulm0NmNE7/NHb9C3ipbhTI9p83o+Q74L/ZvWPLW6yC689pf/tRBNJJcSN/YVIq2WbcMjCNxBiJsA2eCty6R4O0ICv18XsrrQU5lZP/PDiB038uAen8DPCH8xIQn/QKi9JD9mHhf8XUVmKL3u3656YuPvJ2faOjSGbZ9WfmY73GoN/1h4er2wbBmjKyVWYHlx81GLeYWx5rPbaLVaGxke0JaSIt2sS/pH7G5gX0g3gG0Il0VkWuFiJXiK8ifqWiLtutGUGH10ijKPijNNTC5VWvsKzZy6+s+G9EeCIEluuAiMklC0cTfdNLrWZ702hvh22P2PXaWp+tlknGI6Vw+ocaatdkaz5SKz9EdHzf0N/QsIDzqlFsmDHmQ7aElAFkK7KeIq43xjVfqU3b/fgPl7bJGuig6hv3B7csvPcgX1ZuuaFaknTErR2j+mVzBspVfkcVdLhCBLf+aCyenh22EGdCWYJf/gXd2fG+wfBQ=="
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"Name": "-----BEGIN PGP MESSAGE-----\n\nwV4DfdbiRzc5UeoSAQdA7XKU4rc8MiHjOzDkMESI9r/dg5vTTfGQrYL+mmnAaDAw\nf45S5UMFYSTVp8robWoGQqMKUS5/CF3Q2SsgunObM8T5IeWcMDSiNWvvYtPexHk9\n0sBDAU4lEwh4RB9jMhExBETq3F++pWb9GFgu6utjdG52I1Gq7mGJ3rdX56KAwFiC\nyWjFXBflSJRvPSng93FyQ+DxV8OqWfugnXGxtpWbFwMolOG345R4gJ8FTQKZrrak\n1ODV1pIJjcOnTwCAw+uW9ycNTI3v8uOnu+I6J89yK1mICAphQLvxacqyeJsBqnWH\nc8d6KOAfWJdQ+PWHpyVPPjCS16IgxPISahIASnOcR/BYsE5kiHP0dIgWXG+CD6t5\n3Yq+KexiMCsxj+zlm1zHP4OGkR7NGArKDILIkf0Yvn5rcNe6aum0Qyzclfxs3yYs\nPWPBSsnUhO/yzotkTLcLxEkHMJLSuQ==\n=k/P/\n-----END PGP MESSAGE-----",
|
||||
"NameHash": "33d5642922757df8750d3cd867dc34d742d198c12a1a424b3617fe07c9767718",
|
||||
"NodeKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavR2jBYJKwYBBAHaRw8BAQdAhqnFsr4HWUkMEijuIQckOJ1G2K3VuFZe1J/A\nQ83Zmnj+BwMIgOIBis/Izj/gSJkJlrxE15UBHsshXlnDwxWFFG0XY6iQUkum5nsz\nVCsgm34tbr2+kYucI5OH4ZcUwhg1+4m26Y3xCBkeY1kJ1wCzcnZs6s0JRHJpdmUg\na2V5wsAJBBMWCAB7BYJq9HaMAwsJBwkQ2q15SJvUrn01FAAAAAAAHAAQc2FsdEBu\nb3RhdGlvbnMub3BlbnBncGpzLm9yZzWmwVVWqUJfqSK5PtNaUPgCFQgDFgACAhkB\nApsDAh4JFiEEvKrbb+RxgL0NxvP92q15SJvUrn0JJwkDCQIHAwcCAAD2lgEA+H8e\nDXTky36nJadZkkEaX4Ru2O6WLfLFw6DKBmYHiZABAPm6gwAWC24RzSF9mmYLsqu2\nZLSmgew6sz5vR2Arg1cAx4sEavR2jBIKKwYBBAGXVQEFAQEHQJJQPKAO2/cIbeG6\nhzoV4BMQG6HkGU5/IWT8d0Wn9UVRAwEKCf4HAwiA4gGKz8jOP+BjJSL2oj50LhQP\n/iCaAWlZId25bFkNeLnIql2EcBpB8pDitXK4CWzi3GUi414RL1SB6OYDntJ2EdQX\npGqNGQe0g84/Tp1Nwq4EGBYIAGAFgmr0dowJENqteUib1K59NRQAAAAAABwAEHNh\nbHRAbm90YXRpb25zLm9wZW5wZ3Bqcy5vcmc4cBkscMRvctJw6VuhTb4oApsMFiEE\nvKrbb+RxgL0NxvP92q15SJvUrn0AAD2RAQDsyXaWf7qw20fYSnKamaHZghCaKDgE\nAaJ/COvWeR6QtgEAgVeS5bpt69qcjl4M1VeiRg/OKuyz98LKePV/fcfK2gs=\n=mNhc\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"NodePassphrase": "-----BEGIN PGP MESSAGE-----\n\nwV4DfdbiRzc5UeoSAQdAjZU0sND1Y/3dN+xemfdmSTeudx/SPS1cgG9rjtCxmDUw\nWBn2aamgTJK98X+Kbb+ENk9SiZzIIm3q9BgcnrSAGHg5Ea45VIU3c0w2gnxTR/MG\n0l0Bpi944nmqbIlU+VvQLzyQl1TWlBrzJjPHg8VEwYOeKsWgz/VNBlYSdjFsIo0v\nbwNu3xzNRrxs/VL1Na1VG0U8kSSRBoIRrjRz/DdLnj40BCN9i5ah/2t2L0mvLjE=\n=I9+8\n-----END PGP MESSAGE-----",
|
||||
"NodePassphraseSignature": "-----BEGIN PGP SIGNATURE-----\n\nwqsEABYIAF0Fgmr0dowJEOjFeJjESJCeNRQAAAAAABwAEHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmdAM0Z1Zs4qEGhmMqQerzOxFiEEovDWbdbGWAeVBhTm6MV4\nmMRIkJ4AAIacAP4qrIoEuIBveSpsDm4s5atvKIX6axrzdbpk2MmQ/ksCcAEAqj8c\natnZXVguH3HIXykGuYbKLLtgAyNIMFZtdpgDaAM=\n=75bw\n-----END PGP SIGNATURE-----",
|
||||
"SignatureEmail": "test@proton.test",
|
||||
"NameSignatureEmail": "test@proton.test",
|
||||
"MediaType": "text/plain",
|
||||
"ContentKeyPacket": "wWwGFQTuhgfuSeaElfhTcHuhVH9DcR42VBIBB0DG6NeRc4gJ42NpGhhANZZtVCsYSE1BBzntBkqT9yMTWjBP/6Xh1s7lDjQpnIsIBAiqEcEo5peTxmYHuk4Q6Ey6KpaKZcX/NWkINd3tnIXoMYw=",
|
||||
"ContentKeyPacketSignature": "-----BEGIN PGP SIGNATURE-----\n\nwqsEABYIAF0Fgmr0dowJENqteUib1K59NRQAAAAAABwAEHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmc/lk7jZ7uW7qKAVVgwiZCcFiEEvKrbb+RxgL0NxvP92q15\nSJvUrn0AAG4DAP0ZZC3VbaYn+JAB7KY6LKwVLYW1o8S1xvMX0E4RSaUT0QEAjjhy\neaHYH6+HyFxlviiCQiuTZngkkFxIX+6JcHcmRgQ=\n=ZUAK\n-----END PGP SIGNATURE-----"
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"ManifestSignature": "-----BEGIN PGP SIGNATURE-----\n\nwqsEABYIAF0Fgmr0dowJEOjFeJjESJCeNRQAAAAAABwAEHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmf3mNkUGB8MfMbXOgo6z4/WFiEEovDWbdbGWAeVBhTm6MV4\nmMRIkJ4AAKM8AP4t95tQdUeQrgGRbSaCO3vu8bD0n4h7tcU3myfG4jaHGwEAqGbo\nVu7KaVj4etYyw97OG4O3F9fyFytms8K+K4PEIgg=\n=4PZL\n-----END PGP SIGNATURE-----",
|
||||
"XAttr": "-----BEGIN PGP MESSAGE-----\n\nwV4DoVR/Q3EeNlQSAQdAQxqlBfMGZVnnZYjaKMAuuNX9i4IBx6hJrODap23PQjgw\noUJTKYzt7ptpa+hBOMO4WPHZzYoknwub4wXqkK7yUzVekl64pVw0kBIi58/BdddK\n0sDFAXW7t6Rc+GBNYiIG6+2n/fsb6zS7DdiaywNNnI11+/pFuZbFmKYO7mB8euKX\nqjkWfrtYscJvK/ARR5y4+R83LV2ncfipCYogbKiu/TJ1KdGnIlU4O5AzBuMUnALU\nKdY9sUgwKP6UZk22z1+/aTl20sqoK75Lx8recNmWi3YAy2W+rTUfFZ1lkOwqzO5U\n6SG4YjhjjLxaVQbIU/k+01NRyS1MeBHI1iQOXTx1wMUTDiYcGcaKJaI1zLT5LLpa\n1iB5fFHBwmmhYPX+9eoehnq/odTmrioXNFObualcm3lNWkqVZr68cyXol/OKbP4M\nQAkrCu1AXP63Kx58CPy2SZPp/e44/2MpxBpCjhMjPaFwlJp25SyfkbzLVpgl0cPE\nnN1DCwhKV00uxo/OXT3TyX3AZQzkHH87enjsUIErJtzYeQwCOayZ2OsDWEqun4Iz\nm7N7WRrKkBQzXi3fsiGmS7dn8nPZQLKVl1XTRlO2gLUbcgwncvha4KewcCQrm8jn\nsjHFFj3xIys=\n=cF+A\n-----END PGP MESSAGE-----",
|
||||
"SignatureEmail": "test@proton.test",
|
||||
"Blocks": [
|
||||
{
|
||||
"Hash": "l9BiOEe3LP5ocY7N9nje8b5prF1C7cvKC8i8WVNZVSU=",
|
||||
"EncSignature": "-----BEGIN PGP MESSAGE-----\n\nwV4DoVR/Q3EeNlQSAQdA+jC/DCMHLhuD7V07bcumt9RX8kd24DA0ZS+Hey7/rQww\nAXEuq4eGI/x9aNGq0/cKuVGfh3J9WiGIwf2fAe+xyMfwYdBUF0sJboRtZ7mTwEcP\n0sAeAdGSt1FmddNYRpeDyso3xCuERWnZPzO+hcS3OwcCrGW22t4bzvlc68BNyNZb\nwsp7aV/EKweMwSvB9ELkwjvGItXHCo6ch0Ugr67lGLSWFZcfO56Ji6KmYCKuRTDG\nm4uWZJIq0sCOwZHjiwlp7Uxv6ZBU5Q3bm694cyixAzeXPeQ6ZxK1vp9dIfO/FoTA\npiMk+iLN2ByGgBZaV637DpUfmu9+Aoop6JEhymioCSnG1DC4qp7vR/7009nNmftZ\nkicR4+h9JTXx5rDWWemIzCU5agGdSn7QoJm+d/kIM8F/\n=zUHV\n-----END PGP MESSAGE-----",
|
||||
"SignatureEmail": "test@proton.test"
|
||||
}
|
||||
]
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"Comment": "Folder written by go-proton-api v1.0.4 and Proton-API-Bridge v1.0.5 with test-only keys.",
|
||||
"Type": "folder",
|
||||
"ParentKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavRGjxYJKwYBBAHaRw8BAQdAcByzpVvdWeIqgDPVZmcsLXfHFl+Bc9dtNc/t\niBjn+IP+BwMIIaMaC0R//Jvgb8tK2b6Oodb3lCJtvh69ga4uP2ap0mz/xUrVex5v\nMXho5KNxYhW8EepuPH0NiYzjpUvWSEiheC7+EXeY6fGTDcjY83Y4gc0iRHJpdmUg\na2V5IDxub3JlcGx5QHByb3Rvbm1haWwuY29tPsK/BBMWCABxBYJq9EaPAwsJBwkQ\n/w8HJtPlPM81FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ8De\n3CViT8Q6GeqHBeto3S0CFQgDFgACAhkBApsDAh4BFiEE2zfp6URSivY361ih/w8H\nJtPlPM8AANBzAP9p254j7U0XtUMTXtUMAZsFNMfEUUIONbMJ8O+ZWaIKOwEAh3Cw\nz++d8p+bJw+7ovbZrBQybAJVDgSxMrD55HdvkA7HiwRq9EaPEgorBgEEAZdVAQUB\nAQdAS6pel/FW2xZBlcnqiOP1Qb9Z+wIYWp5rHbWhrjCjPxcDAQoJ/gcDCCGjGgtE\nf/yb4O3SqVDOQcANB/1usPldBbIAqfyrHrBYhl7I22qJ8C2dA/0Jd8QZJQ2piogy\nLVCxhigyc7I/khvo0DQtm6yEiG4Ba96d4gDCrgQYFggAYAWCavRGjwkQ/w8HJtPl\nPM81FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ1BBLxxRm18q\nrPmFuxepoKcCmwwWIQTbN+npRFKK9jfrWKH/Dwcm0+U8zwAAriMA/j6dCxHqJGV4\nH7mPiw3MRSgXs3fBfoIbED3GNUnDd1qSAPsFnc+gFS4rVj6ISFeZi9qKSBUkshY8\nODgZOvvp282WCA==\n=d0Pu\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"ParentKeyPassphrase": "epXvUdjEr7MP281MqKlqtLTP8pTp3W8Pd06oK5O5mUI=",
|
||||
"AddressKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavRGjxYJKwYBBAHaRw8BAQdAjfCNZ1vHeSII9FgMPuIpksy8Ru1IYogM3kt5\nbLkbvSr+BwMImAfCaC1ZpIDgOjjorDRLeZeXJmiY4g7sHfWioT/n+KyhQHIejdCX\niwVsun/SAbywJDI410K9rSZc5EgoOjq1AY+p7PkKlVKX6N9drVAF8M0XdGVzdCA8\ndGVzdEBwcm90b24udGVzdD7CvwQTFggAcQWCavRGjwMLCQcJEMrE2wl491S0NRQA\nAAAAABwAEHNhbHRAbm90YXRpb25zLm9wZW5wZ3Bqcy5vcmf5qKaaNXG7tbQ22Gyg\n+0spAhUIAxYAAgIZAQKbAwIeARYhBBt9DXGs6YlhOFA8fcrE2wl491S0AAB4GgD9\nHPYJYDYYlrRIiXSIxwSMoA8N2qFF3dZu+JclcNJkgvQA/A19w5X8hDkSS8JIpxw3\nRZrRTYn84h2sLpc/UaGpfGMFx4sEavRGjxIKKwYBBAGXVQEFAQEHQLaoMtcytvD+\n3VWXtk86FmWAj3lGf6geHDROSdDoVXknAwEKCf4HAwiYB8JoLVmkgOCl5KYyiazU\n36OQT3lv1PN7vNGqhXKinkl3/HCyzZ9XypwUTX4iM3gMxnEcBhM2JzJgQH430u4s\nrEhT+/Bi2+B7LmsblwOxwq4EGBYIAGAFgmr0Ro8JEMrE2wl491S0NRQAAAAAABwA\nEHNhbHRAbm90YXRpb25zLm9wZW5wZ3Bqcy5vcmdPaYn0wfSZLPYbNHOiCY07ApsM\nFiEEG30NcazpiWE4UDx9ysTbCXj3VLQAAKMgAP93VZvAugvAKI3VfBnuUaLyob1j\n+hilZNaVvotWIG12nQEA9zvN8uLVIefMJRwJucbdJLN5NhDnbUerpi8BuCA+vw4=\n=aZOL\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"AddressKeyPassphrase": "LxTkQtWVXufn2S4L0TCVgkK1rIU5jbppjuUZYx8FWBc=",
|
||||
"NodeKey": "-----BEGIN PGP PRIVATE KEY BLOCK-----\n\nxYYEavRGjxYJKwYBBAHaRw8BAQdAgoNl19ESfrQk3oxuSFdeLFiEzaR3oyIoTVkb\nrRAy2av+BwMIbSMlPfZXu5PgW8A+jsuCxlebNv54CSUKk09Je3SIJvnzS7jS1V0k\naZvresVslQFzx0MHpr/yeB/+7TPR5dzuLmXd+aGgdqkC6SckRBoFZM0iRHJpdmUg\na2V5IDxub3JlcGx5QHByb3Rvbm1haWwuY29tPsK/BBMWCABxBYJq9EaPAwsJBwkQ\nfCiqEgYU2gU1FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ+cY\nD6U5omic+nTFDziv0jACFQgDFgACAhkBApsDAh4BFiEEdiCJ15XfeWq8CFT2fCiq\nEgYU2gUAAOR7AP0Ww8YYcvQgrTviO3apsVEZ9q3S6sUDsI29+Bqj1n4C+AEA+e5N\nSPzQnzp5SxU0x3TUxEbCbtwJxi8NCHm/Bc/4AQbHiwRq9EaPEgorBgEEAZdVAQUB\nAQdAvvwZKOA/WYuQVsafq2YaPExJg4gKOJTpkwdATIUkcW0DAQoJ/gcDCG0jJT32\nV7uT4BD4YWn+qvsjXQb3yXpa2rSDpObO3IKs51BiSBzkwhvuZeQ0j/O0W0w6d1ne\n2z5wNfFobuQvPStR/FrE8HcxM2j/vvpPXNXCrgQYFggAYAWCavRGjwkQfCiqEgYU\n2gU1FAAAAAAAHAAQc2FsdEBub3RhdGlvbnMub3BlbnBncGpzLm9yZ6t1L01T3l1V\nMHJ8evZ4088CmwwWIQR2IInXld95arwIVPZ8KKoSBhTaBQAAwxIBANPX36Lsy88R\ngm7CeRx0oUFVMrvVTPoik525Kc4o9GhCAP4pKY9o9jYMZu3SKgK2uczL8COEE5NN\nDngG7LerEf6NAQ==\n=fklD\n-----END PGP PRIVATE KEY BLOCK-----",
|
||||
"NodePassphrase": "-----BEGIN PGP MESSAGE-----\n\nwV4DWp6jv64eIygSAQdAFKUkba5JwRWgNoAP15SVnd0+LIj+gqv04dy89Wrcqjcw\nBoEKO4cg+EoKp8VZ/NkGnpvDGU3NYfd/wWhjL/pk5O08Sq+hy5iGxfOjjRw87Y4U\n0l0BlVAuSvMd4jOnDXZjsHu9ADtRoAD7IyVtr98eKmkY3/qz/Ox1L069MkTUf19j\nElDiOZmcfklfMlmMJvHQVtmwfgiXRdtto/kNFejNfBtSxF6AJw0s0LedtHyftSU=\n=tho6\n-----END PGP MESSAGE-----",
|
||||
"NodePassphraseSignature": "-----BEGIN PGP SIGNATURE-----\n\nwqsEABYIAF0Fgmr0Ro8JEMrE2wl491S0NRQAAAAAABwAEHNhbHRAbm90YXRpb25z\nLm9wZW5wZ3Bqcy5vcmcz9ZAF6bobxc93L+dEnb6ZFiEEG30NcazpiWE4UDx9ysTb\nCXj3VLQAAO67AP4o6cOdU9EdMFbuavjw+i+GOE2WrFqwuf7gl+/jk7hFgAD+Ll8q\n5ZtTs4DQaf10P6P94tL5eqlO4j9jJxBxrX5VnAc=\n=52X6\n-----END PGP SIGNATURE-----",
|
||||
"NodePassphrasePlain": "porsaPWJDYpZlXNOUaiyE9uz+Z4IVpaqppO4rz1Iy/A=",
|
||||
"Name": "-----BEGIN PGP MESSAGE-----\n\nwV4DWp6jv64eIygSAQdAD9q58a7P6MpyqXT3QEI62k4j1hwx68MO+smShKOyZjgw\nDqSyKqnQzixY386+aKQq7l7rlrnSLqALfsnS4bjdSGf1+yu7+ut0eHyKkDc4UQFx\n0sBCAdBVEh8oc/f86Z+BCEOyWL2ffhWYdNQNyCue3jXTrZt0UrbADWijwDMmhL05\n/29TSLf5XcvR5lWfxWWZI+3YCa9upo3Gb4mIpDQuUmHaagzn6TC4HlL9GyG5CaEx\nfFu9Ra93myXTtDwhm/ysUjkguNfMr5VwjARfJmuKac6TpyjSzOh+a7KW/kN094iK\nHvMmO9PTCMSIKALFBRKlEBllqMhWzpiu533ykHW7u7JfzDRm+aSxT48YrfbibF3D\nQVRCiQOLMU3NuwtMwQxcentzCKwNvPTyxMqG9m4IrwFO3CU84hHpnPo0sTeemAp/\nBrum0uIpd3fACmTzT1ZgsKB7S4Mi\n=uiCU\n-----END PGP MESSAGE-----",
|
||||
"NamePlain": "Fichiers d'été 📁",
|
||||
"NodeHashKey": "-----BEGIN PGP MESSAGE-----\n\nwV4DIMIraMSXSp4SAQdAahEjiRUe66lBrrIDwZCkrZ3SbdkVD775SsblwvMFWgEw\nVQ3SJsiAxfYlYivvtDSuT0fR8jfcsJjUuxtE2qUKvogUBo0CxBwuJ5JvCr12oRUf\n0sBlAbdKXFsKDr423bt4NbOOKKgY3SJcfjEbylMA/uEP/t3OLbTn6feKplhgWtoA\nKuI66xgdkPkW+KTg65Ju8GrQISFepYv6mF0VVzuKmLsgyBJwbCvOvTH6X/EnzOAG\nWvcjAEXyBtmQkEkOsNU2CdzfFDOvD7N5bquFUIdM+Mq6gEDUNxBZg3JkPrELe2Ll\nZfeWsdXRvRnliuLn9OeCoUCBIVtIRfUdMu+/qCnE2G9UHc70fiZtOJdNNnJBnKlo\n8kQKG28vZ+NlUtdRGQ9Aoh4qQbRyqKIg209BhuetlTYiWX2odrSXey5HuCkyxXWc\nTdJd+bVOZ0KQtreKAxW5GqWvWYGpkHMpCJ5oHxMABfa4HAJivTtOkRRSkQSue2cj\nD714MR8nb1A=\n=ocOA\n-----END PGP MESSAGE-----",
|
||||
"HashKeyPlain": "i49BgOd0lq0m/q3GWVY3UxmraZAKUmqsZcZCQXznf00=",
|
||||
"ChildName": "Document.txt",
|
||||
"ChildNameHash": "f3d4d9f7ee96a050b95cfd441e91258685cf57fc09fa55fea31e3a68b179d9b4"
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package protondrive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/ProtonMail/gopenpgp/v3/crypto"
|
||||
"github.com/rclone/rclone/backend/protondrive/protontest"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// newTestClient returns a client with caching enabled for a new user
|
||||
// of a new fake server, signed in without a login round trip.
|
||||
func newTestClient(t testing.TB) (*client, *protontest.Server, *protontest.User) {
|
||||
t.Helper()
|
||||
return newTestClientOpt(t, clientOptions{EnableCaching: true})
|
||||
}
|
||||
|
||||
// newTestClientOpt is newTestClient with the options opt.
|
||||
func newTestClientOpt(t testing.TB, opt clientOptions) (*client, *protontest.Server, *protontest.User) {
|
||||
t.Helper()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
return newTestClientFor(t, srv, u, opt), srv, u
|
||||
}
|
||||
|
||||
// newTestClientFor returns a client for a new session of u.
|
||||
func newTestClientFor(t testing.TB, srv *protontest.Server, u *protontest.User, opt clientOptions) *client {
|
||||
t.Helper()
|
||||
c, err := newClient(context.Background(), newAccountSession(srv, u), opt)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(c.close)
|
||||
return c
|
||||
}
|
||||
|
||||
// decryptBlock returns the encrypted block enc (SEIPDv1 or v2)
|
||||
// decrypted with the content session key sk.
|
||||
func decryptBlock(enc []byte, sk *crypto.SessionKey) ([]byte, error) {
|
||||
r, err := blockDecrypter(bytes.NewReader(enc), sk)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plain, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decrypt block: %w", err)
|
||||
}
|
||||
return plain, nil
|
||||
}
|
||||
@@ -0,0 +1,975 @@
|
||||
package protondrive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ProtonMail/gopenpgp/v3/crypto"
|
||||
"github.com/rclone/rclone/backend/protondrive/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// errUploadMismatch is returned when the bytes read from the source of
|
||||
// an upload don't match its declared size or SHA1.
|
||||
var errUploadMismatch = errors.New("source doesn't match its declared size or hash")
|
||||
|
||||
// errUploadOutcomeUnknown is returned when the commit of an upload
|
||||
// failed and whether it took effect can't be found out.
|
||||
var errUploadOutcomeUnknown = errors.New("upload outcome unknown: it may have succeeded")
|
||||
|
||||
// errUploadSuperseded is returned when an upload was committed but the
|
||||
// file read back has another revision, which is not this upload's.
|
||||
var errUploadSuperseded = errors.New("upload committed, but the file now has another revision")
|
||||
|
||||
// errUploadUnconfirmed is returned when an upload was committed but the
|
||||
// server still didn't show it when the file was last read back.
|
||||
var errUploadUnconfirmed = errors.New("upload committed, but the server doesn't show it yet")
|
||||
|
||||
const (
|
||||
// defaultUploadConcurrency and maxUploadConcurrency bound the blocks
|
||||
// of one file being uploaded at once, as in the SDK.
|
||||
defaultUploadConcurrency = 4
|
||||
maxUploadConcurrency = 5
|
||||
|
||||
// blockUploadAttempts is how many times uploading a block is tried.
|
||||
blockUploadAttempts = 3
|
||||
|
||||
// blockSealAttempts is how many times a block whose encryption
|
||||
// fails its check is encrypted, as in the SDK.
|
||||
blockSealAttempts = 2
|
||||
|
||||
// intendedSizePrecision is the precision of the size an upload
|
||||
// declares in advance, which must not tell the server the size.
|
||||
intendedSizePrecision = 100_000
|
||||
|
||||
// draftCleanupTimeout bounds deleting the draft of a failed upload.
|
||||
draftCleanupTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
// uploadRequest is a file to upload.
|
||||
type uploadRequest struct {
|
||||
ParentID string // folder to create the file in; unused if Existing is set
|
||||
Name string // name of the new file; unused if Existing is set
|
||||
Existing *node // file to give a new revision; nil for a new file
|
||||
In io.Reader
|
||||
Size int64 // bytes In holds; < 0 if unknown
|
||||
ModTime time.Time // zero if unknown
|
||||
MIMEType string
|
||||
SHA1 string // hex SHA-1 of the bytes In holds; "" if unknown
|
||||
}
|
||||
|
||||
// draft is an uncommitted revision being uploaded.
|
||||
type draft struct {
|
||||
nodeID string
|
||||
revisionID string
|
||||
parentID string
|
||||
newFile bool // the node is a draft too, not just the revision
|
||||
baseRevisionID string // active revision the draft revision was created from; new revisions only
|
||||
nodeKR *crypto.KeyRing
|
||||
ownKeys bool // nodeKR was generated for the draft and is cleared with it
|
||||
sk *crypto.SessionKey // content key
|
||||
}
|
||||
|
||||
// upload uploads r as a new file, or a new revision of r.Existing, as
|
||||
// the SDK's upload module does, and returns the file.
|
||||
//
|
||||
// A new file whose name is taken by a file is uploaded as a new
|
||||
// revision of it. A draft of the name left by this client (by
|
||||
// DraftClientUID) is deleted and the upload tried again; one left by
|
||||
// another client is errExists unless ReplaceExistingDraft is set. If
|
||||
// the bytes read differ from r.Size or r.SHA1 the upload fails with
|
||||
// errUploadMismatch.
|
||||
//
|
||||
// A failed upload deletes its draft only when nothing may have been
|
||||
// committed: before the commit, or when the server refused the commit
|
||||
// with a 4xx API error on its only attempt. Otherwise nothing is deleted
|
||||
// and a failed commit whose revision the file read back doesn't show is
|
||||
// errUploadOutcomeUnknown: the upload may have succeeded. A commit which
|
||||
// succeeded but whose file has another revision by the time it is read
|
||||
// back is errUploadSuperseded, and one the server still doesn't show
|
||||
// after a few reads is errUploadUnconfirmed. The returned node is always
|
||||
// the upload's own revision.
|
||||
func (c *client) upload(ctx context.Context, r uploadRequest) (*node, error) {
|
||||
what := fmt.Sprintf("upload %q", r.Name)
|
||||
if r.Existing != nil {
|
||||
what = fmt.Sprintf("upload %q", r.Existing.Name)
|
||||
}
|
||||
signKR, err := c.addr.signingKeyRing()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", what, err)
|
||||
}
|
||||
d, err := c.createDraft(ctx, r, signKR)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", what, err)
|
||||
}
|
||||
if d.ownKeys {
|
||||
defer d.nodeKR.ClearPrivateParams()
|
||||
}
|
||||
n, committed, err := c.uploadDraft(ctx, d, r, signKR)
|
||||
if err != nil {
|
||||
if !committed {
|
||||
c.deleteDraft(ctx, d)
|
||||
}
|
||||
return nil, fmt.Errorf("%s: %w", what, err)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// draftClientUID returns the ClientUID marking the client's drafts, or
|
||||
// nil if it has none.
|
||||
func (c *client) draftClientUID() *string {
|
||||
if c.opt.DraftClientUID == "" {
|
||||
return nil
|
||||
}
|
||||
uid := c.opt.DraftClientUID
|
||||
return &uid
|
||||
}
|
||||
|
||||
// derivedDraftClientUID returns a draft client ID for the remote name
|
||||
// derived from the name and the account's volume, so it is the same on
|
||||
// every run.
|
||||
func (c *client) derivedDraftClientUID(name string) string {
|
||||
sum := sha256.Sum256([]byte("rclone draft client\x00" + c.volumeID + "\x00" + name))
|
||||
return hex.EncodeToString(sum[:16])
|
||||
}
|
||||
|
||||
// uploadConcurrency returns the most blocks of one file to upload at
|
||||
// once.
|
||||
func (c *client) uploadConcurrency() int {
|
||||
if c.opt.UploadConcurrency <= 0 {
|
||||
return defaultUploadConcurrency
|
||||
}
|
||||
return min(c.opt.UploadConcurrency, maxUploadConcurrency)
|
||||
}
|
||||
|
||||
// draftIntendedSize returns the IntendedUploadSize of a new file of
|
||||
// size bytes as the SDK's createDraft sends it: rounded down to
|
||||
// intendedSizePrecision, or nil if that is at most the precision or the
|
||||
// size is unknown.
|
||||
func draftIntendedSize(size int64) *int64 {
|
||||
rounded := size / intendedSizePrecision * intendedSizePrecision
|
||||
if size < 0 || rounded <= intendedSizePrecision {
|
||||
return nil
|
||||
}
|
||||
return &rounded
|
||||
}
|
||||
|
||||
// revisionIntendedSize returns the IntendedUploadSize of a new revision
|
||||
// of size bytes as the SDK's createDraftRevision sends it
|
||||
// (reduceSizePrecision), or nil if the size is zero or unknown.
|
||||
func revisionIntendedSize(size int64) *int64 {
|
||||
var v int64
|
||||
switch {
|
||||
case size <= 0:
|
||||
return nil
|
||||
case size < 4096:
|
||||
v = 4095
|
||||
case size < intendedSizePrecision:
|
||||
v = intendedSizePrecision
|
||||
default:
|
||||
v = size / intendedSizePrecision * intendedSizePrecision
|
||||
}
|
||||
return &v
|
||||
}
|
||||
|
||||
// createDraft creates the draft to upload r to: a draft file in
|
||||
// r.ParentID, or a draft revision of r.Existing or of the file which
|
||||
// has the name.
|
||||
func (c *client) createDraft(ctx context.Context, r uploadRequest, signKR *crypto.KeyRing) (*draft, error) {
|
||||
if r.Existing != nil {
|
||||
return c.createRevision(ctx, r.Existing, r.Size)
|
||||
}
|
||||
if err := validateName(r.Name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
parent, err := c.targetFolder(ctx, r.ParentID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := c.s.serverTime()
|
||||
nodeKey, passphrase, passphraseSig, keys, err := generateNodeKeys(parent.keys.keyRing, signKR, true, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
created := false
|
||||
defer func() {
|
||||
if !created {
|
||||
keys.keyRing.ClearPrivateParams()
|
||||
}
|
||||
}()
|
||||
contentKeyPacket, contentKeyPacketSig, sk, err := generateContentKey(keys.keyRing, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
encName, err := encryptName(r.Name, nil, parent.keys.keyRing, signKR, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mimeType := r.MIMEType
|
||||
if mimeType == "" {
|
||||
mimeType = "application/octet-stream"
|
||||
}
|
||||
req := &api.CreateFileRequest{
|
||||
ParentLinkID: parent.ID,
|
||||
Name: encName,
|
||||
Hash: nameHash(r.Name, parent.hashKey),
|
||||
MIMEType: mimeType,
|
||||
ClientUID: c.draftClientUID(),
|
||||
IntendedUploadSize: draftIntendedSize(r.Size),
|
||||
NodeKey: nodeKey,
|
||||
NodePassphrase: passphrase,
|
||||
NodePassphraseSignature: passphraseSig,
|
||||
ContentKeyPacket: contentKeyPacket,
|
||||
ContentKeyPacketSignature: contentKeyPacketSig,
|
||||
SignatureAddress: c.addr.Email,
|
||||
}
|
||||
opts := &rest.Opts{Method: "POST", Path: c.volumePath("/drive/v2/volumes") + "/files"}
|
||||
for retried := false; ; retried = true {
|
||||
var resp api.CreateFileResponse
|
||||
err = c.apiCall(ctx, opts, req, &resp)
|
||||
if err == nil {
|
||||
created = true
|
||||
return &draft{
|
||||
nodeID: resp.File.ID,
|
||||
revisionID: resp.File.RevisionID,
|
||||
parentID: parent.ID,
|
||||
newFile: true,
|
||||
nodeKR: keys.keyRing,
|
||||
ownKeys: true,
|
||||
sk: sk,
|
||||
}, nil
|
||||
}
|
||||
existing, err := c.resolveConflict(ctx, err, retried)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if existing != nil {
|
||||
return c.createRevision(ctx, existing, r.Size)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// resolveConflict decides what to do after creating a draft failed
|
||||
// with err, as the SDK's handleConflictError does. It returns the file
|
||||
// to upload a new revision of if the name is taken by a file, or nil
|
||||
// and no error to try again once a draft in the way is deleted.
|
||||
//
|
||||
// A draft is deleted if it was left by this client or
|
||||
// ReplaceExistingDraft is set, unless retried is set: a draft which
|
||||
// comes back is another client's upload in progress.
|
||||
func (c *client) resolveConflict(ctx context.Context, err error, retried bool) (*node, error) {
|
||||
var apiErr *api.Error
|
||||
if !errors.As(err, &apiErr) || apiErr.Code != api.CodeAlreadyExists {
|
||||
return nil, fmt.Errorf("create draft: %w", err)
|
||||
}
|
||||
var conflict api.ConflictDetails
|
||||
if json.Unmarshal(apiErr.Details, &conflict) != nil || conflict.ConflictLinkID == "" {
|
||||
return nil, mutationError("create draft", err)
|
||||
}
|
||||
if conflict.ConflictRevisionID != "" || conflict.ConflictDraftRevisionID == "" {
|
||||
// Only a file, which has an active revision, can take a new
|
||||
// revision. A draft revision of it doesn't stop that.
|
||||
c.cache.invalidate(conflict.ConflictLinkID)
|
||||
n, getErr := c.getNode(ctx, conflict.ConflictLinkID)
|
||||
if getErr != nil {
|
||||
return nil, fmt.Errorf("create draft: %w", getErr)
|
||||
}
|
||||
if n.IsFolder || n.State != api.LinkStateActive || n.RevisionID == "" {
|
||||
return nil, mutationError("create draft", err)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
own := c.opt.DraftClientUID != "" && conflict.ConflictDraftClientUID == c.opt.DraftClientUID
|
||||
switch {
|
||||
case retried:
|
||||
return nil, fmt.Errorf("create draft: a draft of the file came back after it was deleted: %w", mutationError("another client is uploading it", err))
|
||||
case !own && !c.opt.ReplaceExistingDraft:
|
||||
return nil, fmt.Errorf("create draft: another client is uploading the file (set replace_existing_draft to replace its draft): %w", mutationError("draft exists", err))
|
||||
}
|
||||
fs.Debugf(nil, "protondrive: deleting draft %s left by client %q", conflict.ConflictLinkID, conflict.ConflictDraftClientUID)
|
||||
if delErr := c.deleteDraftLink(ctx, conflict.ConflictLinkID); delErr != nil {
|
||||
return nil, fmt.Errorf("create draft: %w (deleting the draft in the way failed: %v)", mutationError("draft exists", err), delErr)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// createRevision creates a draft revision of file n, which must be
|
||||
// unchanged since it was read.
|
||||
func (c *client) createRevision(ctx context.Context, n *node, size int64) (*draft, error) {
|
||||
if n.IsFolder || n.link == nil || n.link.File == nil || n.RevisionID == "" {
|
||||
return nil, fmt.Errorf("%q is not a file with content", n.Name)
|
||||
}
|
||||
sk, err := c.contentKey(n, "updating")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req := &api.CreateRevisionRequest{
|
||||
CurrentRevisionID: n.RevisionID,
|
||||
ClientUID: c.draftClientUID(),
|
||||
IntendedUploadSize: revisionIntendedSize(size),
|
||||
}
|
||||
var resp api.CreateRevisionResponse
|
||||
err = c.apiCall(ctx, &rest.Opts{Method: "POST", Path: c.volumePath("/drive/v2/volumes", "files", n.ID, "revisions")}, req, &resp)
|
||||
c.cache.invalidate(n.ID)
|
||||
if err != nil {
|
||||
return nil, mutationError("create revision", err)
|
||||
}
|
||||
return &draft{
|
||||
nodeID: n.ID,
|
||||
revisionID: resp.Revision.ID,
|
||||
baseRevisionID: n.RevisionID,
|
||||
parentID: n.ParentID,
|
||||
nodeKR: n.keys.keyRing,
|
||||
sk: sk,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// uploadResult is what was read from the source of an upload and
|
||||
// uploaded.
|
||||
type uploadResult struct {
|
||||
size int64
|
||||
sha1 string // hex
|
||||
hashes [][]byte // SHA-256 of each encrypted block, in order
|
||||
blockSizes []int64 // plaintext size of each block, in order
|
||||
}
|
||||
|
||||
// check checks res against the size and SHA1 r declares, and its block
|
||||
// count against its size.
|
||||
func (res *uploadResult) check(r uploadRequest) error {
|
||||
switch {
|
||||
case r.Size >= 0 && res.size != r.Size:
|
||||
return fmt.Errorf("%w: read %d bytes, expected %d", errUploadMismatch, res.size, r.Size)
|
||||
case r.SHA1 != "" && !strings.EqualFold(r.SHA1, res.sha1):
|
||||
return fmt.Errorf("%w: read bytes with SHA1 %s, expected %s", errUploadMismatch, res.sha1, r.SHA1)
|
||||
case int64(len(res.hashes)) != (res.size+defaultBlockSize-1)/defaultBlockSize:
|
||||
return fmt.Errorf("%d blocks uploaded for %d bytes", len(res.hashes), res.size)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadDraft uploads the content of r to draft d and commits it,
|
||||
// returning the committed file. committed reports whether the revision
|
||||
// may be committed, even if err is set, when the draft must not be
|
||||
// deleted: committed is false only when the commit wasn't sent or the
|
||||
// server refused it on its only attempt. What a read shows never proves
|
||||
// a commit didn't take effect: reads may lag.
|
||||
func (c *client) uploadDraft(ctx context.Context, d *draft, r uploadRequest, signKR *crypto.KeyRing) (n *node, committed bool, err error) {
|
||||
code, verifySK, err := c.verificationData(ctx, d)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
res, err := c.uploadBlocks(ctx, d, r, code, verifySK, signKR)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if err := res.check(r); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
now := c.s.serverTime()
|
||||
manifest, err := signManifest(res.hashes, signKR, now)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
x := &xattrs{Size: res.size, BlockSizes: res.blockSizes, SHA1: res.sha1}
|
||||
if !r.ModTime.IsZero() {
|
||||
x.ModTime = r.ModTime.UTC()
|
||||
}
|
||||
encXAttrs, err := encryptXAttrs(x, d.nodeKR, signKR, now)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
req := &api.CommitRevisionRequest{
|
||||
ManifestSignature: manifest,
|
||||
SignatureAddress: c.addr.Email,
|
||||
XAttr: encXAttrs,
|
||||
ChecksumVerified: r.SHA1 != "",
|
||||
}
|
||||
opts := &rest.Opts{Method: "PUT", Path: c.volumePath("/drive/v2/volumes", "files", d.nodeID, "revisions", d.revisionID)}
|
||||
attempts := 0
|
||||
commitErr := c.pacedCall(ctx, c.apiPacer, func() (*http.Response, error) {
|
||||
attempts++
|
||||
return c.s.call(ctx, opts, req, nil)
|
||||
})
|
||||
if commitErr != nil && attempts == 1 && refusedCommit(commitErr) {
|
||||
c.forget(d.nodeID, d.parentID)
|
||||
return nil, false, fmt.Errorf("commit: %w", commitErr)
|
||||
}
|
||||
// Whether a failed commit took effect is read even if ctx is done,
|
||||
// though only once then, as a commit which did is a success.
|
||||
readCtx, cancel := context.WithTimeout(ctx, draftCleanupTimeout)
|
||||
defer cancel()
|
||||
if commitErr != nil && ctx.Err() != nil {
|
||||
detached, cancelDetached := context.WithTimeout(context.WithoutCancel(ctx), draftCleanupTimeout)
|
||||
defer cancelDetached()
|
||||
n, err = c.fetchNode(detached, d.nodeID)
|
||||
} else {
|
||||
n, err = c.readBack(readCtx, d.nodeID, func(n *node, err error) bool { return !d.lagging(n, err) })
|
||||
}
|
||||
ours := err == nil && n.State == api.LinkStateActive && n.RevisionID == d.revisionID
|
||||
if ours {
|
||||
c.cache.add(n)
|
||||
} else {
|
||||
c.forget(d.nodeID, d.parentID)
|
||||
}
|
||||
switch {
|
||||
case ours && commitErr != nil:
|
||||
// The response of a commit which succeeded may have been lost,
|
||||
// and its retry then failed.
|
||||
fs.Debugf(nil, "protondrive: %q (ID %s): commit failed but the revision is active: %v", n.Name, n.ID, commitErr)
|
||||
return n, true, nil
|
||||
case ours:
|
||||
return n, true, nil
|
||||
case commitErr == nil && (d.lagging(n, err) || ctx.Err() != nil):
|
||||
fs.Debugf(nil, "protondrive: node %s: revision %s committed, but the server doesn't show it yet", d.nodeID, d.revisionID)
|
||||
return nil, true, errUploadUnconfirmed
|
||||
case commitErr == nil && err != nil:
|
||||
return nil, true, fmt.Errorf("read the uploaded file: %w", err)
|
||||
case commitErr == nil:
|
||||
return nil, true, fmt.Errorf("%w (the file is in state %d with revision %q active, not %s)", errUploadSuperseded, n.State, n.RevisionID, d.revisionID)
|
||||
case err != nil:
|
||||
// Any draft left is replaced by the next upload of the name.
|
||||
return nil, true, fmt.Errorf("commit: %w: %w (reading the file failed: %v)", errUploadOutcomeUnknown, commitErr, err)
|
||||
}
|
||||
// A read may lag, or the file have changed since, so what it shows
|
||||
// doesn't prove the commit didn't take effect.
|
||||
return nil, true, fmt.Errorf("commit: %w: %w (the file doesn't show the revision)", errUploadOutcomeUnknown, commitErr)
|
||||
}
|
||||
|
||||
// refusedCommit reports whether err, from a commit sent only once, is
|
||||
// the server refusing it, so that it didn't take effect: an API error
|
||||
// with a 4xx status other than 408 and a Proton code.
|
||||
func refusedCommit(err error) bool {
|
||||
var apiErr *api.Error
|
||||
return errors.As(err, &apiErr) && apiErr.Code != 0 &&
|
||||
apiErr.Status >= 400 && apiErr.Status < 500 && apiErr.Status != http.StatusRequestTimeout
|
||||
}
|
||||
|
||||
// lagging reports whether n and err, read after the commit of d, may
|
||||
// be from before the server shows the commit: the node isn't found, is
|
||||
// active without an active revision, or looks as it did before (a new
|
||||
// file still a draft, or the revision d started from still active).
|
||||
func (d *draft) lagging(n *node, err error) bool {
|
||||
switch {
|
||||
case errors.Is(err, errNotFound):
|
||||
return true
|
||||
case err != nil:
|
||||
return false
|
||||
case n.State == api.LinkStateActive && n.RevisionID == "":
|
||||
return true
|
||||
case d.newFile:
|
||||
return n.State == api.LinkStateDraft
|
||||
}
|
||||
return n.State == api.LinkStateActive && n.RevisionID == d.baseRevisionID
|
||||
}
|
||||
|
||||
// verificationData returns the verification code of draft d and the
|
||||
// content key the server holds for it, which blocks are checked
|
||||
// against as in the SDK.
|
||||
func (c *client) verificationData(ctx context.Context, d *draft) (code []byte, sk *crypto.SessionKey, err error) {
|
||||
var resp api.VerificationResponse
|
||||
opts := &rest.Opts{Method: "GET", Path: c.volumePath("/drive/v2/volumes", "links", d.nodeID, "revisions", d.revisionID, "verification")}
|
||||
if err := c.apiCall(ctx, opts, nil, &resp); err != nil {
|
||||
return nil, nil, fmt.Errorf("get verification data: %w", err)
|
||||
}
|
||||
code, err = base64.StdEncoding.DecodeString(resp.VerificationCode)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("get verification data: %w", err)
|
||||
}
|
||||
sk, _, err = decryptContentKey(resp.ContentKeyPacket, "", d.nodeKR, nil, c.s.serverTime())
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("get verification data: %w", err)
|
||||
}
|
||||
return code, sk, nil
|
||||
}
|
||||
|
||||
// encryptedBlock is a block of a draft, encrypted and checked, which
|
||||
// holds its share of the upload concurrency until freed.
|
||||
type encryptedBlock struct {
|
||||
index int
|
||||
enc *pool.RW
|
||||
encSignature string
|
||||
token []byte // verifier token
|
||||
release func() // gives back the share of the upload concurrency
|
||||
}
|
||||
|
||||
// free frees the block's buffer and its share of the upload
|
||||
// concurrency.
|
||||
func (b *encryptedBlock) free() {
|
||||
_ = b.enc.Close()
|
||||
b.release()
|
||||
}
|
||||
|
||||
// freeBlocks frees every block of blocks.
|
||||
func freeBlocks(blocks []*encryptedBlock) {
|
||||
for _, b := range blocks {
|
||||
b.free()
|
||||
}
|
||||
}
|
||||
|
||||
// blockUpload is a block with the URL to upload it to.
|
||||
type blockUpload struct {
|
||||
block *encryptedBlock
|
||||
link api.UploadLink
|
||||
}
|
||||
|
||||
// uploadBlocks reads, encrypts and uploads the content of r to draft
|
||||
// d: one goroutine reads and encrypts blocks, one requests their upload
|
||||
// URLs in batches, and uploadConcurrency workers upload them. Every
|
||||
// wait is bounded by ctx.
|
||||
func (c *client) uploadBlocks(ctx context.Context, d *draft, r uploadRequest, code []byte, verifySK *crypto.SessionKey, signKR *crypto.KeyRing) (*uploadResult, error) {
|
||||
concurrency := c.uploadConcurrency()
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
ready := make(chan *encryptedBlock)
|
||||
uploads := make(chan blockUpload)
|
||||
res := &uploadResult{}
|
||||
g.Go(func() error {
|
||||
defer close(ready)
|
||||
return c.readBlocks(gCtx, d, r, code, verifySK, signKR, concurrency, res, ready)
|
||||
})
|
||||
g.Go(func() error {
|
||||
defer close(uploads)
|
||||
return c.requestURLs(gCtx, d, ready, uploads)
|
||||
})
|
||||
for range concurrency {
|
||||
g.Go(func() error {
|
||||
for {
|
||||
var u blockUpload
|
||||
var ok bool
|
||||
select {
|
||||
case u, ok = <-uploads:
|
||||
case <-gCtx.Done():
|
||||
return gCtx.Err()
|
||||
}
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
err := c.uploadBlock(gCtx, d, u)
|
||||
u.block.free()
|
||||
if err != nil {
|
||||
return fmt.Errorf("block %d: %w", u.block.index, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
if err := g.Wait(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// readBlocks reads r.In block by block into pool buffers, encrypts and
|
||||
// checks each block and sends it to ready, recording what it read in
|
||||
// res. At most concurrency blocks of the upload, and blocksInFlight of
|
||||
// the client, are held at once.
|
||||
//
|
||||
// A source longer than r.Size fails with errUploadMismatch as soon as
|
||||
// that is seen.
|
||||
func (c *client) readBlocks(ctx context.Context, d *draft, r uploadRequest, code []byte, verifySK *crypto.SessionKey, signKR *crypto.KeyRing, concurrency int, res *uploadResult, ready chan<- *encryptedBlock) error {
|
||||
slots := make(chan struct{}, concurrency)
|
||||
digest := sha1.New()
|
||||
in := io.TeeReader(r.In, digest)
|
||||
for index := 1; ; index++ {
|
||||
release, err := c.reserveBlock(ctx, slots)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
plain := pool.NewRW(pool.Global())
|
||||
n, err := plain.ReadFrom(io.LimitReader(in, defaultBlockSize))
|
||||
res.size += n
|
||||
switch {
|
||||
case err != nil:
|
||||
err = fmt.Errorf("read source: %w", err)
|
||||
case r.Size >= 0 && res.size > r.Size:
|
||||
err = fmt.Errorf("%w: source is longer than %d bytes", errUploadMismatch, r.Size)
|
||||
}
|
||||
if err != nil || n == 0 {
|
||||
_ = plain.Close()
|
||||
release()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
break
|
||||
}
|
||||
b, hash, err := c.sealBlock(d, plain, index, code, verifySK, signKR)
|
||||
_ = plain.Close()
|
||||
if err != nil {
|
||||
release()
|
||||
return fmt.Errorf("block %d: %w", index, err)
|
||||
}
|
||||
b.release = release
|
||||
res.hashes = append(res.hashes, hash)
|
||||
res.blockSizes = append(res.blockSizes, n)
|
||||
select {
|
||||
case ready <- b:
|
||||
case <-ctx.Done():
|
||||
b.free()
|
||||
return ctx.Err()
|
||||
}
|
||||
if n < defaultBlockSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
res.sha1 = hex.EncodeToString(digest.Sum(nil))
|
||||
return nil
|
||||
}
|
||||
|
||||
// reserveBlock takes a share of the upload's concurrency from slots and
|
||||
// of the client's from uploadTokens, returning the function which
|
||||
// gives both back.
|
||||
func (c *client) reserveBlock(ctx context.Context, slots chan struct{}) (release func(), err error) {
|
||||
select {
|
||||
case slots <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
if err := acquireToken(ctx, c.uploadTokens); err != nil {
|
||||
<-slots
|
||||
return nil, err
|
||||
}
|
||||
return func() {
|
||||
c.uploadTokens.Put()
|
||||
<-slots
|
||||
}, nil
|
||||
}
|
||||
|
||||
// acquireToken gets a token from td, or returns the error of ctx if it
|
||||
// is done first. TokenDispenser.Get can't be cancelled, so an abandoned
|
||||
// wait gives the token back when it gets it.
|
||||
func acquireToken(ctx context.Context, td *pacer.TokenDispenser) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
got := make(chan struct{})
|
||||
go func() {
|
||||
td.Get()
|
||||
select {
|
||||
case got <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
td.Put()
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case <-got:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// sealBlock encrypts and signs the block in plain as block index of d,
|
||||
// checks it decrypts with the server's content key verifySK and makes
|
||||
// its verifier token from code. As in the SDK, a block whose check
|
||||
// fails, which may be a bit flip, is encrypted once more.
|
||||
func (c *client) sealBlock(d *draft, plain *pool.RW, index int, code []byte, verifySK *crypto.SessionKey, signKR *crypto.KeyRing) (b *encryptedBlock, hash []byte, err error) {
|
||||
for attempt := 1; ; attempt++ {
|
||||
b, hash, err = c.sealBlockOnce(d, plain, index, code, verifySK, signKR)
|
||||
if err == nil || attempt == blockSealAttempts {
|
||||
return b, hash, err
|
||||
}
|
||||
fs.Debugf(nil, "protondrive: block %d: %v: encrypting it again", index, err)
|
||||
}
|
||||
}
|
||||
|
||||
// sealBlockOnce makes one attempt of sealBlock.
|
||||
func (c *client) sealBlockOnce(d *draft, plain *pool.RW, index int, code []byte, verifySK *crypto.SessionKey, signKR *crypto.KeyRing) (*encryptedBlock, []byte, error) {
|
||||
if _, err := plain.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
enc := pool.NewRW(pool.Global())
|
||||
h := sha256.New()
|
||||
encSignature, err := encryptBlockTo(io.MultiWriter(enc, h), plain, d.sk, d.nodeKR, signKR, c.s.serverTime())
|
||||
if err == nil {
|
||||
err = checkBlock(enc, verifySK)
|
||||
}
|
||||
var token []byte
|
||||
if err == nil {
|
||||
token, err = blockToken(enc, code)
|
||||
}
|
||||
if err != nil {
|
||||
_ = enc.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
if c.verifierHook != nil {
|
||||
c.verifierHook(token)
|
||||
}
|
||||
return &encryptedBlock{index: index, enc: enc, encSignature: encSignature, token: token}, h.Sum(nil), nil
|
||||
}
|
||||
|
||||
// checkBlock checks the encrypted block enc decrypts with sk.
|
||||
func checkBlock(enc *pool.RW, sk *crypto.SessionKey) error {
|
||||
if _, err := enc.Seek(0, io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
dec, err := blockDecrypter(enc, sk)
|
||||
if err == nil {
|
||||
_, err = io.Copy(io.Discard, dec)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("encrypted block doesn't decrypt with the file's content key: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// blockToken returns the verifier token of the encrypted block enc.
|
||||
func blockToken(enc *pool.RW, code []byte) ([]byte, error) {
|
||||
if _, err := enc.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
prefix := make([]byte, len(code))
|
||||
n, err := io.ReadFull(enc, prefix)
|
||||
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||
return nil, err
|
||||
}
|
||||
return verifierToken(code, prefix[:n]), nil
|
||||
}
|
||||
|
||||
// requestURLs takes blocks from ready as they come, requests their
|
||||
// upload URLs in batches of those waiting and passes them to uploads.
|
||||
func (c *client) requestURLs(ctx context.Context, d *draft, ready <-chan *encryptedBlock, uploads chan<- blockUpload) error {
|
||||
for {
|
||||
var batch []*encryptedBlock
|
||||
select {
|
||||
case b, ok := <-ready:
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
batch = append(batch, b)
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
done := false
|
||||
waiting:
|
||||
for !done {
|
||||
select {
|
||||
case b, ok := <-ready:
|
||||
if ok {
|
||||
batch = append(batch, b)
|
||||
}
|
||||
done = !ok
|
||||
default:
|
||||
break waiting
|
||||
}
|
||||
}
|
||||
links, err := c.blockURLs(ctx, d, batch)
|
||||
if err != nil {
|
||||
freeBlocks(batch)
|
||||
return err
|
||||
}
|
||||
for i, b := range batch {
|
||||
select {
|
||||
case uploads <- blockUpload{block: b, link: links[i]}:
|
||||
case <-ctx.Done():
|
||||
freeBlocks(batch[i:])
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
if done {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// blockURLs requests the upload URLs of blocks of draft d, in the
|
||||
// order of blocks.
|
||||
func (c *client) blockURLs(ctx context.Context, d *draft, blocks []*encryptedBlock) ([]api.UploadLink, error) {
|
||||
req := &api.BlockUploadRequest{
|
||||
AddressID: c.addr.ID,
|
||||
VolumeID: c.volumeID,
|
||||
LinkID: d.nodeID,
|
||||
RevisionID: d.revisionID,
|
||||
BlockList: make([]api.BlockUploadInfo, 0, len(blocks)),
|
||||
ThumbnailList: []struct{}{},
|
||||
}
|
||||
for _, b := range blocks {
|
||||
req.BlockList = append(req.BlockList, api.BlockUploadInfo{
|
||||
Index: b.index,
|
||||
EncSignature: b.encSignature,
|
||||
Verifier: api.BlockVerifier{Token: base64.StdEncoding.EncodeToString(b.token)},
|
||||
})
|
||||
}
|
||||
var resp api.BlockUploadResponse
|
||||
if err := c.apiCall(ctx, &rest.Opts{Method: "POST", Path: "/drive/blocks"}, req, &resp); err != nil {
|
||||
return nil, fmt.Errorf("request upload URLs: %w", err)
|
||||
}
|
||||
byIndex := make(map[int]api.UploadLink, len(resp.UploadLinks))
|
||||
for _, l := range resp.UploadLinks {
|
||||
byIndex[l.Index] = l
|
||||
}
|
||||
links := make([]api.UploadLink, len(blocks))
|
||||
for i, b := range blocks {
|
||||
l, ok := byIndex[b.index]
|
||||
if !ok || l.BareURL == "" {
|
||||
return nil, fmt.Errorf("request upload URLs: no URL for block %d", b.index)
|
||||
}
|
||||
links[i] = l
|
||||
}
|
||||
return links, nil
|
||||
}
|
||||
|
||||
// uploadBlock uploads block u to the storage host, trying up to
|
||||
// blockUploadAttempts times. A 404 means the URL expired, so a new one
|
||||
// is requested.
|
||||
func (c *client) uploadBlock(ctx context.Context, d *draft, u blockUpload) error {
|
||||
link := u.link
|
||||
for attempt := 1; ; attempt++ {
|
||||
retry, err := c.postBlock(ctx, link, u.block.enc)
|
||||
var apiErr *api.Error
|
||||
switch {
|
||||
case err == nil:
|
||||
return nil
|
||||
case ctx.Err() != nil:
|
||||
return ctx.Err()
|
||||
case attempt >= blockUploadAttempts:
|
||||
return err
|
||||
case errors.As(err, &apiErr) && apiErr.Status == http.StatusNotFound:
|
||||
fs.Debugf(nil, "protondrive: block %d: upload URL expired: requesting a new one", u.block.index)
|
||||
links, err := c.blockURLs(ctx, d, []*encryptedBlock{u.block})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
link = links[0]
|
||||
case !retry:
|
||||
return err
|
||||
default:
|
||||
fs.Debugf(nil, "protondrive: block %d: upload failed: %v: retrying", u.block.index, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// postBlock makes one attempt to upload the encrypted block enc to
|
||||
// link, paced by the storage pacer, and reports whether a failure may
|
||||
// be retried.
|
||||
func (c *client) postBlock(ctx context.Context, link api.UploadLink, enc *pool.RW) (retry bool, err error) {
|
||||
paced := c.paced(ctx, func() (*http.Response, error) {
|
||||
if _, err := enc.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body := newMultipartBlock(enc)
|
||||
defer body.finish()
|
||||
opts := &rest.Opts{
|
||||
Method: "POST",
|
||||
RootURL: link.BareURL,
|
||||
Body: body,
|
||||
ContentType: body.contentType,
|
||||
ContentLength: &body.size,
|
||||
NoResponse: true,
|
||||
// The storage host gets the block's token, never the
|
||||
// session's credentials, which only session.call adds.
|
||||
ExtraHeaders: map[string]string{"pm-storage-token": link.Token},
|
||||
}
|
||||
return c.s.srv.Call(ctx, opts)
|
||||
})
|
||||
err = c.storagePacer.CallNoRetry(func() (bool, error) {
|
||||
var err error
|
||||
retry, err = paced()
|
||||
return retry, err
|
||||
})
|
||||
return retry, err
|
||||
}
|
||||
|
||||
// errAttemptOver is returned by reads of a multipartBlock whose upload
|
||||
// attempt is over.
|
||||
var errAttemptOver = errors.New("upload attempt is over")
|
||||
|
||||
// multipartBlock is the multipart/form-data body of one attempt to
|
||||
// upload a block, as the SDK's uploadBlock sends it: the block in the
|
||||
// field Block.
|
||||
//
|
||||
// The HTTP transport may still read a request body after the request
|
||||
// returns, so the body stops reading the block once finished: the
|
||||
// block may then be read for a retry, or freed.
|
||||
type multipartBlock struct {
|
||||
contentType string
|
||||
size int64
|
||||
|
||||
mu sync.Mutex
|
||||
r io.Reader
|
||||
done bool
|
||||
}
|
||||
|
||||
// newMultipartBlock returns the body uploading the block in enc from
|
||||
// its read position.
|
||||
func newMultipartBlock(enc *pool.RW) *multipartBlock {
|
||||
var buf bytes.Buffer
|
||||
mw := multipart.NewWriter(&buf)
|
||||
// Writes to a bytes.Buffer don't fail.
|
||||
_, _ = mw.CreateFormFile("Block", "blob")
|
||||
head := buf.String()
|
||||
buf.Reset()
|
||||
_ = mw.Close()
|
||||
tail := buf.String()
|
||||
pos, _ := enc.Seek(0, io.SeekCurrent)
|
||||
return &multipartBlock{
|
||||
contentType: mw.FormDataContentType(),
|
||||
size: int64(len(head)+len(tail)) + enc.Size() - pos,
|
||||
r: io.MultiReader(strings.NewReader(head), enc, strings.NewReader(tail)),
|
||||
}
|
||||
}
|
||||
|
||||
// Read reads the body, or fails once the body is finished.
|
||||
func (b *multipartBlock) Read(p []byte) (int, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.done {
|
||||
return 0, errAttemptOver
|
||||
}
|
||||
return b.r.Read(p)
|
||||
}
|
||||
|
||||
// finish stops the body reading the block, waiting for a read in
|
||||
// progress.
|
||||
func (b *multipartBlock) finish() {
|
||||
b.mu.Lock()
|
||||
b.done = true
|
||||
b.mu.Unlock()
|
||||
}
|
||||
|
||||
// deleteDraft deletes draft d of a failed upload, even if ctx is done,
|
||||
// logging a failure: the draft stands in the way of the next upload of
|
||||
// the name until it is replaced.
|
||||
func (c *client) deleteDraft(ctx context.Context, d *draft) {
|
||||
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), draftCleanupTimeout)
|
||||
defer cancel()
|
||||
var err error
|
||||
if d.newFile {
|
||||
err = c.deleteDraftLink(ctx, d.nodeID)
|
||||
} else {
|
||||
err = c.apiCall(ctx, &rest.Opts{Method: "DELETE", Path: c.volumePath("/drive/v2/volumes", "files", d.nodeID, "revisions", d.revisionID)}, nil, nil)
|
||||
}
|
||||
c.forget(d.nodeID, d.parentID)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "protondrive: failed to delete the draft of a failed upload (node %s, revision %s): %v", d.nodeID, d.revisionID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// deleteDraftLink permanently deletes the draft node id.
|
||||
func (c *client) deleteDraftLink(ctx context.Context, id string) error {
|
||||
var resp api.MultiLinkResponse
|
||||
opts := &rest.Opts{Method: "POST", Path: c.volumePath("/drive/v2/volumes") + "/delete_multiple"}
|
||||
if err := c.apiCall(ctx, opts, &api.LinksRequest{LinkIDs: []string{id}}, &resp); err != nil {
|
||||
return fmt.Errorf("delete draft: %w", err)
|
||||
}
|
||||
for _, r := range resp.Responses {
|
||||
if r.Response.Code/1000 != 1 || r.Response.Error != "" {
|
||||
return mutationError("delete draft "+r.LinkID, &api.Error{Code: r.Response.Code, Message: r.Response.Error})
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,975 @@
|
||||
package protondrive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/protondrive/api"
|
||||
"github.com/rclone/rclone/backend/protondrive/protontest"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
filesPattern = "/drive/v2/volumes/{volumeID}/files"
|
||||
commitPattern = "/drive/v2/volumes/{volumeID}/files/{linkID}/revisions/{revisionID}"
|
||||
deleteDraftsPattern = "/drive/v2/volumes/{volumeID}/delete_multiple"
|
||||
storageUploadPattern = "/storage/upload/{token}"
|
||||
|
||||
testDraftClientUID = "rclone-test-draft-client"
|
||||
)
|
||||
|
||||
// newUploadClient returns a client of a new user of a new fake server
|
||||
// with a draft client UID and the options opt.
|
||||
func newUploadClient(t *testing.T, opt clientOptions) (*client, *protontest.Server, *protontest.User) {
|
||||
t.Helper()
|
||||
if opt.DraftClientUID == "" {
|
||||
opt.DraftClientUID = testDraftClientUID
|
||||
}
|
||||
return newTestClientOpt(t, opt)
|
||||
}
|
||||
|
||||
// testModTime is the modification time of uploaded files.
|
||||
var testModTime = time.Date(2024, 5, 6, 7, 8, 9, 0, time.UTC)
|
||||
|
||||
// sha1Of returns the hex SHA-1 of b.
|
||||
func sha1Of(b []byte) string {
|
||||
sum := sha1.Sum(b)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// uploadContent uploads content as name in the root with size declared
|
||||
// as size.
|
||||
func uploadContent(ctx context.Context, c *client, name string, in io.Reader, size int64) (*node, error) {
|
||||
return c.upload(ctx, uploadRequest{
|
||||
ParentID: c.rootID(),
|
||||
Name: name,
|
||||
In: in,
|
||||
Size: size,
|
||||
ModTime: testModTime,
|
||||
MIMEType: "application/octet-stream",
|
||||
})
|
||||
}
|
||||
|
||||
// uploadBytes uploads content as name in the root, declaring its size.
|
||||
func uploadBytes(t *testing.T, c *client, name string, content []byte) *node {
|
||||
t.Helper()
|
||||
n, err := uploadContent(context.Background(), c, name, bytes.NewReader(content), int64(len(content)))
|
||||
require.NoError(t, err)
|
||||
return n
|
||||
}
|
||||
|
||||
// requireUploaded fails the test unless the fake decrypts and verifies
|
||||
// every node of u without error, and finds the files of want, by path,
|
||||
// with that content, the test modification time and its SHA1.
|
||||
func requireUploaded(t *testing.T, u *protontest.User, want map[string][]byte) {
|
||||
t.Helper()
|
||||
dump := u.Dump()
|
||||
got := make(map[string]bool)
|
||||
for _, d := range dump {
|
||||
require.NoError(t, d.Err, "node %q", d.Path)
|
||||
content, ok := want[d.Path]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
got[d.Path] = true
|
||||
assert.True(t, bytes.Equal(content, d.Content), "%q: content differs: %d bytes, want %d", d.Path, len(d.Content), len(content))
|
||||
assert.Equal(t, sha1Of(content), d.SHA1, "%q: SHA1", d.Path)
|
||||
assert.True(t, testModTime.Equal(d.ModTime), "%q: modification time %v", d.Path, d.ModTime)
|
||||
}
|
||||
for path := range want {
|
||||
assert.True(t, got[path], "%q not in the drive", path)
|
||||
}
|
||||
assert.Empty(t, u.Drafts(), "drafts left")
|
||||
}
|
||||
|
||||
// files returns the paths of the files in u's drive.
|
||||
func files(u *protontest.User) []string {
|
||||
var paths []string
|
||||
for _, d := range u.Dump() {
|
||||
if !d.IsFolder {
|
||||
paths = append(paths, d.Path)
|
||||
}
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func TestUploadSizes(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newUploadClient(t, clientOptions{})
|
||||
want := make(map[string][]byte)
|
||||
for _, size := range []int{0, 1, 4<<20 - 1, 4 << 20, 4<<20 + 1, bigSize} {
|
||||
t.Run(strconv.Itoa(size), func(t *testing.T) {
|
||||
name := "f" + strconv.Itoa(size)
|
||||
content := testContent(size)
|
||||
want[name] = content
|
||||
srv.ResetRequests()
|
||||
n := uploadBytes(t, c, name, content)
|
||||
blocks := (size + 4<<20 - 1) / (4 << 20)
|
||||
assert.Equal(t, blocks, srv.Requests()["POST "+storageUploadPattern], "blocks uploaded")
|
||||
assert.Equal(t, name, n.Name)
|
||||
assert.Equal(t, int64(size), n.size(true))
|
||||
assert.Equal(t, sha1Of(content), n.sha1())
|
||||
assert.True(t, testModTime.Equal(n.modTime()))
|
||||
require.NotNil(t, n.X)
|
||||
assert.Len(t, n.X.BlockSizes, blocks)
|
||||
})
|
||||
}
|
||||
requireUploaded(t, u, want)
|
||||
}
|
||||
|
||||
func TestUploadUnknownSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, _, u := newUploadClient(t, clientOptions{})
|
||||
want := make(map[string][]byte)
|
||||
for _, size := range []int{0, 5 << 20} {
|
||||
name := "f" + strconv.Itoa(size)
|
||||
want[name] = testContent(size)
|
||||
n, err := uploadContent(context.Background(), c, name, bytes.NewReader(want[name]), -1)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(size), n.size(true))
|
||||
}
|
||||
requireUploaded(t, u, want)
|
||||
}
|
||||
|
||||
func TestUploadNewRevision(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, _, u := newUploadClient(t, clientOptions{})
|
||||
first := uploadBytes(t, c, "f", testContent(5<<20))
|
||||
content := []byte("new content")
|
||||
n, err := c.upload(context.Background(), uploadRequest{
|
||||
Existing: first,
|
||||
In: bytes.NewReader(content),
|
||||
Size: int64(len(content)),
|
||||
ModTime: testModTime,
|
||||
SHA1: sha1Of(content),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, first.ID, n.ID)
|
||||
assert.NotEqual(t, first.RevisionID, n.RevisionID)
|
||||
requireUploaded(t, u, map[string][]byte{"f": content})
|
||||
assert.Equal(t, []string{"f"}, files(u))
|
||||
old, err := c.revisionBlocks(context.Background(), first, 1, 1)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, old.State, "old revision not superseded")
|
||||
}
|
||||
|
||||
func TestUploadOverExistingNameCreatesRevision(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, _, u := newUploadClient(t, clientOptions{})
|
||||
first := uploadBytes(t, c, "f", []byte("first"))
|
||||
n := uploadBytes(t, c, "f", []byte("second"))
|
||||
assert.Equal(t, first.ID, n.ID)
|
||||
assert.NotEqual(t, first.RevisionID, n.RevisionID)
|
||||
requireUploaded(t, u, map[string][]byte{"f": []byte("second")})
|
||||
assert.Equal(t, []string{"f"}, files(u))
|
||||
|
||||
// A folder in the way is not replaced.
|
||||
_, err := c.createFolder(context.Background(), c.rootID(), "dir", time.Time{})
|
||||
require.NoError(t, err)
|
||||
_, err = uploadContent(context.Background(), c, "dir", strings.NewReader("x"), 1)
|
||||
assert.ErrorIs(t, err, errExists)
|
||||
assert.Empty(t, u.Drafts())
|
||||
}
|
||||
|
||||
// testSourceMismatch uploads actual bytes declared as 5 MiB, which must
|
||||
// fail cleanly.
|
||||
func testSourceMismatch(t *testing.T, actual int) {
|
||||
c, srv, u := newUploadClient(t, clientOptions{})
|
||||
goroutines, buffers := leakBaseline(srv)
|
||||
_, err := uploadContent(context.Background(), c, "f", bytes.NewReader(testContent(actual)), 5<<20)
|
||||
assert.ErrorIs(t, err, errUploadMismatch)
|
||||
assert.Empty(t, u.Drafts())
|
||||
assert.Empty(t, files(u))
|
||||
requireNoLeaks(t, srv, goroutines, buffers)
|
||||
}
|
||||
|
||||
func TestUploadShortSource(t *testing.T) {
|
||||
testSourceMismatch(t, 3<<20)
|
||||
testSourceMismatch(t, 0)
|
||||
}
|
||||
|
||||
func TestUploadLongSource(t *testing.T) {
|
||||
testSourceMismatch(t, 6<<20)
|
||||
testSourceMismatch(t, 9<<20)
|
||||
}
|
||||
|
||||
func TestUploadSHA1Mismatch(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, _, u := newUploadClient(t, clientOptions{})
|
||||
content := testContent(1000)
|
||||
_, err := c.upload(context.Background(), uploadRequest{
|
||||
ParentID: c.rootID(), Name: "f", In: bytes.NewReader(content), Size: 1000,
|
||||
SHA1: sha1Of([]byte("something else")),
|
||||
})
|
||||
assert.ErrorIs(t, err, errUploadMismatch)
|
||||
assert.Empty(t, u.Drafts())
|
||||
assert.Empty(t, files(u))
|
||||
|
||||
_, err = c.upload(context.Background(), uploadRequest{
|
||||
ParentID: c.rootID(), Name: "f", In: bytes.NewReader(content), Size: 1000,
|
||||
ModTime: testModTime, SHA1: strings.ToUpper(sha1Of(content)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
requireUploaded(t, u, map[string][]byte{"f": content})
|
||||
}
|
||||
|
||||
func TestUploadBlock502Once(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newUploadClient(t, clientOptions{})
|
||||
srv.Inject(protontest.Fault{Method: "POST", Pattern: storageUploadPattern, Status: http.StatusBadGateway, Times: 1})
|
||||
srv.ResetRequests()
|
||||
content := testContent(5 << 20)
|
||||
uploadBytes(t, c, "f", content)
|
||||
requireUploaded(t, u, map[string][]byte{"f": content})
|
||||
assert.Equal(t, 3, srv.Requests()["POST "+storageUploadPattern], "two blocks and the resent one")
|
||||
}
|
||||
|
||||
func TestRefusedCommit(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{"422 with a code", &api.Error{Status: 422, Code: api.CodeInvalidRequirements}, true},
|
||||
{"408 with a code", &api.Error{Status: 408, Code: api.CodeInvalidRequirements}, false},
|
||||
{"422 without a code", &api.Error{Status: 422}, false},
|
||||
{"500 with a code", &api.Error{Status: 500, Code: api.CodeInvalidRequirements}, false},
|
||||
{"200 with a code", &api.Error{Status: 200, Code: api.CodeInvalidRequirements}, false},
|
||||
{"network error", io.ErrUnexpectedEOF, false},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
assert.Equal(t, test.want, refusedCommit(fmt.Errorf("commit: %w", test.err)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestUploadBlock502Forever is issue 10224: a block which can't be
|
||||
// uploaded must fail the upload, not hang it.
|
||||
func TestUploadBlock502Forever(t *testing.T) {
|
||||
c, srv, u := newUploadClient(t, clientOptions{})
|
||||
goroutines, buffers := leakBaseline(srv)
|
||||
srv.Inject(protontest.Fault{Method: "POST", Pattern: storageUploadPattern, Status: http.StatusBadGateway})
|
||||
start := time.Now()
|
||||
_, err := uploadContent(context.Background(), c, "f", bytes.NewReader(testContent(9<<20)), 9<<20)
|
||||
require.Error(t, err)
|
||||
assert.Less(t, time.Since(start), 30*time.Second)
|
||||
var apiErr *api.Error
|
||||
require.ErrorAs(t, err, &apiErr)
|
||||
assert.Equal(t, http.StatusBadGateway, apiErr.Status)
|
||||
assert.Empty(t, u.Drafts())
|
||||
requireNoLeaks(t, srv, goroutines, buffers)
|
||||
}
|
||||
|
||||
func TestUploadStorageTokenExpired(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
var expired atomic.Bool
|
||||
c := newTestClientTransport(t, srv, u, clientOptions{DraftClientUID: testDraftClientUID}, func(next http.RoundTripper) http.RoundTripper {
|
||||
return roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if strings.HasPrefix(r.URL.Path, "/storage/upload/") && !expired.Swap(true) {
|
||||
srv.ExpireUploadTokens()
|
||||
}
|
||||
return next.RoundTrip(r)
|
||||
})
|
||||
})
|
||||
content := testContent(9 << 20)
|
||||
uploadBytes(t, c, "f", content)
|
||||
assert.Greater(t, srv.Requests()["POST /drive/blocks"], 1, "no new URL requested")
|
||||
assert.Greater(t, srv.Requests()["POST "+storageUploadPattern], 3, "no upload refused")
|
||||
requireUploaded(t, u, map[string][]byte{"f": content})
|
||||
}
|
||||
|
||||
func TestUploadCommitErrorButCommitted(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newUploadClient(t, clientOptions{})
|
||||
srv.Inject(protontest.Fault{Method: "PUT", Pattern: commitPattern, Status: http.StatusBadGateway, AfterHandler: true, Times: 1})
|
||||
content := testContent(1 << 20)
|
||||
n := uploadBytes(t, c, "f", content)
|
||||
assert.Equal(t, 2, srv.Requests()["PUT "+commitPattern], "commit not retried")
|
||||
assert.Equal(t, "f", n.Name)
|
||||
requireUploaded(t, u, map[string][]byte{"f": content})
|
||||
assert.Equal(t, []string{"f"}, files(u))
|
||||
}
|
||||
|
||||
func TestUploadDraftResponseLost(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newUploadClient(t, clientOptions{})
|
||||
srv.Inject(protontest.Fault{Method: "POST", Pattern: filesPattern, Status: http.StatusBadGateway, AfterHandler: true, Times: 1})
|
||||
content := testContent(1000)
|
||||
uploadBytes(t, c, "f", content)
|
||||
assert.Equal(t, 3, srv.Requests()["POST "+filesPattern], "draft creation: first, retry, after deleting the draft")
|
||||
requireUploaded(t, u, map[string][]byte{"f": content})
|
||||
assert.Equal(t, []string{"f"}, files(u))
|
||||
}
|
||||
|
||||
// leaveDraft makes an upload of name by c fail at commit and its
|
||||
// cleanup fail too, leaving its draft behind.
|
||||
func leaveDraft(t *testing.T, c *client, srv *protontest.Server, u *protontest.User, name string) {
|
||||
t.Helper()
|
||||
srv.Inject(protontest.Fault{Method: "PUT", Pattern: commitPattern, Status: http.StatusUnprocessableEntity, Code: api.CodeInvalidValue, Times: 1})
|
||||
srv.Inject(protontest.Fault{Method: "POST", Pattern: deleteDraftsPattern, Status: http.StatusUnprocessableEntity, Code: api.CodeInvalidValue, Times: 1})
|
||||
_, err := uploadContent(context.Background(), c, name, strings.NewReader("draft"), 5)
|
||||
require.Error(t, err)
|
||||
require.Len(t, u.Drafts(), 1)
|
||||
}
|
||||
|
||||
// TestUploadOwnDraftReplaced is issue 9622: a draft left by a failed
|
||||
// upload must not block the next upload of the name.
|
||||
func TestUploadOwnDraftReplaced(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newUploadClient(t, clientOptions{})
|
||||
leaveDraft(t, c, srv, u, "f")
|
||||
content := testContent(1000)
|
||||
uploadBytes(t, c, "f", content)
|
||||
requireUploaded(t, u, map[string][]byte{"f": content})
|
||||
assert.Equal(t, []string{"f"}, files(u))
|
||||
}
|
||||
|
||||
func TestUploadForeignDraft(t *testing.T) {
|
||||
t.Parallel()
|
||||
other, srv, u := newUploadClient(t, clientOptions{DraftClientUID: "another client"})
|
||||
leaveDraft(t, other, srv, u, "f")
|
||||
|
||||
c := newTestClientFor(t, srv, u, clientOptions{DraftClientUID: testDraftClientUID})
|
||||
_, err := uploadContent(context.Background(), c, "f", strings.NewReader("x"), 1)
|
||||
assert.ErrorIs(t, err, errExists)
|
||||
assert.Len(t, u.Drafts(), 1, "another client's draft deleted")
|
||||
|
||||
// Without a draft client UID no draft is the client's own.
|
||||
anon := newTestClientFor(t, srv, u, clientOptions{})
|
||||
_, err = uploadContent(context.Background(), anon, "f", strings.NewReader("x"), 1)
|
||||
assert.ErrorIs(t, err, errExists)
|
||||
|
||||
replacer := newTestClientFor(t, srv, u, clientOptions{DraftClientUID: testDraftClientUID, ReplaceExistingDraft: true})
|
||||
content := testContent(1000)
|
||||
uploadBytes(t, replacer, "f", content)
|
||||
requireUploaded(t, u, map[string][]byte{"f": content})
|
||||
assert.Equal(t, []string{"f"}, files(u))
|
||||
}
|
||||
|
||||
func TestUploadCancel(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
fault protontest.Fault
|
||||
drafts int // drafts left: a cancelled commit may have taken effect
|
||||
}{
|
||||
{"draft", protontest.Fault{Method: "POST", Pattern: filesPattern}, 0},
|
||||
{"blocks", protontest.Fault{Method: "POST", Pattern: storageUploadPattern, Skip: 1}, 0},
|
||||
{"commit", protontest.Fault{Method: "PUT", Pattern: commitPattern}, 1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
c, srv, u := newUploadClient(t, clientOptions{})
|
||||
goroutines, buffers := leakBaseline(srv)
|
||||
tc.fault.Delay = time.Minute
|
||||
srv.Inject(tc.fault)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
_, err := uploadContent(ctx, c, "f", bytes.NewReader(testContent(9<<20)), 9<<20)
|
||||
assert.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
assert.Less(t, time.Since(start), 10*time.Second)
|
||||
assert.Len(t, u.Drafts(), tc.drafts)
|
||||
assert.Empty(t, files(u))
|
||||
requireNoLeaks(t, srv, goroutines, buffers)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadConcurrencyBound(t *testing.T) {
|
||||
t.Run("PerFile", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newUploadClient(t, clientOptions{UploadConcurrency: 2})
|
||||
srv.Inject(protontest.Fault{Method: "POST", Pattern: storageUploadPattern, Delay: 200 * time.Millisecond})
|
||||
content := testContent(12<<20 + 1)
|
||||
uploadBytes(t, c, "f", content)
|
||||
assert.LessOrEqual(t, srv.StorageUploadPeak(), 2)
|
||||
requireUploaded(t, u, map[string][]byte{"f": content})
|
||||
})
|
||||
t.Run("PerClient", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newUploadClient(t, clientOptions{UploadConcurrency: maxUploadConcurrency})
|
||||
srv.Inject(protontest.Fault{Method: "POST", Pattern: storageUploadPattern, Delay: 300 * time.Millisecond})
|
||||
want := make(map[string][]byte)
|
||||
var wg sync.WaitGroup
|
||||
errs := make([]error, 16)
|
||||
for i := range errs {
|
||||
name := "f" + strconv.Itoa(i)
|
||||
content := testContent(1000 + i)
|
||||
want[name] = content
|
||||
wg.Go(func() {
|
||||
_, errs[i] = uploadContent(context.Background(), c, name, bytes.NewReader(content), int64(len(content)))
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
for _, err := range errs {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
peak := srv.StorageUploadPeak()
|
||||
assert.LessOrEqual(t, peak, blocksInFlight)
|
||||
assert.Greater(t, peak, 1, "uploads not concurrent")
|
||||
requireUploaded(t, u, want)
|
||||
})
|
||||
}
|
||||
|
||||
func TestUploadVerifierChecked(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, _, u := newUploadClient(t, clientOptions{})
|
||||
c.verifierHook = func(token []byte) { token[0] ^= 1 }
|
||||
_, err := uploadContent(context.Background(), c, "f", bytes.NewReader(testContent(1000)), 1000)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, api.CodeInvalidRequirements, apiErrorCode(err))
|
||||
assert.Contains(t, err.Error(), "verifier token")
|
||||
assert.Empty(t, u.Drafts())
|
||||
assert.Empty(t, files(u))
|
||||
}
|
||||
|
||||
func TestUploadClockSkew(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, skew := range []time.Duration{3 * time.Hour, -3 * time.Hour} {
|
||||
t.Run(skew.String(), func(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
srv.SetClockSkew(skew)
|
||||
u := srv.AddUser("user", "pass")
|
||||
c := newTestClientFor(t, srv, u, clientOptions{DraftClientUID: testDraftClientUID})
|
||||
first := uploadBytes(t, c, "f", []byte("first"))
|
||||
content := testContent(5 << 20)
|
||||
_, err := c.upload(context.Background(), uploadRequest{Existing: first, In: bytes.NewReader(content), Size: -1, ModTime: testModTime})
|
||||
require.NoError(t, err)
|
||||
requireUploaded(t, u, map[string][]byte{"f": content})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadThenDownload(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, _, u := newUploadClient(t, clientOptions{})
|
||||
content := testContent(bigSize)
|
||||
n := uploadBytes(t, c, "f", content)
|
||||
data, err := readRange(t, c, n, 0, -1)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, bytes.Equal(content, data), "content differs")
|
||||
data, err = readRange(t, c, n, bigSize-1, -1)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content[bigSize-1:], data)
|
||||
requireUploaded(t, u, map[string][]byte{"f": content})
|
||||
}
|
||||
|
||||
func TestIntendedUploadSize(t *testing.T) {
|
||||
ptr := func(v int64) *int64 { return &v }
|
||||
for _, tc := range []struct {
|
||||
size int64
|
||||
draft, revise *int64
|
||||
}{
|
||||
{-1, nil, nil},
|
||||
{0, nil, nil},
|
||||
{1, nil, ptr(4095)},
|
||||
{4096, nil, ptr(100_000)},
|
||||
{150_000, nil, ptr(100_000)},
|
||||
{199_999, nil, ptr(100_000)},
|
||||
{200_000, ptr(200_000), ptr(200_000)},
|
||||
{9_876_543, ptr(9_800_000), ptr(9_800_000)},
|
||||
} {
|
||||
assert.Equal(t, tc.draft, draftIntendedSize(tc.size), "draft %d", tc.size)
|
||||
assert.Equal(t, tc.revise, revisionIntendedSize(tc.size), "revision %d", tc.size)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAcquireTokenCancel checks a cancelled wait for a token returns at
|
||||
// once and gives the token back when it gets it.
|
||||
func TestAcquireTokenCancel(t *testing.T) {
|
||||
td := pacer.NewTokenDispenser(1)
|
||||
td.Get()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error)
|
||||
go func() { done <- acquireToken(ctx, td) }()
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
assert.ErrorIs(t, err, context.Canceled)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("cancelled wait for a token didn't return")
|
||||
}
|
||||
td.Put()
|
||||
got := make(chan error)
|
||||
go func() { got <- acquireToken(context.Background(), td) }()
|
||||
select {
|
||||
case err := <-got:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("token kept by the cancelled wait")
|
||||
}
|
||||
assert.True(t, errors.Is(acquireToken(ctx, td), context.Canceled))
|
||||
}
|
||||
|
||||
const (
|
||||
linksPattern = "/drive/v2/volumes/{volumeID}/links"
|
||||
deleteRevisionRequest = "DELETE " + commitPattern
|
||||
deleteDraftsRequest = "POST " + deleteDraftsPattern
|
||||
)
|
||||
|
||||
// TestUploadCancelDuringCommit cancels an upload while a commit which
|
||||
// the server makes is in flight: the committed file must not be deleted.
|
||||
func TestUploadCancelDuringCommit(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, revision := range []bool{false, true} {
|
||||
t.Run("revision="+strconv.FormatBool(revision), func(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
var armed atomic.Bool
|
||||
c := newTestClientTransport(t, srv, u, clientOptions{DraftClientUID: testDraftClientUID}, func(next http.RoundTripper) http.RoundTripper {
|
||||
return roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if !armed.Load() || r.Method != "PUT" || !strings.Contains(r.URL.Path, "/revisions/") {
|
||||
return next.RoundTrip(r)
|
||||
}
|
||||
// The server commits, then the caller gives up
|
||||
// before the response arrives.
|
||||
resp, err := next.RoundTrip(r)
|
||||
if err == nil {
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
cancel()
|
||||
return nil, context.Canceled
|
||||
})
|
||||
})
|
||||
var existing *node
|
||||
if revision {
|
||||
existing = uploadBytes(t, c, "f", []byte("old"))
|
||||
}
|
||||
srv.ResetRequests()
|
||||
armed.Store(true)
|
||||
content := testContent(5 << 20)
|
||||
_, err := c.upload(ctx, uploadRequest{ParentID: c.rootID(), Name: "f", Existing: existing, In: bytes.NewReader(content), Size: int64(len(content)), ModTime: testModTime})
|
||||
require.NoError(t, err, "a commit which is active is a success")
|
||||
reqs := srv.Requests()
|
||||
assert.Zero(t, reqs[deleteDraftsRequest], "draft deletion sent")
|
||||
assert.Zero(t, reqs[deleteRevisionRequest], "revision deletion sent")
|
||||
requireUploaded(t, u, map[string][]byte{"f": content})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestUploadCommitOutcomeUnknown makes the commit fail without a
|
||||
// refusal and the read which would tell whether it took effect fail
|
||||
// too: nothing may be deleted, and the error must say the outcome is
|
||||
// unknown.
|
||||
func TestUploadCommitOutcomeUnknown(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newUploadClient(t, clientOptions{})
|
||||
srv.Inject(protontest.Fault{Method: "PUT", Pattern: commitPattern, Status: http.StatusUnprocessableEntity})
|
||||
srv.Inject(protontest.Fault{Method: "POST", Pattern: linksPattern, Status: http.StatusUnprocessableEntity, Code: api.CodeInvalidValue})
|
||||
_, err := uploadContent(context.Background(), c, "f", strings.NewReader("content"), 7)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, errUploadOutcomeUnknown)
|
||||
assert.Contains(t, err.Error(), "may have succeeded")
|
||||
reqs := srv.Requests()
|
||||
assert.Zero(t, reqs[deleteDraftsRequest], "draft deletion sent")
|
||||
assert.Zero(t, reqs[deleteRevisionRequest], "revision deletion sent")
|
||||
assert.Len(t, u.Drafts(), 1, "the draft is left to the own-draft rule")
|
||||
}
|
||||
|
||||
// TestUploadCommitRefused makes the commit fail without effect: the
|
||||
// draft is deleted.
|
||||
func TestUploadCommitRefused(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newUploadClient(t, clientOptions{})
|
||||
srv.Inject(protontest.Fault{Method: "PUT", Pattern: commitPattern, Status: http.StatusUnprocessableEntity, Code: api.CodeInvalidValue, Times: 1})
|
||||
_, err := uploadContent(context.Background(), c, "f", strings.NewReader("content"), 7)
|
||||
require.Error(t, err)
|
||||
assert.NotErrorIs(t, err, errUploadOutcomeUnknown)
|
||||
assert.Equal(t, 1, srv.Requests()[deleteDraftsRequest])
|
||||
assert.Empty(t, u.Drafts())
|
||||
assert.Empty(t, files(u))
|
||||
}
|
||||
|
||||
// failingReader reads r, then fails with err where r ends.
|
||||
type failingReader struct {
|
||||
r io.Reader
|
||||
err error
|
||||
}
|
||||
|
||||
// Read reads from r, then fails with err.
|
||||
func (f *failingReader) Read(p []byte) (int, error) {
|
||||
n, err := f.r.Read(p)
|
||||
if err == io.EOF {
|
||||
return n, f.err
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func TestUploadSourceError(t *testing.T) {
|
||||
c, srv, u := newUploadClient(t, clientOptions{})
|
||||
goroutines, buffers := leakBaseline(srv)
|
||||
errSource := errors.New("source broke")
|
||||
in := &failingReader{r: bytes.NewReader(testContent(5<<20 + 10)), err: errSource}
|
||||
_, err := uploadContent(context.Background(), c, "f", in, 9<<20)
|
||||
assert.ErrorIs(t, err, errSource)
|
||||
assert.Empty(t, u.Drafts())
|
||||
assert.Empty(t, files(u))
|
||||
requireNoLeaks(t, srv, goroutines, buffers)
|
||||
}
|
||||
|
||||
// TestUploadBlockFailsProducerBlocked fails every block upload with one
|
||||
// worker, so the producer is waiting to hand on a block when the
|
||||
// worker gives up.
|
||||
func TestUploadBlockFailsProducerBlocked(t *testing.T) {
|
||||
c, srv, u := newUploadClient(t, clientOptions{UploadConcurrency: 1})
|
||||
goroutines, buffers := leakBaseline(srv)
|
||||
srv.Inject(protontest.Fault{Method: "POST", Pattern: storageUploadPattern, Status: http.StatusBadGateway})
|
||||
_, err := uploadContent(context.Background(), c, "f", bytes.NewReader(testContent(12<<20+1)), 12<<20+1)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, 3, srv.Requests()["POST "+storageUploadPattern], "one block, tried 3 times")
|
||||
assert.Empty(t, u.Drafts())
|
||||
assert.Empty(t, files(u))
|
||||
requireNoLeaks(t, srv, goroutines, buffers)
|
||||
}
|
||||
|
||||
// revisionIn returns the revision ID in the path of a request to a
|
||||
// revision route, and the node ID.
|
||||
func revisionIn(path string) (nodeID, revisionID string) {
|
||||
parts := strings.Split(path, "/")
|
||||
for i, p := range parts {
|
||||
if p == "files" && i+3 < len(parts) && parts[i+2] == "revisions" {
|
||||
return parts[i+1], parts[i+3]
|
||||
}
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// interveneAfterCommit returns a client of u whose commit requests reach
|
||||
// the server, which commits, after which act runs with the node and
|
||||
// revision IDs; the client then sees a network error unless keep is set.
|
||||
func interveneAfterCommit(t *testing.T, srv *protontest.Server, u *protontest.User, keep bool, act func(nodeID, revisionID string)) (c *client, armed *atomic.Bool) {
|
||||
armed = new(atomic.Bool)
|
||||
c = newTestClientTransport(t, srv, u, clientOptions{DraftClientUID: testDraftClientUID}, func(next http.RoundTripper) http.RoundTripper {
|
||||
return roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Method != "PUT" || !armed.Load() {
|
||||
return next.RoundTrip(r)
|
||||
}
|
||||
resp, err := next.RoundTrip(r)
|
||||
armed.Store(false)
|
||||
act(revisionIn(r.URL.Path))
|
||||
if keep {
|
||||
return resp, err
|
||||
}
|
||||
if err == nil {
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
return nil, errors.New("connection reset")
|
||||
})
|
||||
})
|
||||
return c, armed
|
||||
}
|
||||
|
||||
// requireNoDeletes fails the test if a draft or revision deletion was
|
||||
// sent.
|
||||
func requireNoDeletes(t *testing.T, srv *protontest.Server) {
|
||||
t.Helper()
|
||||
reqs := srv.Requests()
|
||||
assert.Zero(t, reqs[deleteDraftsRequest], "draft deletion sent")
|
||||
assert.Zero(t, reqs[deleteRevisionRequest], "revision deletion sent")
|
||||
}
|
||||
|
||||
// TestUploadCommitThenChanged commits an upload whose response is lost,
|
||||
// after which another client changes the file: the follow-up read
|
||||
// can't prove what happened, so nothing may be deleted.
|
||||
func TestUploadCommitThenChanged(t *testing.T) {
|
||||
t.Parallel()
|
||||
other := []byte("another client's content")
|
||||
t.Run("NewFileNewRevision", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
c, armed := interveneAfterCommit(t, srv, u, false, func(nodeID, _ string) { u.SeedRevision(nodeID, other, testModTime) })
|
||||
armed.Store(true)
|
||||
_, err := uploadContent(context.Background(), c, "f", strings.NewReader("mine"), 4)
|
||||
assert.ErrorIs(t, err, errUploadOutcomeUnknown)
|
||||
requireNoDeletes(t, srv)
|
||||
requireUploaded(t, u, map[string][]byte{"f": other})
|
||||
})
|
||||
t.Run("NewFileTrashed", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
trasher := newTestClientFor(t, srv, u, clientOptions{})
|
||||
c, armed := interveneAfterCommit(t, srv, u, false, func(nodeID, _ string) {
|
||||
require.NoError(t, trasher.trash(context.Background(), nodeID))
|
||||
})
|
||||
armed.Store(true)
|
||||
_, err := uploadContent(context.Background(), c, "f", strings.NewReader("mine"), 4)
|
||||
assert.ErrorIs(t, err, errUploadOutcomeUnknown)
|
||||
requireNoDeletes(t, srv)
|
||||
dump := requireDump(t, u)
|
||||
require.Contains(t, dump, trashedPrefix+"f")
|
||||
assert.Equal(t, []byte("mine"), dump[trashedPrefix+"f"].Content)
|
||||
})
|
||||
t.Run("NewRevision", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
var ours string
|
||||
c, armed := interveneAfterCommit(t, srv, u, false, func(nodeID, revisionID string) {
|
||||
ours = revisionID
|
||||
u.SeedRevision(nodeID, other, testModTime)
|
||||
})
|
||||
first := uploadBytes(t, c, "f", []byte("first"))
|
||||
srv.ResetRequests()
|
||||
armed.Store(true)
|
||||
_, err := c.upload(context.Background(), uploadRequest{Existing: first, In: strings.NewReader("mine"), Size: 4, ModTime: testModTime})
|
||||
assert.ErrorIs(t, err, errUploadOutcomeUnknown)
|
||||
requireNoDeletes(t, srv)
|
||||
requireUploaded(t, u, map[string][]byte{"f": other})
|
||||
// Our committed revision is kept, superseded.
|
||||
mine := *first
|
||||
mine.RevisionID = ours
|
||||
rev, err := c.revisionBlocks(context.Background(), &mine, 1, 1)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, rev.State)
|
||||
})
|
||||
}
|
||||
|
||||
// TestUploadCommittedThenSuperseded commits an upload which succeeds,
|
||||
// after which another client gives the file a new revision before it
|
||||
// is read back: the other revision is not returned as the upload's.
|
||||
func TestUploadCommittedThenSuperseded(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
other := []byte("another client's content")
|
||||
c, armed := interveneAfterCommit(t, srv, u, true, func(nodeID, _ string) { u.SeedRevision(nodeID, other, testModTime) })
|
||||
armed.Store(true)
|
||||
n, err := uploadContent(context.Background(), c, "f", strings.NewReader("mine"), 4)
|
||||
assert.ErrorIs(t, err, errUploadSuperseded)
|
||||
assert.Nil(t, n)
|
||||
requireNoDeletes(t, srv)
|
||||
requireUploaded(t, u, map[string][]byte{"f": other})
|
||||
}
|
||||
|
||||
// TestUploadRevisionCommitRefused makes the commit of a new revision
|
||||
// fail without effect: the draft revision is deleted and the file keeps
|
||||
// its content.
|
||||
func TestUploadRevisionCommitRefused(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newUploadClient(t, clientOptions{})
|
||||
first := uploadBytes(t, c, "f", []byte("first"))
|
||||
srv.ResetRequests()
|
||||
srv.Inject(protontest.Fault{Method: "PUT", Pattern: commitPattern, Status: http.StatusUnprocessableEntity, Code: api.CodeInvalidValue, Times: 1})
|
||||
_, err := c.upload(context.Background(), uploadRequest{Existing: first, In: strings.NewReader("mine"), Size: 4, ModTime: testModTime})
|
||||
require.Error(t, err)
|
||||
assert.NotErrorIs(t, err, errUploadOutcomeUnknown)
|
||||
assert.Equal(t, 1, srv.Requests()[deleteRevisionRequest])
|
||||
requireUploaded(t, u, map[string][]byte{"f": []byte("first")})
|
||||
}
|
||||
|
||||
// TestUploadReadBackLag uploads while the server takes a moment to show
|
||||
// what was committed, as Proton's does: the upload still succeeds with
|
||||
// its own revision and nothing is deleted.
|
||||
func TestUploadReadBackLag(t *testing.T) {
|
||||
t.Parallel()
|
||||
content := []byte("content")
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
view protontest.LagView
|
||||
existing bool
|
||||
}{
|
||||
{name: "NewFileStale", view: protontest.LagBefore},
|
||||
{name: "NewFileMissing", view: protontest.LagMissing},
|
||||
{name: "NewFileNoRevision", view: protontest.LagNoRevision},
|
||||
{name: "NewRevisionStale", view: protontest.LagBefore, existing: true},
|
||||
{name: "NewRevisionMissing", view: protontest.LagMissing, existing: true},
|
||||
{name: "NewRevisionNoRevision", view: protontest.LagNoRevision, existing: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newUploadClient(t, clientOptions{})
|
||||
r := uploadRequest{ParentID: c.rootID(), Name: "f", In: bytes.NewReader(content), Size: int64(len(content)), ModTime: testModTime}
|
||||
var first *node
|
||||
if tc.existing {
|
||||
first = uploadBytes(t, c, "f", []byte("first"))
|
||||
r.Existing = first
|
||||
}
|
||||
srv.ResetRequests()
|
||||
srv.SetReadLag(2, tc.view)
|
||||
n, err := c.upload(context.Background(), r)
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, n.RevisionID)
|
||||
if first != nil {
|
||||
assert.NotEqual(t, first.RevisionID, n.RevisionID)
|
||||
}
|
||||
assert.Equal(t, sha1Of(content), n.sha1())
|
||||
requireNoDeletes(t, srv)
|
||||
requireUploaded(t, u, map[string][]byte{"f": content})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestUploadIntoNewFolderReadBackLag creates nested folders and uploads
|
||||
// into the deepest without a cache while the server takes a moment to
|
||||
// show new nodes.
|
||||
func TestUploadIntoNewFolderReadBackLag(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newTestClientOpt(t, clientOptions{DraftClientUID: testDraftClientUID})
|
||||
srv.SetReadLag(2, protontest.LagMissing)
|
||||
ctx := context.Background()
|
||||
parentID := c.rootID()
|
||||
for _, name := range []string{"A1", "B1", "C1"} {
|
||||
n, err := c.createFolder(ctx, parentID, name, time.Time{})
|
||||
require.NoError(t, err)
|
||||
parentID = n.ID
|
||||
}
|
||||
content := []byte("aaa")
|
||||
_, err := c.upload(ctx, uploadRequest{ParentID: parentID, Name: "one", In: bytes.NewReader(content), Size: int64(len(content)), ModTime: testModTime})
|
||||
require.NoError(t, err)
|
||||
requireNoDeletes(t, srv)
|
||||
requireUploaded(t, u, map[string][]byte{"A1/B1/C1/one": content})
|
||||
}
|
||||
|
||||
// TestUploadReadBackLagOutlasted commits an upload which the server
|
||||
// doesn't show for longer than the client reads it back: the upload
|
||||
// fails as unconfirmed and nothing is deleted.
|
||||
func TestUploadReadBackLagOutlasted(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, view := range []protontest.LagView{protontest.LagBefore, protontest.LagMissing, protontest.LagNoRevision} {
|
||||
t.Run(fmt.Sprintf("view=%d", view), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newUploadClient(t, clientOptions{})
|
||||
c.readBackDelay = time.Millisecond
|
||||
srv.SetReadLag(readBackAttempts, view)
|
||||
n, err := uploadContent(context.Background(), c, "f", strings.NewReader("mine"), 4)
|
||||
assert.ErrorIs(t, err, errUploadUnconfirmed)
|
||||
assert.NotErrorIs(t, err, errUploadSuperseded)
|
||||
assert.Nil(t, n)
|
||||
assert.Equal(t, readBackAttempts, srv.Requests()["POST /drive/v2/volumes/{volumeID}/links"])
|
||||
requireNoDeletes(t, srv)
|
||||
requireUploaded(t, u, map[string][]byte{"f": []byte("mine")})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// commitFailure returns a client of u whose next commit lands on srv
|
||||
// and then fails.
|
||||
type commitFailure func(t *testing.T, srv *protontest.Server, u *protontest.User) *client
|
||||
|
||||
// TestUploadAmbiguousCommitFailureUnderLag makes a commit land on the
|
||||
// server and then fail in a way which doesn't prove it didn't, while
|
||||
// the read back shows the node as before the commit: nothing may be
|
||||
// deleted, as the deletion could hit the committed file or revision.
|
||||
func TestUploadAmbiguousCommitFailureUnderLag(t *testing.T) {
|
||||
t.Parallel()
|
||||
failures := map[string]commitFailure{
|
||||
// The commit lands and its response is lost; the resent commit
|
||||
// is refused, as the draft revision is gone.
|
||||
"ResentThenRefused": func(t *testing.T, srv *protontest.Server, u *protontest.User) *client {
|
||||
srv.Inject(protontest.Fault{Method: "PUT", Pattern: commitPattern, Status: http.StatusBadGateway, AfterHandler: true, Times: 1})
|
||||
srv.Inject(protontest.Fault{Method: "PUT", Pattern: commitPattern, Status: http.StatusUnprocessableEntity, Code: api.CodeInvalidValue, AfterHandler: true, Skip: 1, Times: 1})
|
||||
return newTestClientFor(t, srv, u, clientOptions{DraftClientUID: testDraftClientUID})
|
||||
},
|
||||
// The commit lands and the connection drops before its response.
|
||||
"ConnectionLost": func(t *testing.T, srv *protontest.Server, u *protontest.User) *client {
|
||||
c, armed := interveneAfterCommit(t, srv, u, false, func(string, string) {})
|
||||
armed.Store(true)
|
||||
return c
|
||||
},
|
||||
}
|
||||
for name, fail := range failures {
|
||||
for _, existing := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/existing=%v", name, existing), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
var first *node
|
||||
if existing {
|
||||
c := newTestClientFor(t, srv, u, clientOptions{DraftClientUID: testDraftClientUID})
|
||||
first = uploadBytes(t, c, "f", []byte("first"))
|
||||
}
|
||||
c := fail(t, srv, u)
|
||||
c.readBackDelay = time.Millisecond
|
||||
srv.ResetRequests()
|
||||
srv.SetReadLag(readBackAttempts, protontest.LagBefore)
|
||||
r := uploadRequest{ParentID: c.rootID(), Name: "f", Existing: first, In: strings.NewReader("mine"), Size: 4, ModTime: testModTime}
|
||||
_, err := c.upload(context.Background(), r)
|
||||
assert.ErrorIs(t, err, errUploadOutcomeUnknown)
|
||||
requireNoDeletes(t, srv)
|
||||
requireUploaded(t, u, map[string][]byte{"f": []byte("mine")})
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestUploadCommitRefusedNoReadBack makes the server refuse a commit on
|
||||
// its only attempt: the refusal alone proves it didn't land, so the
|
||||
// draft is deleted without reading the file back, as a read may lag.
|
||||
func TestUploadCommitRefusedNoReadBack(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, existing := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("existing=%v", existing), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newUploadClient(t, clientOptions{})
|
||||
var first *node
|
||||
want := map[string][]byte{}
|
||||
if existing {
|
||||
first = uploadBytes(t, c, "f", []byte("first"))
|
||||
want["f"] = []byte("first")
|
||||
}
|
||||
srv.ResetRequests()
|
||||
srv.Inject(protontest.Fault{Method: "PUT", Pattern: commitPattern, Status: http.StatusUnprocessableEntity, Code: api.CodeInvalidValue, Times: 1})
|
||||
_, err := c.upload(context.Background(), uploadRequest{ParentID: c.rootID(), Name: "f", Existing: first, In: strings.NewReader("mine"), Size: 4, ModTime: testModTime})
|
||||
require.Error(t, err)
|
||||
assert.NotErrorIs(t, err, errUploadOutcomeUnknown)
|
||||
reqs := srv.Requests()
|
||||
assert.Equal(t, 1, reqs["PUT "+commitPattern])
|
||||
assert.Zero(t, reqs["POST "+linksPattern], "file read back")
|
||||
assert.Equal(t, 1, reqs[deleteDraftsRequest]+reqs[deleteRevisionRequest], "draft not deleted")
|
||||
assert.Empty(t, u.Drafts())
|
||||
requireUploaded(t, u, want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestUploadCommittedThenCancelledUnderLag cancels an upload once its
|
||||
// commit has succeeded and the first read back lagged: it returns at
|
||||
// once as unconfirmed and deletes nothing.
|
||||
func TestUploadCommittedThenCancelledUnderLag(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
c := newTestClientTransport(t, srv, u, clientOptions{DraftClientUID: testDraftClientUID}, func(next http.RoundTripper) http.RoundTripper {
|
||||
var committed atomic.Bool
|
||||
return roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
resp, err := next.RoundTrip(r)
|
||||
switch {
|
||||
case r.Method == "PUT" && strings.Contains(r.URL.Path, "/revisions/"):
|
||||
committed.Store(true)
|
||||
case committed.Load() && strings.HasSuffix(r.URL.Path, "/links"):
|
||||
cancel()
|
||||
}
|
||||
return resp, err
|
||||
})
|
||||
})
|
||||
srv.SetReadLag(readBackAttempts, protontest.LagBefore)
|
||||
start := time.Now()
|
||||
_, err := uploadContent(ctx, c, "f", strings.NewReader("mine"), 4)
|
||||
assert.ErrorIs(t, err, errUploadUnconfirmed)
|
||||
assert.Less(t, time.Since(start), 2*time.Second, "kept reading after cancel")
|
||||
requireNoDeletes(t, srv)
|
||||
requireUploaded(t, u, map[string][]byte{"f": []byte("mine")})
|
||||
}
|
||||
@@ -161,6 +161,7 @@ In `backend/s3/provider/YourProvider.yaml`
|
||||
type Quirks struct {
|
||||
ListVersion *int `yaml:"list_version,omitempty"` // 1 or 2
|
||||
ForcePathStyle *bool `yaml:"force_path_style,omitempty"` // true = path-style
|
||||
NoCheckBucket *bool `yaml:"no_check_bucket,omitempty"` // true = skip HeadBucket/CreateBucket
|
||||
ListURLEncode *bool `yaml:"list_url_encode,omitempty"`
|
||||
UseMultipartEtag *bool `yaml:"use_multipart_etag,omitempty"`
|
||||
UseAlreadyExists *bool `yaml:"use_already_exists,omitempty"`
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
name: AbrhaStorage
|
||||
description: Abrha Storage (ParsPack)
|
||||
region: {}
|
||||
endpoint: {}
|
||||
quirks:
|
||||
list_version: 1
|
||||
force_path_style: true # per-bucket hosts, e.g. https://c123456.parspack.net
|
||||
use_multipart_etag: false
|
||||
# ParsPack documents --s3-no-check-bucket for bucket check access errors.
|
||||
# https://parspack.com/blog/hosting/connecting-to-s3-bucket-with-cloud-storage
|
||||
no_check_bucket: true
|
||||
@@ -22,6 +22,7 @@ var NewYamlMap = orderedmap.New[string, string]
|
||||
type Quirks struct {
|
||||
ListVersion *int `yaml:"list_version,omitempty"` // 1 or 2
|
||||
ForcePathStyle *bool `yaml:"force_path_style,omitempty"` // true = path-style
|
||||
NoCheckBucket *bool `yaml:"no_check_bucket,omitempty"` // true = skip HeadBucket/CreateBucket
|
||||
ListURLEncode *bool `yaml:"list_url_encode,omitempty"`
|
||||
UseMultipartEtag *bool `yaml:"use_multipart_etag,omitempty"`
|
||||
UseAlreadyExists *bool `yaml:"use_already_exists,omitempty"`
|
||||
|
||||
@@ -470,6 +470,8 @@ rclone does if you know the bucket exists already.
|
||||
It can also be needed if the user you are using does not have bucket
|
||||
creation permissions. Before v1.52.0 this would have passed silently
|
||||
due to a bug.
|
||||
|
||||
Some providers (e.g. Abrha Storage) set this automatically.
|
||||
`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
@@ -1747,6 +1749,12 @@ func setQuirks(opt *Options, provider *Provider) {
|
||||
opt.ForcePathStyle = false
|
||||
}
|
||||
|
||||
// Some providers reject HeadBucket and CreateBucket for keys that can
|
||||
// still read and write objects. Skip that check when required.
|
||||
if provider.Quirks.NoCheckBucket != nil && *provider.Quirks.NoCheckBucket {
|
||||
opt.NoCheckBucket = true
|
||||
}
|
||||
|
||||
// Set the correct list version if not manually set
|
||||
if opt.ListVersion == 0 {
|
||||
if provider.Quirks.ListVersion != nil {
|
||||
|
||||
@@ -973,3 +973,13 @@ func TestBufferForObjectLockMD5(t *testing.T) {
|
||||
assert.Equal(t, inUse, pool.Global().InUse(), "pool buffers leaked")
|
||||
})
|
||||
}
|
||||
|
||||
func TestSetQuirksNoCheckBucket(t *testing.T) {
|
||||
opt := &Options{}
|
||||
setQuirks(opt, loadProvider("AbrhaStorage"))
|
||||
assert.True(t, opt.NoCheckBucket)
|
||||
|
||||
opt = &Options{}
|
||||
setQuirks(opt, loadProvider("AWS"))
|
||||
assert.False(t, opt.NoCheckBucket)
|
||||
}
|
||||
@@ -912,11 +912,13 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, obj
|
||||
|
||||
// close commits the upload, failing with ErrInvalidPart and the
|
||||
// upload left open if the streamed parts don't form the complete
|
||||
// object. Any other failure is of the commit, so the upload can't be
|
||||
// retried and forgetUpload tears it down. (After a failed commit the
|
||||
// abort it does is a no-op.)
|
||||
if err := up.close(); err != nil {
|
||||
if !errors.Is(err, gofakes3.ErrInvalidPart) {
|
||||
// object, with ErrSlowDown if acknowledged parts are still being
|
||||
// written, or with the context's error if the request is cancelled
|
||||
// while waiting for them. Any other failure is of the commit, so the
|
||||
// upload can't be retried and forgetUpload tears it down. (After a
|
||||
// failed commit the abort it does is a no-op.)
|
||||
if err := up.close(ctx); err != nil {
|
||||
if !errors.Is(err, gofakes3.ErrInvalidPart) && !errors.Is(err, gofakes3.ErrSlowDown) && err != ctx.Err() {
|
||||
b.forgetUpload(uploadID, up)
|
||||
}
|
||||
return "", "", err
|
||||
@@ -1138,11 +1140,30 @@ func (up *multipartUpload) validate(input *gofakes3.CompleteMultipartUploadReque
|
||||
// the upload is left open. The check and the commit share one critical
|
||||
// section so no late part can slip into the stream between them.
|
||||
//
|
||||
// Parts which were acknowledged but are still being written to the sink
|
||||
// are waited for. If that takes longer than multipartWaitTimeout,
|
||||
// gofakes3.ErrSlowDown is returned and the upload is left open. If ctx is
|
||||
// done first, ctx.Err() is returned and the upload is left open.
|
||||
//
|
||||
// Returns ErrNoSuchUpload if the upload was aborted by a concurrent
|
||||
// AbortMultipartUpload, in which case nothing has been committed - the
|
||||
// upload must not be reported as complete.
|
||||
func (up *multipartUpload) close() error {
|
||||
func (up *multipartUpload) close(ctx context.Context) error {
|
||||
timer := time.NewTimer(multipartWaitTimeout)
|
||||
defer timer.Stop()
|
||||
up.mu.Lock()
|
||||
for up.pumping && !up.closed {
|
||||
changed := up.changed
|
||||
up.mu.Unlock()
|
||||
select {
|
||||
case <-changed:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return gofakes3.ErrSlowDown
|
||||
}
|
||||
up.mu.Lock()
|
||||
}
|
||||
if up.closed {
|
||||
aborted := up.aborted
|
||||
up.mu.Unlock()
|
||||
|
||||
@@ -565,13 +565,13 @@ func TestMultipartCloseAfterAbort(t *testing.T) {
|
||||
up := newMultipartUpload("bucket", "key", "bucket/key", "bucket/key", nil, 0)
|
||||
up.fh = &stubSink{}
|
||||
require.NoError(t, up.abort())
|
||||
require.ErrorIs(t, up.close(), gofakes3.ErrNoSuchUpload)
|
||||
require.ErrorIs(t, up.close(context.Background()), gofakes3.ErrNoSuchUpload)
|
||||
|
||||
// close stays idempotent after a successful close.
|
||||
up = newMultipartUpload("bucket", "key", "bucket/key", "bucket/key", nil, 0)
|
||||
up.fh = &stubSink{}
|
||||
require.NoError(t, up.close())
|
||||
require.NoError(t, up.close())
|
||||
require.NoError(t, up.close(context.Background()))
|
||||
require.NoError(t, up.close(context.Background()))
|
||||
}
|
||||
|
||||
// TestMultipartCloseIncomplete checks that close refuses to commit while a
|
||||
@@ -593,7 +593,7 @@ func TestMultipartCloseIncomplete(t *testing.T) {
|
||||
require.Equal(t, turnBuffer, turn)
|
||||
require.NoError(t, up.bufferPart(context.Background(), 2, int64(len(contents)), md5Sum[:], rw))
|
||||
|
||||
require.ErrorIs(t, up.close(), gofakes3.ErrInvalidPart)
|
||||
require.ErrorIs(t, up.close(context.Background()), gofakes3.ErrInvalidPart)
|
||||
assert.False(t, sink.closed)
|
||||
|
||||
// The upload is still open so abort tears it down.
|
||||
@@ -1644,6 +1644,162 @@ func (s *flakySink) Write(p []byte) (int, error) {
|
||||
return n, errBoom
|
||||
}
|
||||
|
||||
// gatedSink is a sink which blocks writes past its first gateAt bytes until
|
||||
// release is closed, signalling blocked when the first such write starts.
|
||||
type gatedSink struct {
|
||||
bufferSink
|
||||
gateAt int
|
||||
once sync.Once
|
||||
blocked chan struct{}
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func newGatedSink(gateAt int) *gatedSink {
|
||||
return &gatedSink{gateAt: gateAt, blocked: make(chan struct{}), release: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (s *gatedSink) Write(p []byte) (int, error) {
|
||||
s.mu.Lock()
|
||||
written := s.buf.Len()
|
||||
s.mu.Unlock()
|
||||
if written+len(p) > s.gateAt {
|
||||
s.once.Do(func() { close(s.blocked) })
|
||||
<-s.release
|
||||
}
|
||||
return s.bufferSink.Write(p)
|
||||
}
|
||||
|
||||
// startPumpBehindRetry recreates a client retrying part 1 because its
|
||||
// request was slow to answer: part 2 arrives first and is buffered, then
|
||||
// the request for part 1 streams it and blocks pumping part 2 into sink,
|
||||
// and the retry of part 1 is acknowledged. It returns the result of the
|
||||
// original part 1 request.
|
||||
func startPumpBehindRetry(t *testing.T, b *s3Backend, bucket, object string, uploadID gofakes3.UploadID, sink *gatedSink, part1, part2 []byte) <-chan error {
|
||||
ctx := context.Background()
|
||||
_, err := b.UploadPart(ctx, bucket, object, uploadID, 2, int64(len(part2)), bytes.NewReader(part2))
|
||||
require.NoError(t, err)
|
||||
part1Done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := b.UploadPart(ctx, bucket, object, uploadID, 1, int64(len(part1)), bytes.NewReader(part1))
|
||||
part1Done <- err
|
||||
}()
|
||||
<-sink.blocked
|
||||
_, err = b.UploadPart(ctx, bucket, object, uploadID, 1, int64(len(part1)), bytes.NewReader(part1))
|
||||
require.NoError(t, err)
|
||||
return part1Done
|
||||
}
|
||||
|
||||
// TestMultipartCloseWaitsForPump checks that completing an upload whose
|
||||
// parts have all been acknowledged waits for parts still being written to
|
||||
// the sink, rather than rejecting the completion.
|
||||
func TestMultipartCloseWaitsForPump(t *testing.T) {
|
||||
b, _, bucket := newPutTestBackend(t, "", nil)
|
||||
const object = "close-while-pumping.bin"
|
||||
uploadID, _ := newBufferSinkUpload(t, b, bucket, object)
|
||||
up, err := b.loadUpload(context.Background(), uploadID)
|
||||
require.NoError(t, err)
|
||||
part1 := []byte(random.String(1024))
|
||||
part2 := []byte(random.String(1024))
|
||||
sink := newGatedSink(len(part1))
|
||||
up.fh = sink
|
||||
part1Done := startPumpBehindRetry(t, b, bucket, object, uploadID, sink, part1, part2)
|
||||
|
||||
closed := make(chan error, 1)
|
||||
go func() { closed <- up.close(context.Background()) }()
|
||||
select {
|
||||
case err := <-closed:
|
||||
t.Fatalf("close returned %v while part 2 was still being written", err)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
|
||||
close(sink.release)
|
||||
require.NoError(t, <-closed)
|
||||
require.NoError(t, <-part1Done)
|
||||
assert.Equal(t, append(part1, part2...), sink.Bytes())
|
||||
}
|
||||
|
||||
// TestMultipartCompleteWaitingForPumpGivesUp checks that a completion which
|
||||
// waits too long for acknowledged parts to be written tells the client to
|
||||
// slow down and keeps the upload, so the client can retry it.
|
||||
func TestMultipartCompleteWaitingForPumpGivesUp(t *testing.T) {
|
||||
b, _, bucket := newPutTestBackend(t, "", nil)
|
||||
oldTimeout := multipartWaitTimeout
|
||||
multipartWaitTimeout = 100 * time.Millisecond
|
||||
t.Cleanup(func() { multipartWaitTimeout = oldTimeout })
|
||||
ctx := context.Background()
|
||||
const object = "complete-gives-up.bin"
|
||||
uploadID, _ := newBufferSinkUpload(t, b, bucket, object)
|
||||
up, err := b.loadUpload(ctx, uploadID)
|
||||
require.NoError(t, err)
|
||||
part1 := []byte(random.String(1024))
|
||||
part2 := []byte(random.String(1024))
|
||||
sink := newGatedSink(len(part1))
|
||||
up.fh = sink
|
||||
part1Done := startPumpBehindRetry(t, b, bucket, object, uploadID, sink, part1, part2)
|
||||
md5s := [][16]byte{md5.Sum(part1), md5.Sum(part2)}
|
||||
|
||||
_, _, err = b.CompleteMultipartUpload(ctx, bucket, object, uploadID, &gofakes3.CompleteMultipartUploadRequest{
|
||||
Parts: []gofakes3.CompletedPart{
|
||||
{PartNumber: 1, ETag: hex.EncodeToString(md5s[0][:])},
|
||||
{PartNumber: 2, ETag: hex.EncodeToString(md5s[1][:])},
|
||||
},
|
||||
})
|
||||
assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrSlowDown), "want SlowDown, got %v", err)
|
||||
|
||||
close(sink.release)
|
||||
require.NoError(t, <-part1Done)
|
||||
_, err = b.loadUpload(ctx, uploadID)
|
||||
require.NoError(t, err, "the upload was torn down instead of being kept for a retry")
|
||||
require.NoError(t, up.close(context.Background()))
|
||||
assert.Equal(t, append(part1, part2...), sink.Bytes())
|
||||
}
|
||||
|
||||
// TestMultipartCompleteWaitingForPumpCancelled checks that a completion
|
||||
// whose request is cancelled while acknowledged parts are being written
|
||||
// returns at once, rather than after multipartWaitTimeout, and keeps the
|
||||
// upload so the client can retry it.
|
||||
func TestMultipartCompleteWaitingForPumpCancelled(t *testing.T) {
|
||||
b, _, bucket := newPutTestBackend(t, "", nil)
|
||||
ctx := context.Background()
|
||||
const object = "complete-cancelled.bin"
|
||||
uploadID, _ := newBufferSinkUpload(t, b, bucket, object)
|
||||
up, err := b.loadUpload(ctx, uploadID)
|
||||
require.NoError(t, err)
|
||||
part1 := []byte(random.String(1024))
|
||||
part2 := []byte(random.String(1024))
|
||||
sink := newGatedSink(len(part1))
|
||||
up.fh = sink
|
||||
part1Done := startPumpBehindRetry(t, b, bucket, object, uploadID, sink, part1, part2)
|
||||
md5s := [][16]byte{md5.Sum(part1), md5.Sum(part2)}
|
||||
|
||||
completeCtx, cancel := context.WithCancel(ctx)
|
||||
completed := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, err := b.CompleteMultipartUpload(completeCtx, bucket, object, uploadID, &gofakes3.CompleteMultipartUploadRequest{
|
||||
Parts: []gofakes3.CompletedPart{
|
||||
{PartNumber: 1, ETag: hex.EncodeToString(md5s[0][:])},
|
||||
{PartNumber: 2, ETag: hex.EncodeToString(md5s[1][:])},
|
||||
},
|
||||
})
|
||||
completed <- err
|
||||
}()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
cancel()
|
||||
select {
|
||||
case err := <-completed:
|
||||
assert.ErrorIs(t, err, context.Canceled)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the cancelled completion kept waiting for the pump")
|
||||
}
|
||||
|
||||
close(sink.release)
|
||||
require.NoError(t, <-part1Done)
|
||||
_, err = b.loadUpload(ctx, uploadID)
|
||||
require.NoError(t, err, "the upload was torn down instead of being kept for a retry")
|
||||
require.NoError(t, up.close(ctx))
|
||||
assert.Equal(t, append(part1, part2...), sink.Bytes())
|
||||
}
|
||||
|
||||
// TestMultipartPumpFailure checks that an upload is failed when a part
|
||||
// which was buffered, and so already acknowledged to the client, can't be
|
||||
// written to the sink, rather than a resend of it being appended to what
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
# Architecture
|
||||
|
||||
Fork-only orientation for agents. The summary in `AGENTS.md` ("Architecture")
|
||||
is upstream's and is not repeated here; this page adds what that summary
|
||||
leaves out. Update in the same commit as any structural change.
|
||||
|
||||
## Map
|
||||
|
||||
| Path | Responsibility |
|
||||
|---|---|
|
||||
| `rclone.go` | Entry point; blank-imports `backend/all` and `cmd/all` |
|
||||
| `backend/<name>/` | One storage backend per package, registered in `init()` |
|
||||
| `cmd/<name>/` | One cobra command per package |
|
||||
| `fs/` | Core interfaces, config, operations, sync, accounting |
|
||||
| `vfs/` | Virtual filesystem used by mount and serve |
|
||||
| `lib/` | Shared helpers: `rest`, `pacer`, `dircache`, `oauthutil`, `encoder` |
|
||||
| `fstest/` | Test framework: `fstests` (backend contract), `test_all` (runner) |
|
||||
| `docs/content/` | Hugo website source; partly autogenerated |
|
||||
| `docs/specs/`, `docs/plans/` | Fork-only agent workflow artefacts |
|
||||
| `bin/` | Build and docs scripts |
|
||||
|
||||
## Test layers
|
||||
|
||||
- Unit tests: `RCLONE_CONFIG=/notfound go test ./...` - no credentials.
|
||||
- Backend contract: each `backend/<name>/<name>_test.go` calls
|
||||
`fstests.Run()` against a remote named `Test<Name>:` in the user's rclone
|
||||
config. Without that remote the test skips, so a green `go test` on a
|
||||
backend package says nothing about the backend against the real service.
|
||||
- Integration: `fstest/test_all` runs the backend contract plus the
|
||||
`fs/operations`, `fs/sync`, `vfs` etc. suites against the backends listed in
|
||||
`fstest/test_all/config.yaml`.
|
||||
|
||||
## Invariants
|
||||
|
||||
- Backwards compatibility of commands, flags, config keys and the rc API.
|
||||
- `fs` and `vfs` contain no backend-specific code.
|
||||
- cgo use differs per release target (the `cgo` and `build_flags` columns of
|
||||
the matrix in `.github/workflows/build.yml`), so a backend cannot assume cgo.
|
||||
|
||||
## `backend/protondrive`
|
||||
|
||||
A native client of the Proton Drive API, following Proton's Drive SDK
|
||||
(spec: `docs/specs/001-protondrive-tier1.md`).
|
||||
|
||||
| File | Responsibility |
|
||||
|---|---|
|
||||
| `protondrive.go` | `Fs`, `Object`, options, `Config` sign-in, dircache glue, `translateError` |
|
||||
| `session.go` | SRP and browser (session fork) sign-in, token refresh, app version |
|
||||
| `account.go` | User and address keys, public-key lookups |
|
||||
| `crypto.go` | Every OpenPGP operation |
|
||||
| `nodes.go` | The `client`: volume bootstrap, pacing, listing, folder/rename/move/copy/trash |
|
||||
| `cache.go` | In-memory node and listing cache |
|
||||
| `upload.go`, `download.go` | Block pipelines; `download.go` also keeps the per-client cache of block sizes learnt for files that don't record them |
|
||||
| `api/` | Wire types |
|
||||
| `protontest/` | Fake Proton server, imported by tests only; never imports the backend |
|
||||
|
||||
- **The seam:** `protondrive.go` holds no cryptography and no API routes;
|
||||
the other files never take or return `fs.Fs`/`fs.Object`.
|
||||
- No package-level mutable state except `unlockMu` (key unlocking is
|
||||
serialised across `Fs` instances). Each `Fs` has its own client, session
|
||||
and cache; credentials go through a per-`Fs` `configStore` over the
|
||||
remote's config.
|
||||
- `fake_test.go` runs the full `fstests` contract suite against
|
||||
`protontest` through the hidden `endpoint` option, on every
|
||||
`go test`; `protondrive_test.go` is the live suite (`TestProtonDrive:`).
|
||||
- `fstest/testserver/init.d/TestProtonDriveFake` builds and starts
|
||||
`protontest/cmd/protonfake` for `test_all`, which runs the wider suites
|
||||
against the `TestProtonDriveFake:` remote. Fork CI is
|
||||
`.gitea/workflows/ci.yml`.
|
||||
|
||||
- User docs: `docs/content/protondrive.md` (hand-written parts; the
|
||||
options section is generated from the `Help:` fields).
|
||||
`docs/specs/001-conformance-checklist.md` maps each SDK module to its Go
|
||||
file and the tag it was last checked against.
|
||||
- Not implemented yet: events and `ChangeNotify` (M5); the persistent
|
||||
metadata cache, rclone Metadata and the small-file upload path (M6).
|
||||
Ranged reads check block hashes but not the signed manifest. Open items
|
||||
and unrun live checks are in `docs/TODO.md`.
|
||||
|
||||
## Known rough edges
|
||||
|
||||
- See `docs/TODO.md` for host tooling gaps.
|
||||
@@ -0,0 +1,481 @@
|
||||
# Decisions
|
||||
|
||||
Fork-only. Append-only, newest first. Never edit or delete an entry - supersede
|
||||
it with a new one that links back. One entry per decision a future maintainer
|
||||
would otherwise have to reverse-engineer.
|
||||
No index: `grep -n '^## ' docs/DECISIONS.md` lists every entry.
|
||||
|
||||
## 2026-11-12 - Proton Drive maps only the app-version codes to "upgrade rclone"
|
||||
|
||||
**Status:** accepted (handoff ruling 30). Supersedes spec 6.6's
|
||||
"2000/2028 ... to a fatal error telling the user to upgrade rclone".
|
||||
**Context:** 2000 is Proton's generic "requirements not met" code, which
|
||||
the server returns for a stale `OriginalHash` or `CurrentRevisionID`, as
|
||||
after a rename whose response was lost and which was resent. Mapping it to
|
||||
a fatal "upgrade rclone" error aborted whole runs over a rename that had
|
||||
succeeded. 2028 means a feature is disabled, rate-limited or blocked for
|
||||
abuse (`driveTypes.ts`). The code Proton returns for a rejected app
|
||||
version is 5003 ("You need to update this app", `coreTypes.ts`), and 5001
|
||||
for a missing one (go-proton-api), neither of which was mapped.
|
||||
**Decision:** 5001 and 5003 are a fatal error telling the user to upgrade
|
||||
rclone. 2000 is a no-retry error keeping Proton's message. 2028 is retried
|
||||
through the pacer (honouring `Retry-After` when sent), then returned as
|
||||
it is. 2064 (malformed header) is left unmapped.
|
||||
**Alternatives:** Keeping the spec's mapping (false fatal errors);
|
||||
treating 2028 as a no-retry error (fails at once on rate limiting).
|
||||
**Consequences:** A feature Proton disables outright costs the low-level
|
||||
retries before its error is returned.
|
||||
|
||||
## 2026-11-12 - Proton Drive caps the Argon2 cost of node and share keys
|
||||
|
||||
**Status:** accepted
|
||||
**Context:** go-crypto unlocks an Argon2-locked key with whatever
|
||||
parameters it carries, up to 2 TiB of memory and 255 passes, and runs the
|
||||
KDF before it can tell the passphrase is wrong. Node and share keys come
|
||||
from the server, or from other users' clients in shared folders, so one
|
||||
crafted key could crash every listing that includes it.
|
||||
**Decision:** Before unlocking a node or share key, every secret key
|
||||
packet's S2K is checked: Argon2 may use at most 4 passes, parallelism 8
|
||||
and 2^17 KiB (128 MiB). gopenpgp and OpenPGP.js write 3, 4 and 64 MiB by
|
||||
default, and Proton's node keys use an iterated and salted S2K, whose
|
||||
one-byte count encoding already bounds it to 4 times the default. A key
|
||||
beyond the limits is an integrity error, making the node unreadable, and
|
||||
`ignore_signature_errors` doesn't change that.
|
||||
**Alternatives:** Refusing Argon2 on node keys outright (no genuine node
|
||||
key is known to use it, but a future client might); capping the iterated
|
||||
count below the encoding's maximum (could refuse genuine old keys for a
|
||||
bounded cost).
|
||||
**Consequences:** At most 128 MiB per unlock while a listing decrypts
|
||||
nodes. The account's own user keys aren't checked yet (`docs/TODO.md`).
|
||||
|
||||
## 2026-11-12 - Proton Drive refuses a file whose content key is unsigned
|
||||
|
||||
**Status:** accepted (handoff ruling 29)
|
||||
**Context:** Anyone holding a file's public node key, which the server
|
||||
stores, can make a content key packet for it. Proton's SDK checks the
|
||||
content key packet's signature only when one is present. A server could
|
||||
then send its own content key without a signature: rclone would encrypt
|
||||
every update of the file with a key the server knows, and a ranged read
|
||||
(which can't check the manifest) would return whatever the server
|
||||
encrypted under it.
|
||||
**Decision:** A content key whose signature is missing or invalid makes
|
||||
the file unverified, as an unverified node key does (rulings 18 and 22):
|
||||
reads, full and ranged, and updates are refused unless
|
||||
`ignore_signature_errors` is set, when rclone logs an `ERROR` once per file
|
||||
and operation and goes ahead. The pre-2023 fallback to the account's other
|
||||
address keys still applies to a signature which is present but fails, as
|
||||
in the SDK. Integrity checks are never turned off. This is stricter than
|
||||
the SDK.
|
||||
**Alternatives:** Following the SDK (leaves the hole); a creation-date
|
||||
exemption for unsigned keys (the date is server-supplied, so it would
|
||||
exempt anything the server back-dates).
|
||||
**Consequences:** A genuine file whose content key was never signed, if
|
||||
any client ever wrote one, needs `ignore_signature_errors`. Every file in
|
||||
the golden fixtures, written by Proton's web app, CLI and the old
|
||||
library, has a signed content key.
|
||||
|
||||
## 2026-11-12 - Proton Drive deletes an upload's draft only on an explicit refusal
|
||||
|
||||
**Status:** accepted. Supersedes the read-back half of handoff ruling 11
|
||||
("a draft is deleted only on positive proof it never committed", where a
|
||||
read showing the draft or the old revision counted as proof).
|
||||
**Context:** Proton's server can take a moment to show a commit (entry
|
||||
below). After a failed commit, one stale read showing the draft or the
|
||||
base revision made the client delete the draft, but the commit may
|
||||
have landed: `delete_multiple` against what is now an active file, or
|
||||
`DELETE .../revisions/{id}` against what is now the active revision. What
|
||||
the real server does with those is unconfirmed, so this could lose data.
|
||||
**Decision:** The draft is deleted only when the server explicitly
|
||||
refused the commit, meaning a 4xx `api.Error` (not 408) with a Proton code,
|
||||
on a single attempt that the pacer did not resend. A resend after
|
||||
5xx/408/network/timeout makes the outcome unknown even if the final
|
||||
answer is a refusal, because an earlier attempt may have landed. No
|
||||
read-back decides deletion, and a refusal is not read back at all. Every
|
||||
other failure (5xx, network, timeout, cancelled ctx, resent attempts)
|
||||
returns `errUploadOutcomeUnknown` and deletes nothing. The read after
|
||||
such a failure can still turn it into a success when it shows the
|
||||
upload's revision. It retries through lag while the caller's ctx lives,
|
||||
and reads once, detached, if ctx is done.
|
||||
**Alternatives:** Requiring every read across the whole lag window to
|
||||
show the pre-commit state (a heuristic, as the lag length is unmeasured);
|
||||
the old single-read proof.
|
||||
**Consequences:** More drafts are left behind after ambiguous failures.
|
||||
The next upload of the name replaces them, otherwise they stay (see
|
||||
`docs/TODO.md`). A refused commit fails at once, with no reads.
|
||||
|
||||
## 2026-11-12 - Proton Drive uploads tolerate read-after-write lag, but never invent the result
|
||||
|
||||
**Status:** accepted
|
||||
**Context:** Against the real service an upload whose commit succeeded
|
||||
failed because the node read back just after was missing or still a
|
||||
draft with an empty active revision (`errUploadSuperseded` with revision
|
||||
"", and "object not found"). The client read the node once and trusted it.
|
||||
**Decision:** After a commit the node is read again with backoff (100 ms
|
||||
doubling to 2 s, 8 reads, about 7 s) while it is missing, active with no
|
||||
revision, or - when the commit returned success - still as it was before
|
||||
the commit. `errUploadSuperseded` is kept for a non-empty revision which
|
||||
is neither ours nor the one we started from. If the lag outlasts the
|
||||
reads, the upload returns `errUploadUnconfirmed` (no low-level retry, as
|
||||
for `errUploadOutcomeUnknown`), deletes nothing, and a later sync pass
|
||||
finds the file. The same not-found retry covers a copy's read back and,
|
||||
with `enable_caching` off, a folder to write into (with it on, a folder
|
||||
the client made is cached, so a miss means it is gone).
|
||||
**Alternatives:** Returning success with a node built from what was
|
||||
committed (the caller then reads, moves or updates through it, and that
|
||||
needs the decrypted keys and server metadata the client would be
|
||||
inventing; a wrong node would be cached and acted on); retrying every
|
||||
read after any write (slows every genuine not-found).
|
||||
**Consequences:** With `enable_caching` off, a put into a folder deleted
|
||||
elsewhere takes about 7 s longer to fail. A lag longer than 7 s still fails the upload,
|
||||
now with an error saying it was committed. See `docs/TODO.md` for the
|
||||
read-after-write spots not yet tolerant.
|
||||
|
||||
## 2026-11-12 - Proton Drive golden captures fail closed
|
||||
|
||||
**Status:** accepted; amends "Proton Drive golden fixtures are re-keyed
|
||||
captures" below
|
||||
**Context:** Review found the capture could copy account data into a
|
||||
fixture: an intended-recipient fingerprint of a key not being replaced (in
|
||||
the notes), and the user ID of a node key. It also reused `-remote`, so a
|
||||
mistyped `-run` would start the writing integration suite against the
|
||||
account.
|
||||
**Decision:** The capture names its folder with `-capture-remote` and
|
||||
refuses to run with `-remote` set. Every key written has the user ID
|
||||
`golden <golden@example.invalid>`; a node key whose user ID holds an email
|
||||
other than the account's or the generic ones Proton's clients use
|
||||
(`no-reply@proton.me`, `noreply@protonmail.com`) fails the capture. A
|
||||
signature by, or intended for, a key not being replaced fails the capture,
|
||||
and the error names no key. A salt notation is written only where the
|
||||
original had one (with a new salt). `TestGoldenFixtures` also walks every
|
||||
key, key packet and signature, inside encrypted messages too, and fails
|
||||
on any key ID, fingerprint or user ID not of the test keys.
|
||||
**Consequences:** What the committed fixtures expose, by design: with the
|
||||
test keys in `keys.json`, anyone can decrypt the captured names, extended
|
||||
attributes (as the client wrote them: check them for camera or location
|
||||
fields before committing), literal-packet file names and dates, folder
|
||||
hash keys (so whether a guessed name exists in those real folders can be
|
||||
tested) and file contents; and the address key's creation time is kept.
|
||||
Node passphrases and the account's keys are not exposed.
|
||||
|
||||
## 2026-11-12 - Proton Drive golden fixtures are re-keyed captures
|
||||
|
||||
**Status:** accepted
|
||||
**Context:** Plan Task 21 asks for fixtures of nodes written by Proton's
|
||||
web app and official CLI. Captured as they are, they would carry the test
|
||||
account's keys, IDs and emails. The owner chose to re-encrypt them under
|
||||
keys made for the tests.
|
||||
**Decision:** `TestGoldenCapture` (`-capture DIR -capture-label NAME
|
||||
-remote NAME:PATH`) writes `backend/protondrive/testdata/golden/<label>/`,
|
||||
read by `TestGoldenFixtures` from the fake server with every signature
|
||||
checked. Preserved: every block byte for byte; every SEIPD packet which
|
||||
holds no signature (re-wrapped session key only); the session key, cipher,
|
||||
SEIPD version, AEAD mode and chunk size, compression algorithm and literal
|
||||
data packet (format, file name, date, content) of names, hash keys and
|
||||
extended attributes; signature version, type, hash, creation time,
|
||||
lifetime and notations; key algorithm, version, creation time and AEAD
|
||||
support; timestamps, sizes, flags, media types, block sizes and whether
|
||||
they are recorded. Replaced: all keys (address, node, the captured
|
||||
folder's parent) by new ones; every PKESK, re-wrapped in the same version;
|
||||
every signature, re-made by the key replacing its signer (an embedded one
|
||||
forces its SEIPD packet to be encrypted again with the same session key
|
||||
and parameters, as a signature can't change in place); node passphrases,
|
||||
new ones in a message of the same format; the captured folder's name hash;
|
||||
intended recipients and signer user IDs; emails by
|
||||
`golden@example.invalid`; IDs by placeholders. Dropped: block signatures
|
||||
(the backend doesn't read them), block URLs and tokens. Each node lists
|
||||
what was done to it in `Rekeyed`. Files with thumbnails are refused (the
|
||||
fake can't serve them); the address key passphrase is sealed to a user key
|
||||
the fake generates, so no user key is written.
|
||||
**Alternatives:** Raw captures (account secrets in the repository);
|
||||
fixtures re-made with gopenpgp from the decrypted content (loses the
|
||||
other clients' packet formats, the point of the exercise).
|
||||
**Consequences:** The fixtures test the formats of names, hash keys and
|
||||
extended attributes as other clients lay them out, but their signature
|
||||
bytes are ours. Packet headers are re-serialised with fixed lengths.
|
||||
|
||||
## 2026-11-12 - Proton Drive is a native port of Proton's Drive SDK
|
||||
|
||||
**Status:** accepted
|
||||
**Context:** With the bridge libraries gone (plan Task 19), the backend
|
||||
is a Go client written for rclone. Its behaviour has to be checked against
|
||||
something as Proton changes the service, and the user docs still
|
||||
described the old libraries.
|
||||
**Decision:** The backend is maintained as a port of Proton's Drive SDK,
|
||||
module by module, tracked in `docs/specs/001-conformance-checklist.md`
|
||||
against SDK commit `05c9115955589d9f6ab7392a36f8f0cb39327133` (whose
|
||||
`client/js/CHANGELOG.md` ends at `js/v0.23.0`; that tag is not on the
|
||||
commit), with its deliberate differences (stricter trust
|
||||
rules, no small-file path, no events, no thumbnails) listed per module.
|
||||
The user docs describe only what the code does and the protocol limits
|
||||
the SDK shares, with no fork-only content, so they could go upstream.
|
||||
**Alternatives:** Binding to the SDK itself (it is TypeScript and C#;
|
||||
rclone refuses cgo and helper binaries); writing from observed traffic
|
||||
alone, as the old libraries did (no reference to re-check against).
|
||||
**Consequences:** Each SDK update needs a walk of the checklist: a diff
|
||||
of the listed files between the checked commit and the new one. The
|
||||
docs page must change with any behaviour change, and its generated
|
||||
options section is left to `make backenddocs` at release.
|
||||
|
||||
## 2026-11-12 - Proton Drive keeps frozen old-library fixtures, not the old library
|
||||
|
||||
**Status:** accepted
|
||||
**Context:** `go-proton-api` v1.0.4 and `Proton-API-Bridge` v1.0.5 stayed
|
||||
in `go.mod` only for differential tests checking that the new client and
|
||||
the old library read each other's output (plan Task 19).
|
||||
**Decision:** The differential tests and both libraries were removed. What
|
||||
the old library wrote stays checked through the frozen fixtures in
|
||||
`backend/protondrive/testdata/oldlib/` (`TestFrozenFixtures`,
|
||||
`TestFrozenFileFixture`), which can no longer be regenerated.
|
||||
**Alternatives:** Keeping the libraries as test-only requirements (they
|
||||
would stay in `go.mod` and its vulnerability surface, and upstream would
|
||||
carry them).
|
||||
**Consequences:** Multi-block and empty old-library files, and old-reader
|
||||
compatibility of new writes, are no longer tested (`docs/TODO.md`).
|
||||
|
||||
## 2026-11-12 - Test-account usernames removed from git history by force-push
|
||||
|
||||
**Status:** accepted
|
||||
**Context:** The first Proton Drive handoff commit contained the two
|
||||
test-account usernames (no passwords or tokens); a later commit removed
|
||||
them from the file but they stayed in history on the Gitea remote.
|
||||
**Decision:** On the owner's explicit instruction, that commit was
|
||||
rebuilt without the usernames, the following commits were replayed with
|
||||
identical trees and metadata, and `master` was force-pushed once with
|
||||
`--force-with-lease`. The rule against force-pushing still stands.
|
||||
**Consequences:** Commit hashes from `901d7fd5b` onward changed (mapping
|
||||
in the handoff's "Open questions" section). The old objects may stay
|
||||
reachable on the server until Gitea garbage-collects the repository.
|
||||
|
||||
## 2026-11-12 - test_all shares one protonfake session across test processes
|
||||
|
||||
**Status:** accepted
|
||||
**Context:** `fstest/testserver/init.d/TestProtonDriveFake` starts one
|
||||
`protonfake` per `test_all` run (reference counted by `run.bash`), and
|
||||
every test process gets the same session through
|
||||
`RCLONE_CONFIG_TESTPROTONDRIVEFAKE_*` variables. Refresh tokens are
|
||||
single use and a process can't see another's refresh, so one process
|
||||
refreshing would leave the rest with dead tokens.
|
||||
**Decision:** Share the session; change nothing in `protontest`,
|
||||
`protonfake` or the client. The client refreshes only after an HTTP 401
|
||||
(`session.call`); the fake answers 401 only for a session expired by
|
||||
`Server.ExpireAccessToken` or ended by logout or `RevokeSession`, and
|
||||
`protonfake` calls none of them. The only logouts are `Fs.Disconnect`
|
||||
and re-running `rclone config`, which no test in `fs/operations`,
|
||||
`fs/sync`, `vfs`, `cmd/bisync`, `cmd/gitannex` or `fstests` calls.
|
||||
**Alternatives:** A session per test process (`protonfake` would need a
|
||||
sign-in route for `test_all` to call, and the init.d protocol has one
|
||||
output per server); non-expiring or logout-tolerant tokens in
|
||||
standalone mode (behaviour no test needs, and a fake that differs from
|
||||
Proton).
|
||||
**Consequences:** A future test that calls `Disconnect`, or a fake that
|
||||
expires tokens on a timer, breaks every process sharing the run: the
|
||||
symptom is "not signed in" or refresh failures across all suites at
|
||||
once. Revisit then.
|
||||
|
||||
## 2026-11-11 - Proton Drive rewrite paused mid-plan and merged to master
|
||||
|
||||
- **Decision:** at the owner's request, work on plan 001 stopped after Task
|
||||
17 of 21 and the branch was fast-forwarded into `master` and pushed, with
|
||||
the plan, a handoff document and the execution workspace committed under
|
||||
`docs/plans/`.
|
||||
- **Why:** the owner wanted development wound down with nothing left only
|
||||
in an agent's conversation.
|
||||
- **Consequences:** `master` carries an unfinished plan, against the usual
|
||||
rule that plans are removed before merging; the user docs still describe
|
||||
the old backend; the old libraries are still in `go.mod`. The push to
|
||||
`master` was a one-off instruction - the rule against pushing to
|
||||
`master` stands. Resume from `docs/plans/001-protondrive-tier1-HANDOFF.md`.
|
||||
|
||||
## 2026-11-11 - Proton Drive logs what ignore_signature_errors lets through
|
||||
|
||||
**Status:** accepted
|
||||
**Context:** With `ignore_signature_errors` set, reads, writes, moves,
|
||||
copies and renames of nodes whose keys are unverified went ahead
|
||||
silently, although spec 6.4 promises a logged ERROR (Task 16 re-review
|
||||
2).
|
||||
**Decision:** Each such operation logs one ERROR per node and kind of
|
||||
operation for the life of the `Fs`, naming the node, its ID, the
|
||||
operation and why its key is unverified. Completes "Proton Drive won't
|
||||
vouch for a node it couldn't verify".
|
||||
**Alternatives:** Logging every call (floods a sync); a NOTICE (too
|
||||
easily missed for something permanent).
|
||||
**Consequences:** With the option set, moving or copying a node out of
|
||||
an unverified folder re-signs or re-homes it as the user's, so it is
|
||||
trusted from then on; the ERROR says so.
|
||||
|
||||
## 2026-11-11 - Proton Drive matches email addresses by ASCII case only
|
||||
|
||||
**Status:** accepted
|
||||
**Context:** The own-address check of "Proton Drive roots trust in the
|
||||
user's own addresses" ignored case, but the key lookup didn't, so a
|
||||
server could name the user's address in capitals and serve its own keys
|
||||
for it, at the root and below (Task 16 re-review 2).
|
||||
**Decision:** One rule everywhere: emails are compared after lowercasing
|
||||
ASCII letters only. An email matching one of the account's addresses
|
||||
under it is verified with the account's own keys and never looked up on
|
||||
the server, and other users' keys are cached under the same key. Unicode
|
||||
case folding is not used, as it maps the Kelvin sign to k and the long s
|
||||
to s, letting a non-ASCII address pass for an ASCII one.
|
||||
**Alternatives:** Exact matching (an address the server stores in other
|
||||
case would be looked up remotely); `strings.EqualFold` (Unicode folding).
|
||||
**Consequences:** Two addresses differing only in non-ASCII case are
|
||||
different addresses to rclone.
|
||||
|
||||
## 2026-11-11 - Proton Drive roots trust in the user's own addresses
|
||||
|
||||
**Status:** accepted
|
||||
**Context:** "Proton Drive distrusts what a forged folder key vouches
|
||||
for" and "Proton Drive refuses a file whose node key signature fails"
|
||||
read as if every key were checked against keys rclone can trust. But
|
||||
the keys of another user's email come from the server, so a server can
|
||||
forge a share, root or node "signed" by a foreign address it vouches
|
||||
for (Task 16 re-review).
|
||||
**Decision:** Corrects those entries. The My Files share's creator and
|
||||
the root folder's signer must be addresses whose keys the account
|
||||
holds (compared ignoring case); otherwise the share or root key is
|
||||
unverified, which makes the whole drive unreadable and unwritable
|
||||
without `ignore_signature_errors`, and the error says why. Below the
|
||||
root, nodes signed under another user's email are still verified
|
||||
against server-supplied keys: a protocol limit Proton's SDK shares,
|
||||
documented in spec section 7 and TODO.
|
||||
**Alternatives:** Pinning other users' keys on first use (state rclone
|
||||
doesn't keep); trusting server keys at the root too.
|
||||
**Consequences:** A folder genuinely signed by an address since deleted
|
||||
from the account, or whose keys no longer unlock, is unverified: writes
|
||||
into it, and reads beneath it, need `ignore_signature_errors`.
|
||||
|
||||
## 2026-11-11 - Proton Drive won't vouch for a node it couldn't verify
|
||||
|
||||
**Status:** accepted
|
||||
**Context:** Refusing writes into a folder with an unverified key left
|
||||
the other direction open: moving or copying an anonymous node out of it
|
||||
has rclone sign its server-made passphrase as the user's, after which it
|
||||
and everything beneath it verified (Task 16 re-review). And a node made
|
||||
under such a folder with `ignore_signature_errors` set was signed by the
|
||||
user, so verified later, although whoever made the folder's key can
|
||||
decrypt its passphrase.
|
||||
**Decision:** Supersedes the inheritance rule of "Proton Drive distrusts
|
||||
what a forged folder key vouches for": every node beneath a folder with
|
||||
an unverified key is unverified, however it is signed, and so is a
|
||||
folder rclone makes there. Moving, copying, directory-moving or
|
||||
renaming a node whose key is unverified fails unless
|
||||
`ignore_signature_errors` is set.
|
||||
**Alternatives:** Inheriting only into anonymous nodes (leaves the
|
||||
user-signed children of a forged folder trusted).
|
||||
**Consequences:** A genuine folder whose key fails to verify, as one
|
||||
signed by a since-deleted address, makes its whole subtree unreadable
|
||||
and unwritable without `ignore_signature_errors`.
|
||||
|
||||
## 2026-11-11 - Proton Drive verifies the manifest before releasing a full read
|
||||
|
||||
**Status:** accepted
|
||||
**Context:** A full read checked the manifest signature at EOF, as the
|
||||
SDK does, so blocks swapped, truncated or from an older revision, each
|
||||
matching its listed hash and recorded size, reached the reader before
|
||||
the error: `rclone cat` and streaming mounts showed wrong bytes (spec
|
||||
9.6 asks for "an integrity error, not data").
|
||||
**Decision:** A full read lists every block page first, verifies the
|
||||
manifest over the listed hashes, then fetches; each block is checked
|
||||
against its listed hash, and a block listed again with another hash
|
||||
is an integrity error. Pages are fetched one after another. Ranged
|
||||
reads keep checking blocks against their listed hashes only.
|
||||
**Alternatives:** Keeping the SDK's EOF check; verifying ranged reads
|
||||
too by listing the whole file on each ranged open (costly for mounts
|
||||
and multi-thread copies without a hash cache).
|
||||
**Consequences:** No extra requests, but a file of more than 20 blocks
|
||||
waits for its whole list before the first byte. Block metadata, not
|
||||
data, is held meanwhile. A whole earlier revision served under the
|
||||
current ID stays undetectable (protocol limit).
|
||||
|
||||
## 2026-11-11 - Proton Drive distrusts what a forged folder key vouches for
|
||||
|
||||
**Status:** accepted
|
||||
**Context:** An anonymous node's passphrase, name and content signatures
|
||||
are checked against its parent folder's key. A server can make a folder
|
||||
key of its own, seal it to the real parent and sign everything beneath
|
||||
it, so a refusal based only on each node's own signature check is
|
||||
bypassed (Task 16 review).
|
||||
**Decision:** A node key is unverified if its own passphrase signature
|
||||
fails, or if it was checked against an unverified parent key; this
|
||||
passes down the tree. Reading a file with an unverified key, and
|
||||
creating, uploading, renaming, moving or copying into a folder with
|
||||
one, fail unless `ignore_signature_errors` is set. Listing such a folder
|
||||
still works, with one ERROR for the folder itself.
|
||||
**Alternatives:** Leaving the folder's children to their own checks
|
||||
(bypassable); refusing to list the folder (a sync would delete its
|
||||
destination copy).
|
||||
**Consequences:** Builds on the 2026-11-11 entry "Proton Drive refuses a
|
||||
file whose node key signature fails". Files in a public-link upload
|
||||
folder whose key fails to verify need `ignore_signature_errors`.
|
||||
|
||||
## 2026-11-11 - Proton Drive refuses a file whose node key signature fails
|
||||
|
||||
**Status:** accepted
|
||||
**Context:** The node passphrase signature was only logged when it
|
||||
failed, as the SDK does, so a server could seal a key of its own to
|
||||
the parent folder's public key and serve any content under it (spec 7
|
||||
puts the server in the threat model; 9.6 asks for an error).
|
||||
**Decision:** Opening such a file fails with an integrity error naming
|
||||
`ignore_signature_errors`, like a content key or manifest signature
|
||||
failure; with the option set it is read, the failure logged once. The
|
||||
node is still listed. Sign-in errors that need a reconnect name the
|
||||
remote through a hint the seam gives the session, so read errors, which
|
||||
don't pass through `translateError`, name it too.
|
||||
**Alternatives:** Skipping the node from listings (a sync would then
|
||||
delete its copy at the destination); leaving it to the SDK's behaviour.
|
||||
**Consequences:** Legacy files whose keys were signed by a since-removed
|
||||
address need `ignore_signature_errors`, which the error says.
|
||||
|
||||
## 2026-11-11 - Proton Drive backend runs on its own client
|
||||
|
||||
**Status:** accepted
|
||||
**Context:** The backend sat on `rclone/Proton-API-Bridge` and
|
||||
`rclone/go-proton-api`, which share credentials through package globals,
|
||||
write files the official clients can't always read, and can't be tested
|
||||
without a live account (spec 001).
|
||||
**Decision:** `protondrive.go` was rewritten on the native client built in
|
||||
plan Tasks 2-12, keeping every option name and meaning and reading v1.76
|
||||
saved sessions without signing in again. `Config` offers browser sign-in
|
||||
(default unless a username and password are already set, so existing
|
||||
scripted setups keep using them) and password sign-in. An upload whose
|
||||
commit outcome can't be known is a no-low-level-retry error: an immediate
|
||||
retry could add a revision that already landed, while a later sync pass
|
||||
reads the destination first.
|
||||
**Alternatives:** Patching the bridge (globals and crypto stay outside the
|
||||
tree); keeping browser sign-in the default even with a password set
|
||||
(non-interactive `config create` would wait on a browser).
|
||||
**Consequences:** The full contract suite runs hermetically against
|
||||
`protontest`. Several `Fs` of one account in one process each cache
|
||||
separately until events land (spec 6.2).
|
||||
|
||||
## 2026-11-11 - Fork agent tooling is appended, not merged, into upstream files
|
||||
|
||||
**Status:** accepted
|
||||
**Context:** Upstream rclone already ships a maintainer-written `AGENTS.md`
|
||||
and a `CLAUDE.md` that imports it. This fork tracks upstream and intends to
|
||||
send work back, so every edit to an upstream file is a future merge conflict
|
||||
and a risk of leaking fork-only material into an upstream PR.
|
||||
**Decision:** Upstream's `AGENTS.md` text is kept verbatim. Fork rules are one
|
||||
"Fork Workflow" section appended at the end. `CLAUDE.md` and `README.md` are
|
||||
untouched. Fork-only docs live at the top of `docs/` (Hugo only builds
|
||||
`docs/content/`, so they do not reach the website).
|
||||
**Alternatives:** Restructuring `AGENTS.md` with the fork rules first (better
|
||||
agent attention, permanent conflicts with upstream); a separate fork rules
|
||||
file (two sources of truth).
|
||||
**Consequences:** `AGENTS.md` sits at 148 of a 150 line budget - a new fork
|
||||
rule must replace an old one. Upstream PR branches must be cut so they exclude
|
||||
the paths listed in the Fork Workflow section.
|
||||
|
||||
## 2026-11-11 - Adopt AGENTS.md as single source of truth
|
||||
|
||||
**Status:** accepted
|
||||
**Context:** Agents had no fork-specific workflow or permission config between
|
||||
sessions.
|
||||
**Decision:** `AGENTS.md` is canonical; `CLAUDE.md`/`GEMINI.md` are pointers.
|
||||
Specs are permanent in `docs/specs/`. Plans live in `docs/plans/` only while
|
||||
work is in progress and are removed before merge. Both override the
|
||||
superpowers default paths.
|
||||
**Alternatives:** Per-tool rule files (drift); no rules file (agents
|
||||
re-explore the repo every session).
|
||||
**Consequences:** Rule changes happen in one file; pointer files stay thin.
|
||||
The spec, not the plan, is the record.
|
||||
@@ -0,0 +1,65 @@
|
||||
# Agent permissions
|
||||
|
||||
Fork-only. Source of truth for what AI coding agents may run here. Tool files
|
||||
are renderings of this page: `.claude/settings.json`, `.codex/config.toml`,
|
||||
`.codex/rules/project.rules`, `.gemini/settings.json`. Change this page and
|
||||
every tool file in the same commit. Review changes to these files and to
|
||||
`AGENTS.md` like CI config: one line redirects agent behaviour repo-wide.
|
||||
|
||||
## Verified commands
|
||||
|
||||
Run on the host on 2026-11-12 with Go 1.27.4 (`~/.local/opt/go`) and
|
||||
golangci-lint 2.15.0 (`~/.local/bin`), from the repository root, with
|
||||
`RCLONE_CONFIG=/notfound` set so no test can reach a real remote. The rows
|
||||
marked (container) are from 2026-11-11, before Go was on the host, run in
|
||||
the `golang:1.26` container and not repeated.
|
||||
|
||||
Only `worked` rows may appear in a tier, a tool file, or `AGENTS.md`.
|
||||
|
||||
| Command | Result |
|
||||
|---|---|
|
||||
| `go mod download` | worked |
|
||||
| `go build` | worked - binary reports `rclone v1.77.0-DEV` |
|
||||
| `go test ./backend/memory/` | worked |
|
||||
| `go test -run 'TestIntegration/FsMkdir/FsPutFiles/FsCopy' ./backend/memory/` | worked |
|
||||
| `go test ./backend/protondrive/...` | worked - the contract suite runs against the fake server; the live `TestProtonDrive:` suite skips without a config |
|
||||
| `go vet ./backend/protondrive/...` | worked |
|
||||
| `gofmt -l backend/protondrive` | worked - no output |
|
||||
| `golangci-lint run ./backend/protondrive/...` | worked - 0 issues |
|
||||
| `golangci-lint run ./...` | worked - 0 issues |
|
||||
| `go run ./fstest/test_all -backends protondrive -remotes TestProtonDriveFake: -maxtries 1 -n 4` | worked - starts the fake server through `fstest/testserver/init.d/TestProtonDriveFake` |
|
||||
| `RCLONE_CONFIG=/notfound go test ./...` | failed - only `cmd/serve/s3`, undiagnosed (2026-11-11, `docs/TODO.md` "Known issues"); in the container 7 packages failed: `backend/smb`, `cmd/gitannex`, `cmd/mount`, `cmd/mount2`, `cmd/serve/docker`, `cmd/serve/nfs`, `fs/logger` (container) |
|
||||
| `make`, `make quicktest`, `make racequicktest` | not run - `make` is in Ask |
|
||||
|
||||
## Tiers
|
||||
|
||||
| Tier | Commands | Why |
|
||||
|---|---|---|
|
||||
| Allow | `git status`, `git diff`, `git log`, `git show`, `git branch --list`, `git rev-parse`, `git add`, `git commit`, `go mod download`, `go build`, `go test`, `go vet`, `gofmt -l`, `golangci-lint run` | Inspection and the verified dev loop; no prompt needed. |
|
||||
| Ask | `git push`, `git merge`, `git rebase`, `git switch`, `git restore`, `git commit --amend`, `rm` (any), `go get`, `go mod tidy`, `go install`, `make` (any target), `docker` (any) | Leaves the machine, rewrites work, changes dependencies, or is unverified. |
|
||||
| Deny | `git push --force` / `-f` / `--force-with-lease`, `git reset --hard`, `git clean` (any), `git add -f` / `--force`, `git commit --no-verify` / `-n`, `rm -r` / `-R` / `-rf` / `-fr` / `-Rf` / `-fR` / `--recursive`, `sudo`; reading `.env`, `.env.*`, `secrets/**` | Destroys work, skips checks, or exposes secrets. |
|
||||
|
||||
## Known gaps
|
||||
|
||||
- The Claude Code deny rule `git push --force-with-lease:*` does not match
|
||||
`git push --force-with-lease=<ref>:<sha> ...` (no word boundary before
|
||||
`=`); a force-push the owner approved on 2026-11-12 went through it.
|
||||
- Allowlists save prompts; they are not a security boundary. Use the tool's
|
||||
sandbox for isolation.
|
||||
- Prefix rules are bypassable: `git -C dir push`, flags placed later
|
||||
(`git push origin main --force`), `+refspec` pushes. Claude Code's `:*`
|
||||
matches at a word boundary, so each flag cluster must be listed - which is
|
||||
why `git clean` is denied whole and `rm` always asks.
|
||||
- Allowed test and build commands run code from the repo; an agent that can
|
||||
edit Go files or tests can run anything through `go test`.
|
||||
- Denying reads of `.env` does not stop a program that opens it itself, and
|
||||
git can print staged or committed secrets. Rclone test credentials live in
|
||||
the user's rclone config, outside this repository - keep them there.
|
||||
- Codex: loaded only once the project is trusted; `approval_policy` here can
|
||||
loosen a stricter user setting. No `allow` rules are written because in
|
||||
Codex `allow` means "run outside the sandbox". Codex cannot deny file reads.
|
||||
`--force-with-lease=<ref>`, `+refspec` and late flags only match Ask.
|
||||
- Gemini CLI: ignored until the folder is trusted; no Ask tier (anything not
|
||||
allowed already asks); cannot deny file reads.
|
||||
- Cursor and Copilot files are not written: nothing in the repository shows
|
||||
the fork uses them (`.vscode` is gitignored).
|
||||
+394
@@ -0,0 +1,394 @@
|
||||
# TODO
|
||||
|
||||
Fork-only durable backlog for things noticed and deliberately not fixed.
|
||||
Small, specific, independently actionable. Remove only when actually done.
|
||||
(In Go source, upstream's `FIXME`/`TODO` comment convention still applies.)
|
||||
|
||||
## Now
|
||||
|
||||
- [ ] protondrive: the backend rewrite (plan 001) has Tasks 1-21 done
|
||||
as far as the test accounts allow (live results 2026-11-12 in the
|
||||
handoff, section 6). Start from
|
||||
`docs/plans/001-protondrive-tier1-HANDOFF.md`. Remaining: the
|
||||
whole-branch review and its fix wave.
|
||||
|
||||
- [ ] protondrive: the free test account's derived key passphrase (the
|
||||
`client_salted_key_pass` of a browser sign-in on 2026-11-12) was shown
|
||||
in an agent's tool output when `rclone config create` printed the new
|
||||
remote; that session was logged out at once, but the passphrase only
|
||||
changes with the account password. Owner: change the free account's
|
||||
password if that matters, then update `~/proton_rclone_test_acc_password`
|
||||
and the remote's saved password.
|
||||
|
||||
## Next
|
||||
|
||||
- [ ] protondrive M5: volume events polled no more often than every 30 s,
|
||||
invalidating the node cache and dircache and feeding `ChangeNotify`, so
|
||||
`mount` and `serve` see other clients' changes (spec 6.2). Needs its own
|
||||
plan; also decide the cross-`Fs` cache question below.
|
||||
- [ ] protondrive M6: opt-in persistent metadata cache of encrypted node
|
||||
records and the event cursor through `lib/kv` (`metadata_cache`), rclone
|
||||
Metadata (`mtime`, `btime`, `content-type`), and the SDK's single-request
|
||||
upload for files under 128 KiB (`files/small`). Needs its own plan; the
|
||||
docs page and `protondrive.yaml` features change with it.
|
||||
- [ ] protondrive M7: the upstream path (spec 13), deferred by the owner;
|
||||
also the new crypto model once Proton publishes it in the SDK.
|
||||
- [ ] protondrive live checks not yet run (plan Task 21), each of which
|
||||
the user docs avoid claiming: 2FA, by code and by `otp_secret_key`;
|
||||
CAPTCHA handling; Proton's desktop and mobile clients reading files
|
||||
rclone wrote (the web app and the official CLI 0.9.0 have been
|
||||
checked); whether the real server lists drafts or trashed items as
|
||||
children (affects `Rmdir`); whether any real address differs from
|
||||
another of the account only by non-ASCII case (counted as foreign);
|
||||
how long the server's read-after-write lag after a commit really is
|
||||
(the client retries for about 7 s).
|
||||
- [ ] protondrive: `test_all` against the real service (2026-11-12,
|
||||
free account) fails only the five cross-`Fs` cache tests listed
|
||||
below; `fstest/test_all/config.yaml`'s `TestProtonDrive:` entry has no
|
||||
`ignore` for them, so plan Task 21's "clean with no ignore entries" is
|
||||
not met until that is decided.
|
||||
- [ ] protondrive: `docs/content/protondrive.md`'s generated options
|
||||
section still shows the old options and help texts; regenerate it with
|
||||
`make backenddocs` when the work goes upstream (generated doc changes
|
||||
are not committed in the fork, per `AGENTS.md`).
|
||||
|
||||
- [ ] protondrive: API responses have no body-size limit, and a block
|
||||
list's `Token` and `BareURL` and the first page's `Thumbnails` are
|
||||
unbounded, so the 2^19-block cap bounds how many blocks a full read
|
||||
holds but not the bytes (about 250-350 MiB at the cap with realistic
|
||||
values, unbounded against a hostile server). Consider a limit on API
|
||||
response bodies and on those fields.
|
||||
- [ ] protondrive: confirm on an account whose drive was made by
|
||||
Proton's own apps (both test accounts' drives were most likely created
|
||||
by rclone itself) that the
|
||||
root-of-trust rule passes (the My Files share created, and its root
|
||||
signed, by one of the account's addresses), and count how many folders
|
||||
fail key verification.
|
||||
- [ ] protondrive: a range from offset 0 of a file whose size isn't
|
||||
recorded (`bytes=0-N`, N at or past the end) is read as a ranged read,
|
||||
so its manifest isn't checked; only a known size identifies a range
|
||||
covering the whole file.
|
||||
- [ ] protondrive: `TestGoldenCapture` refuses files with thumbnails (the
|
||||
fake server serves none, and the manifest covers them) and keys other
|
||||
than RSA, EdDSA, Ed25519 and Ed448; images and photos from Proton's apps
|
||||
can't become golden fixtures until the fake serves thumbnails.
|
||||
- [ ] protondrive: below the root, a node signed under another user's
|
||||
email (a shared folder's members) is verified against public keys the
|
||||
server supplies (`account.publicKeys`), so a hostile server can forge
|
||||
it; the SDK does the same. Revisit when Proton offers key transparency
|
||||
or address-key pinning.
|
||||
- [ ] protondrive: a full read lists every block page before releasing
|
||||
the first byte, one request per 20 blocks in sequence, so a very large
|
||||
file waits (about 12 requests for 1 GiB, about 1280 for 100 GiB) before
|
||||
data flows. Fetching pages in parallel under the 15-request metadata
|
||||
limit would cut that.
|
||||
- [ ] protondrive: a ranged read (offset > 0 or a limit, as multi-thread
|
||||
copies and mounts make) checks each block against the hash the server
|
||||
lists for it, its decryption and its recorded size, but not the
|
||||
manifest, so a server can serve validly encrypted blocks of the same
|
||||
file swapped or from an older revision of the same size. The planned
|
||||
fix is a bounded per-revision cache of manifest-verified block hashes.
|
||||
- [ ] protondrive: a server can serve an earlier revision whole (blocks,
|
||||
manifest and extended attributes, all validly signed) under the current
|
||||
revision ID undetected, as nothing signed binds them to the revision;
|
||||
the SDK can't detect it either. Raise with Proton.
|
||||
- [ ] protondrive: `--dump headers` logs Proton's `pm-storage-token` and
|
||||
`x-pm-uid` (fshttp redacts only `Authorization`), and `--dump bodies`
|
||||
logs the access, refresh and upload tokens in JSON bodies (seen in
|
||||
`TestNoSecretsInLogs`). The docs page warns about it; a header
|
||||
redaction list in `fshttp` would be an upstream change.
|
||||
- [ ] protondrive: a name whose signature fails is listed and the file
|
||||
read with only an `ERROR` logged (spec 6.2, the SDK's behaviour), and
|
||||
names aren't bound to their node, so a server can swap the encrypted
|
||||
names of two files in one folder undetected. Raise with Proton.
|
||||
- [ ] protondrive: a hostile 10 MiB name is decrypted in full before
|
||||
`validateName` rejects it (about 230 MiB allocated listing it, the
|
||||
fake included). Consider refusing an encrypted name over a few KiB
|
||||
before decrypting it.
|
||||
|
||||
- [ ] lib/pacer: `TokenDispenser.Get` takes no context, so a cancelled
|
||||
protondrive block download still waits for a download token (held by
|
||||
other in-flight blocks, which finish or fail soon) before it notices.
|
||||
Consider a context-aware `Get`.
|
||||
|
||||
- [ ] protondrive: confirm that Proton's desktop and mobile clients read
|
||||
a file written by Task 5's `crypto.go` (the web app and the official CLI
|
||||
0.9.0 do, with signatures verified, as of 2026-11-12) (v4 node key
|
||||
advertising SEIPDv2, v6 PKESK content key, SEIPDv2 GCM blocks, block
|
||||
signatures encrypted with the content session key). This is what the SDK
|
||||
(js/v0.23.0) writes, but go-proton-api v1.0.4 `link_file_types.go`
|
||||
(`SetContentKeyPacketAndSignature`) and Proton-API-Bridge v1.0.5
|
||||
`crypto.go` (`generateLockedKey`) say the official clients do not reliably
|
||||
read crypto-refresh files and that the server rejects a v6 content key
|
||||
packet for a v4 node key; the bridge never sent one with a key advertising
|
||||
SEIPDv2, so its claim may not apply. Released rclone itself is not the
|
||||
problem: Task 6's `TestNewWriterOldReader` (removed with the old
|
||||
libraries in Task 19) showed go-proton-api v1.0.4 and the bridge's block
|
||||
reader read this format (v6 PKESK, SEIPDv2 blocks and block signatures)
|
||||
and the new `toISOString` modification time.
|
||||
- [ ] protondrive: files the old library (go-proton-api v1.0.4 and
|
||||
Proton-API-Bridge v1.0.5) wrote are covered only by the single-block
|
||||
frozen fixtures in `backend/protondrive/testdata/oldlib/`
|
||||
(`TestFrozenFixtures`, `TestFrozenFileFixture`); the multi-block and
|
||||
empty-file cases left with the differential tests in Task 19. A
|
||||
multi-block fixture would need the old library back to write it. Nothing
|
||||
checks any more that released rclone (v1.76 and older) can read what the
|
||||
new backend writes; that matters to users who downgrade.
|
||||
Commit `b667a1ab0` still has the differential tests and both libraries,
|
||||
so such fixtures can be generated from a worktree of it.
|
||||
|
||||
- [ ] protondrive: a ranged read of a file which doesn't record its
|
||||
block sizes (written by some other clients) starts at the furthest
|
||||
block whose size an earlier read in the same `Fs` learnt
|
||||
(download.go `blockSizeCache`, 16 revisions, 65536 blocks), else at
|
||||
the first block. The first chunks of a multi-thread copy, which start
|
||||
together, still each read from the first block, and a random-offset
|
||||
read beyond what was read before through a mount is slow. A ranged
|
||||
read with sizes unknown also fetches up to `downloadAhead` blocks past
|
||||
its end. Neither cache nor listing tells where a block starts without
|
||||
reading the ones before it, as the packet framing varies by writer.
|
||||
|
||||
- [ ] protondrive: `client.apiCall` retries POST and PUT mutations
|
||||
(create folder, move, rename, copy) after 5xx and network errors, as the
|
||||
SDK does. If a lost response hid a success, the retry fails with
|
||||
`errExists` (2500) or a stale `OriginalHash` (2000) although the change
|
||||
was made. Consider re-reading the node after such a retry failure.
|
||||
|
||||
- [ ] protondrive: `createRevision` (POST `.../files/{id}/revisions`) is
|
||||
retried by `apiCall` after a lost response like any POST; a retry that
|
||||
succeeds leaves the first draft revision orphaned on the server (the
|
||||
fake allows several draft revisions). Find out whether Proton refuses a
|
||||
second draft revision, and delete or reuse our own if so.
|
||||
- [ ] protondrive: confirm on a real account what `delete_multiple` does to
|
||||
an active (non-draft) link. The upload cleanup only sends it for drafts,
|
||||
and the fake refuses non-drafts, but the SDK's conflict handler sends it
|
||||
for any `ConflictDraftRevisionID`, which an active file with a draft
|
||||
revision also reports; Task 11 uploads a new revision in that case
|
||||
instead.
|
||||
A `--retries` pass after an upload whose outcome was unknown can reach
|
||||
the own-draft `delete_multiple` in `resolveConflict` (`upload.go`); it
|
||||
is decided by Proton's conflict details, which could in theory lag.
|
||||
- [ ] protondrive: confirm the HTTP status and code Proton returns for a
|
||||
bad verifier token, a missing block and a stale `CurrentRevisionID`
|
||||
(the fake uses 422 / 2000, which the spec maps to "upgrade rclone").
|
||||
- [ ] protondrive: Proton's server can take a moment to show a change
|
||||
(seen live: a node read just after its commit was missing, or still a
|
||||
draft with no active revision). The fake models this only through
|
||||
`Server.SetReadLag`, on the links route after a create, copy or
|
||||
commit; how long the real lag lasts and whether listings
|
||||
(`folders/{id}/children`) lag too is unmeasured. Only the reads back
|
||||
after a commit, a copy and, with `enable_caching` off, of a folder to
|
||||
write into (`targetFolder`) retry, for about 7 s. Not tolerant yet, and the next
|
||||
symptoms to expect: `activeNode` (Update, Move, Copy, DirMove of an
|
||||
object just written) fails "object not found" while the node is
|
||||
missing or still a draft; Move's read of the moved node may be stale.
|
||||
- [ ] protondrive: drafts left after an ambiguous commit failure (5xx,
|
||||
network error, timeout, cancel, or a refusal after the commit was
|
||||
resent) are never deleted by the client, by design (see DECISIONS,
|
||||
"a draft is deleted only on an explicit refusal"). The next upload of
|
||||
the same name by this remote replaces its own draft, but a name never
|
||||
uploaded again keeps an orphan draft (a new file's draft node, or a
|
||||
draft revision) taking quota. `CleanUp` only empties the trash. Find
|
||||
out whether Proton expires drafts by itself, and consider having
|
||||
`CleanUp` delete this remote's own drafts (by `DraftClientUID`).
|
||||
- [ ] protondrive: `errUploadUnconfirmed` is not retried at once, but a
|
||||
whole-sync `--retries` pass lists again; if the listing still lags it
|
||||
uploads the file again, adding a revision with identical content
|
||||
(wasted work and version history, no data loss).
|
||||
|
||||
- [ ] protondrive: several `Fs` of one account in one process each have
|
||||
their own node cache; only Move/DirMove sources are invalidated across
|
||||
them. A name missing from one `Fs`'s cached listing stays missing after
|
||||
another `Fs` creates it, so `--compare-dest`, `--copy-dest` (with
|
||||
`--immutable`) and `checksum --download` decide on stale listings when
|
||||
the other `Fs` writes during the run, and an `Fs` made on a path that
|
||||
doesn't exist yet never finds it once it is made elsewhere. Five
|
||||
`test_all` tests fail for this and are in the `ignore` list of
|
||||
`TestProtonDriveFake:` (`fstest/test_all/config.yaml`):
|
||||
`TestCheckSumDownload`, `TestCopyFileCompareDest`,
|
||||
`TestCopyFileCopyDestImmutable`, `TestSyncCompareDest`,
|
||||
`TestSyncCopyDestImmutable`; all pass with `enable_caching = false`.
|
||||
Events polled every 30 s (spec 6.2) will not close a window of
|
||||
seconds. Re-listing on every cache miss fixes them but breaks
|
||||
`TestBudgetUpload` and `TestBudgetMkdir` (a folder re-listed per new
|
||||
file). A cache shared by the `Fs` of one account would fix them, but
|
||||
is package state, which the spec rules out: owner decision.
|
||||
Related and wider: with no cache expiry and no events (M5), a long
|
||||
`rclone mount` never sees changes made by other clients (web app,
|
||||
another rclone) to folders it has already listed. A listing TTL would
|
||||
bound that but would not fix the five tests above.
|
||||
- [ ] protondrive: `cmd/gitannex` is in `ignoretests` for
|
||||
`TestProtonDriveFake:` because its remote name check
|
||||
(`cmd/gitannex/configparse.go`, `config.GetRemoteNames`) sees remotes
|
||||
set by environment variables lower cased, as for every other init.d
|
||||
remote; it says nothing about the backend. Run it against a remote in
|
||||
a config file to cover it.
|
||||
- [ ] `.gitea/workflows/ci.yml` could not be run locally. The Gitea has
|
||||
an Actions runner (slow, so the workflow is kept light: 20 s per fuzz
|
||||
target, a 45 minute job timeout, no `test_all`); the first push must
|
||||
confirm the runner picks the workflow up.
|
||||
- [ ] protondrive: browser sign-in in `rclone config` waits up to 10 minutes
|
||||
polling; a non-interactive `config create` without username and password
|
||||
waits the same.
|
||||
|
||||
- [ ] Propose a CI check for this fork: `AGENTS.md`, `CLAUDE.md`, `GEMINI.md`
|
||||
and the `docs/` agent files exist, and `AGENTS.md` is at most 150 lines.
|
||||
|
||||
### final review minors (2026-11-12)
|
||||
|
||||
- [ ] protondrive (final re-review): ranged reads place their start from
|
||||
block sizes in extended attributes even when those are unsigned, so a
|
||||
forging server can shift the bytes a ranged read returns (full reads
|
||||
are unaffected).
|
||||
- [ ] protondrive (final re-review): `TestMain` applies `RCLONE_CONFIG`
|
||||
only when it is set; with it unset, tests may still find the user's
|
||||
config.
|
||||
- [ ] protondrive (final re-review): the draft client ID stays random per
|
||||
run when the remote has a config section that can't be saved.
|
||||
|
||||
Minors from the whole-branch review (`final-review-*.md` in the plan
|
||||
workspace) not fixed in its fix wave. Paths are under `backend/protondrive/`.
|
||||
|
||||
- [ ] A node fetched while it is being changed can overwrite the newer
|
||||
cached node and stay cached: `put` after `fetchLinks` has no per-node
|
||||
generation (`nodes.go` `fetchNode`, `cache.go` `put`).
|
||||
- [ ] Unbounded per-client maps: `client.folderKeys` (kept whatever the
|
||||
cache), the node cache, `client.reported`, `account.foreign`; `close()`
|
||||
doesn't clear the private keys of cached file nodes.
|
||||
- [ ] `listChildren` loops until ctx ends if the server keeps answering
|
||||
`More` with the same `AnchorID` (`nodes.go`); stop when a page adds
|
||||
nothing new.
|
||||
- [ ] The name and hash key signature date exemptions
|
||||
(`nameSignedSince`, `hashKeySignedSince`) and the content key fallback
|
||||
read the server-supplied `CreateTime`, so a server can back-date a node
|
||||
to exempt it (commented in `nodes.go`, as in the SDK).
|
||||
- [ ] Earlier deferred crypto minors still present: the node passphrase
|
||||
isn't checked as UTF-8 (`crypto.go` `decryptNodeKeys`); `publicKeys`
|
||||
waiters get the fetching caller's ctx error (`account.go`);
|
||||
`setChildrenLocked` overwrites a cached node without `put`'s
|
||||
changed-parent rule (`cache.go`); an xattr `"Size": null` reads as 0
|
||||
and skips the block-size fallback (`crypto.go` `parseXAttrs`); Move
|
||||
detects an anonymous node by `SignatureEmail == ""` (`nodes.go`
|
||||
`encryptForNewParent`).
|
||||
- [ ] `publicKeyRing` rebuilds the own-address public ring on every
|
||||
lookup, 2-3 times per decrypted node (`account.go`); build it once in
|
||||
`loadAccount`.
|
||||
- [ ] Two processes sharing one config (a mount and a cron job) can
|
||||
delete each other's in-progress draft of the same new name, since the
|
||||
draft client ID is per remote (`upload.go` `resolveConflict`).
|
||||
- [ ] `pm-storage-token` is still sent if a storage request is
|
||||
redirected to another host (`download.go` `downloadBlock`, `upload.go`
|
||||
block upload); set `CheckRedirect` or refuse redirects.
|
||||
- [ ] `errUploadMismatch` is a `NoRetryError`, so a source that changes
|
||||
while read fails for the whole run, `--retries` passes included
|
||||
(`protondrive.go` `translateError`); consider `NoLowLevelRetryError`.
|
||||
- [ ] An own leftover draft revision on an active file is never deleted
|
||||
and keeps taking quota (`upload.go` conflict path).
|
||||
- [ ] A `createDraft` resend can report the client's own new draft as
|
||||
one that "came back after it was deleted" (`upload.go` `createDraft`).
|
||||
- [ ] Two `Fs` of an environment-configured remote in one process can't
|
||||
see each other's token refresh, so the second fails with "session
|
||||
expired" (`protondrive.go` `configStore`); document it.
|
||||
- [ ] `rclone config` on a remote with a saved session asks "Already
|
||||
signed in - sign in again?" with default no, even when the session is
|
||||
dead and the user followed the reconnect hint (`protondrive.go`
|
||||
`Config`, ruling 17).
|
||||
- [ ] After a CAPTCHA in `rclone config`, the method question defaults
|
||||
back to password sign-in (`protondrive.go` `Config`).
|
||||
- [ ] A wrong key password on a saved session says "is the password
|
||||
right?" without the reconnect hint (`account.go` `loadAccount`,
|
||||
`protondrive.go` `connect`).
|
||||
- [ ] Token refresh hasn't been seen working against the real server,
|
||||
and its request lacks the `UID` and `RedirectURI` go-proton-api sends
|
||||
(`api/types.go` `AuthRefreshRequest`); add it to the live checks.
|
||||
- [ ] Session minors deferred since Tasks 2-4: an adopted stored token
|
||||
whose access token is stale fails one call with a 401; the refresh uses
|
||||
the first waiter's ctx; `serverTime` can step back; the half-signed-in
|
||||
cleanup `DELETE` uses a possibly cancelled ctx; an empty SRP salt isn't
|
||||
refused up front; `GET /core/v4/users` is fetched twice at sign-in
|
||||
(`session.go`, `account.go`).
|
||||
- [ ] Server-side Copy onto an existing name falls back to download and
|
||||
re-upload (`protondrive.go` `Copy`).
|
||||
- [ ] `Precision()` is 1 s while `ModTime` has milliseconds;
|
||||
cross-account Move/Copy/DirMove aren't refused; the on-the-fly remote
|
||||
hint names the bare backend; logging passes `nil`, not the `Fs`,
|
||||
throughout (sweep before upstream).
|
||||
- [ ] Goroutine-count leak checks remain (`download_internal_test.go`
|
||||
`requireNoLeaks`, `nodes_internal_test.go`); don't add more.
|
||||
- [ ] `TestListingRacingChange` can hang to the package timeout if it
|
||||
fails before closing `release` (`nodes_internal_test.go`).
|
||||
- [ ] `protonfake` `TestRun` leaks the `run` goroutine on a failed
|
||||
require (`protontest/cmd/protonfake/main_test.go`).
|
||||
- [ ] `TestClientBootstrapCreateVolumeRace` waits a fixed 2 s, and
|
||||
`TestAcquireTokenCancel` sleeps 20 ms to let the wait start
|
||||
(`nodes_internal_test.go`, `upload_internal_test.go`).
|
||||
- [ ] The init.d `stop` has no SIGKILL escalation, and shellcheck reports
|
||||
SC2034 (`fstest/testserver/init.d/TestProtonDriveFake`).
|
||||
- [ ] CI runs on both `push` and `pull_request` with no `concurrency`
|
||||
group, so twice per PR push (`.gitea/workflows/ci.yml`); check Gitea
|
||||
supports `concurrency` first.
|
||||
- [ ] The fake's read lag (`protontest` `SetReadLag`) never runs with the
|
||||
contract suite; consider a second `fstests.Run` or a `protonfake
|
||||
-read-lag` flag.
|
||||
- [ ] The docs page describes CAPTCHA handling, 2FA prompting and
|
||||
account-wide `about` that are proven only against the fake, while this
|
||||
file says the docs avoid claiming them; soften one or the other.
|
||||
- [ ] `rclone config create` and `config show` print the new remote's
|
||||
`client_*` tokens and salted key password, which survives a logout;
|
||||
add a docs line not to paste that output.
|
||||
- [ ] `docs/PERMISSIONS.md` and `.claude/`/`.gemini/` allow
|
||||
`golangci-lint run` but `.codex/rules/project.rules` doesn't; bring it
|
||||
in step (owner, as it is a permission change).
|
||||
|
||||
## Someday
|
||||
|
||||
## Recurring
|
||||
|
||||
- [ ] Maintain `AGENTS.md`:
|
||||
- **Add a rule only in response to a real failure.** Do not pre-empt
|
||||
hypothetical mistakes.
|
||||
- **Update in the same commit as the convention change.**
|
||||
- **Prune quarterly**, and re-check the Fork Workflow section against
|
||||
upstream's `AGENTS.md` after each upstream merge.
|
||||
|
||||
## Known issues
|
||||
|
||||
- [ ] On the development host (Go 1.27.4) `RCLONE_CONFIG=/notfound go test ./...`
|
||||
fails only `cmd/serve/s3`, unrelated to this work and undiagnosed (seen
|
||||
2026-11-11 before the Proton Drive rewrite began). That is the baseline
|
||||
for plan Task 19's "no new failures"; the container list below is not.
|
||||
- [ ] On the development host on 2026-11-12, `cmd/bisync` `TestRcBackupDir`
|
||||
(`backupDir1`, `backupDir2`) fails every run with "unable to find file
|
||||
.../backup/file0.txt", also at `06700aaf2` (before the Proton Drive
|
||||
rewrite), so it is not caused by that work; undiagnosed. `cmd/serve/s3`
|
||||
passed in the same full run.
|
||||
- [ ] `RCLONE_CONFIG=/notfound go test ./...` in the bare `golang:1.26`
|
||||
container fails 7 packages (list in `docs/PERMISSIONS.md`). `fs/logger`
|
||||
needs `rsync`; the mount, serve and smb failures are undiagnosed and are
|
||||
probably missing container tooling (FUSE, git-annex), not code. Confirm on a
|
||||
host with Go installed before treating a full-suite failure as a regression.
|
||||
- [ ] protondrive: `TestRegressionManyNewFs` can't see from the fake
|
||||
server that key unlocks run one at a time; only
|
||||
`TestLoadAccountSerialised` (internal, `unlockHook`) checks it.
|
||||
- [ ] protondrive: an address key's `Token` (account.go
|
||||
`unlockAddressKey`) is decrypted without a decompression limit, unlike
|
||||
node metadata (`decryptMessage`); a hostile server could send a
|
||||
compression bomb there too. Seen in Task 17.
|
||||
- [ ] protondrive: API error bodies are read whole and their `Error`
|
||||
message is logged as the server sent it, of any length and with any
|
||||
control characters (session.go `errorHandler`). Bound and sanitise it
|
||||
if a hostile server is in scope for logs. Seen in Task 17.
|
||||
- [ ] protondrive: a new revision (`createRevision`), a move, rename or
|
||||
trash still forget the parent folder's cached listing, so `Put` over
|
||||
existing files in one folder lists it again each time (`sync` is not
|
||||
affected: it calls `Update`). New files and folders are added to the
|
||||
listing instead since Task 17.
|
||||
- [ ] protondrive: spec 9.5 asks for a fuzz target for event decoding;
|
||||
add it with events (fuzz_internal_test.go).
|
||||
- [ ] protondrive: the account's own user keys (account.go `loadAccount`)
|
||||
are unlocked without the Argon2 limits node and share keys get
|
||||
(crypto.go `checkKeyS2K`): the server stores them too, so a hostile
|
||||
server could make sign-in run a KDF of up to 2 TiB. Left out because a
|
||||
genuine user key's S2K parameters aren't known to be within the
|
||||
limits; check what Proton's apps write before applying them. Address
|
||||
keys are safe: their token's signature is checked first.
|
||||
@@ -113,6 +113,7 @@ WebDAV or S3, that work out of the box.)
|
||||
|
||||
{{< provider_list >}}
|
||||
{{< provider name="1Fichier" home="https://1fichier.com/" config="/fichier/" start="true">}}
|
||||
{{< provider name="Abrha Storage" home="https://parspack.com/cloud-storage" config="/s3/#abrhastorage" >}}
|
||||
{{< provider name="Akamai Netstorage" home="https://www.akamai.com/us/en/products/media-delivery/netstorage.jsp" config="/netstorage/" >}}
|
||||
{{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
|
||||
{{< provider name="Amazon S3" home="https://aws.amazon.com/s3/" config="/s3/" >}}
|
||||
|
||||
+294
-73
@@ -7,28 +7,41 @@ versionIntroduced: "v1.64.0"
|
||||
# Proton Drive
|
||||
|
||||
[Proton Drive](https://proton.me/drive) is an end-to-end encrypted Swiss vault
|
||||
for your files that protects your data.
|
||||
for your files that protects your data.
|
||||
|
||||
This is an rclone backend for Proton Drive which supports the file transfer
|
||||
features of Proton Drive using the same client-side encryption.
|
||||
This backend reads and writes Proton Drive with the same client-side
|
||||
encryption as Proton's own apps. It is a client of the Proton Drive API
|
||||
written for rclone, following the behaviour of
|
||||
[Proton's Drive SDK](https://github.com/ProtonDriveApps/sdk).
|
||||
|
||||
Due to the fact that Proton Drive doesn't publish its API documentation, this
|
||||
backend is implemented with best efforts by reading the open-sourced client
|
||||
source code and observing the Proton Drive traffic in the browser.
|
||||
|
||||
**NB** This backend is currently in Beta. It is believed to be correct
|
||||
and all the integration tests pass. However the Proton Drive protocol
|
||||
has evolved over time there may be accounts it is not compatible
|
||||
with. Please [post on the rclone forum](https://forum.rclone.org/) if
|
||||
you find an incompatibility.
|
||||
rclone is a third-party application not officially supported by Proton.
|
||||
|
||||
Paths are specified as `remote:path`
|
||||
|
||||
Paths may be as deep as required, e.g. `remote:directory/subdirectory`.
|
||||
|
||||
## Configurations
|
||||
## Configuration
|
||||
|
||||
Here is an example of how to make a remote called `remote`. First run:
|
||||
There are two ways to sign in:
|
||||
|
||||
- **With a web browser** (the default when no username and password are
|
||||
set). rclone shows a link to Proton's sign-in page, which you can open
|
||||
on any device, including a different computer from the one running
|
||||
rclone. Your password, two-factor authentication and any CAPTCHA are
|
||||
handled by Proton's page; rclone never sees or stores your password.
|
||||
- **With a username and password**, for existing configurations and
|
||||
automated setups. rclone signs in with the `username`, `password`,
|
||||
and, if needed, `mailbox_password`, `2fa` or `otp_secret_key` options.
|
||||
If Proton asks for a CAPTCHA, password sign-in fails and you need to
|
||||
run `rclone config reconnect remote:` and choose browser sign-in.
|
||||
|
||||
Either way rclone saves the session in the config file and uses it from
|
||||
then on, refreshing it as needed. The saved session gives full access to
|
||||
your account, so consider [encrypting your
|
||||
configuration](/docs/#configuration-encryption).
|
||||
|
||||
Here is an example of how to make a remote called `remote` with browser
|
||||
sign-in. First run:
|
||||
|
||||
```console
|
||||
rclone config
|
||||
@@ -50,37 +63,84 @@ XX / Proton Drive
|
||||
\ "protondrive"
|
||||
[snip]
|
||||
Storage> protondrive
|
||||
User name
|
||||
user> you@protonmail.com
|
||||
Password.
|
||||
y) Yes type in my own password
|
||||
g) Generate random password
|
||||
n) No leave this optional password blank
|
||||
y/g/n> y
|
||||
Enter the password:
|
||||
password:
|
||||
Confirm the password:
|
||||
password:
|
||||
Option 2fa.
|
||||
2FA code (if the account requires one)
|
||||
Option username.
|
||||
The username of your Proton account.
|
||||
Only needed to sign in with a password. Signing in with a web browser,
|
||||
which "rclone config" offers, needs neither username nor password.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
2fa> 123456
|
||||
Remote config
|
||||
username>
|
||||
Option password.
|
||||
The password of your Proton account.
|
||||
Only needed to sign in with a password.
|
||||
Choose an alternative below. Press Enter for the default (n).
|
||||
y) Yes, type in my own password
|
||||
g) Generate random password
|
||||
n) No, leave this optional password blank (default)
|
||||
y/g/n> n
|
||||
Option 2fa.
|
||||
[snip]
|
||||
2fa>
|
||||
Option otp_secret_key.
|
||||
[snip]
|
||||
y/g/n> n
|
||||
Edit advanced config?
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n> n
|
||||
Sign in with a web browser?
|
||||
|
||||
rclone is a third-party application not officially supported by Proton.
|
||||
|
||||
Choose yes to sign in on Proton's sign-in page, on any device; rclone
|
||||
never sees your password. Choose no to sign in with the username and
|
||||
password set above.
|
||||
y) Yes (default)
|
||||
n) No
|
||||
y/n> y
|
||||
Open this link in a web browser, on any device, and sign in to Proton:
|
||||
|
||||
https://account.proton.me/desktop/login?app=drive&pv=3#payload=XXXXXXXX
|
||||
|
||||
Waiting for the sign-in to complete...
|
||||
Configuration complete.
|
||||
Options:
|
||||
- type: protondrive
|
||||
- user: you@protonmail.com
|
||||
- pass: *** ENCRYPTED ***
|
||||
[snip]
|
||||
Keep this "remote" remote?
|
||||
y) Yes this is OK
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
**NOTE:** The Proton Drive encryption keys need to have been already generated
|
||||
after a regular login via the browser, otherwise attempting to use the
|
||||
credentials in `rclone` will fail.
|
||||
The sign-in link contains a secret which lets whoever completes the
|
||||
sign-in hand their session to this rclone, so don't share it. rclone
|
||||
waits up to 10 minutes for the sign-in to complete.
|
||||
|
||||
To sign in with a password instead, enter your username and password at
|
||||
the `username` and `password` prompts (and, for a two-password account,
|
||||
the `mailbox_password` advanced option), then answer `n` to "Sign in with
|
||||
a web browser?". If your account uses two-factor authentication, rclone
|
||||
asks for a code:
|
||||
|
||||
```text
|
||||
Two-factor authentication code
|
||||
Enter a value.
|
||||
config_2fa> 123456
|
||||
```
|
||||
|
||||
The code is only needed while signing in. If rclone has to sign in again
|
||||
later without you, for example because the saved session has expired, it
|
||||
uses the `otp_secret_key` option to generate a code; without it, or
|
||||
without a username and password, it stops and asks you to run
|
||||
`rclone config reconnect remote:`.
|
||||
|
||||
Running `rclone config reconnect remote:` (or editing the remote) on a
|
||||
remote which is already signed in asks whether to sign in again; if you
|
||||
do, the previous session is signed out.
|
||||
|
||||
A new account doesn't need to have used Proton Drive before: if the
|
||||
account has no Drive yet, rclone creates it.
|
||||
|
||||
Once configured you can then use `rclone` like this (replace `remote` with the
|
||||
name you gave your remote):
|
||||
@@ -97,7 +157,7 @@ List all the files in your Proton Drive
|
||||
rclone ls remote:
|
||||
```
|
||||
|
||||
To copy a local directory to an Proton Drive directory called backup
|
||||
To copy a local directory to a Proton Drive directory called backup
|
||||
|
||||
```console
|
||||
rclone copy /home/source remote:backup
|
||||
@@ -105,33 +165,198 @@ rclone copy /home/source remote:backup
|
||||
|
||||
### Modification times and hashes
|
||||
|
||||
Proton Drive Bridge does not support updating modification times yet.
|
||||
The modification time of a file is stored, encrypted, with each upload
|
||||
and read back with a precision of 1 second. It can't be changed without
|
||||
uploading the file again, so rclone can't set it on an existing file.
|
||||
The modification time of a folder is set when it is created.
|
||||
|
||||
The SHA1 hash algorithm is supported.
|
||||
The SHA1 hash algorithm is supported. The hash is the one the uploader
|
||||
recorded in the file's encrypted attributes; files uploaded by clients
|
||||
which don't record one have no hash.
|
||||
|
||||
### Integrity and signatures
|
||||
|
||||
Proton Drive is end-to-end encrypted, and rclone checks what it reads
|
||||
the way Proton's own apps do, so that a server which altered your data
|
||||
would be noticed:
|
||||
|
||||
- Every encrypted block of a file is checked against the SHA-256 hash
|
||||
the server lists for it before it is decrypted. A block which doesn't
|
||||
match is downloaded again, up to three attempts in all, then the read
|
||||
fails.
|
||||
- A read of a whole file first fetches the list of all its blocks and
|
||||
checks the uploader's signature over it (the manifest) before
|
||||
returning any data, so a file can't be truncated, extended or have its
|
||||
blocks reordered. For a very large file this adds about one request
|
||||
per 80 MiB before the first byte arrives.
|
||||
- The keys of files and folders and the key of each file's content must
|
||||
be signed by their owner.
|
||||
|
||||
On upload rclone signs everything it writes, checks that each encrypted
|
||||
block decrypts again before sending it, and, before committing the
|
||||
upload, checks the number of bytes read against the size of the source
|
||||
and the SHA1 of what was read against the source's SHA1 if the source
|
||||
has one.
|
||||
|
||||
**Ranged reads.** A read of only part of a file, as `rclone mount`,
|
||||
`rclone serve` and multi-thread downloads (`--multi-thread-streams`)
|
||||
make, checks each block against its listed hash but can't check the
|
||||
manifest, so a malicious server could serve validly encrypted blocks of
|
||||
the same file in the wrong place, or from an older version of the same
|
||||
size. Proton's SDK has the same limit for ranged reads. A range which
|
||||
covers the whole file is checked as a full read. A file uploaded by a
|
||||
client which doesn't record its block sizes can only be entered in the
|
||||
middle as far as earlier reads in the same rclone process have reached:
|
||||
beyond that a ranged read downloads, checks and discards every block
|
||||
before the range, which makes it slower. rclone remembers the sizes of
|
||||
up to 65536 such blocks, so each chunk of a multi-thread download starts
|
||||
near where an earlier one ended. Its manifest still isn't checked.
|
||||
|
||||
**Trust is rooted in your own addresses.** The share holding your files
|
||||
must have been created, and its root folder signed, by one of your
|
||||
account's addresses, otherwise nothing in the drive is treated as
|
||||
verified. A folder whose key isn't verified makes everything beneath it
|
||||
unverified, even items you signed yourself, because whoever made that
|
||||
key can read and replace what is in it. That includes a genuine folder
|
||||
signed by an address since removed from your account.
|
||||
|
||||
By default rclone refuses to:
|
||||
|
||||
- read a file whose key, content key or manifest signature doesn't
|
||||
verify, or which is beneath a folder whose key isn't verified;
|
||||
- update a file whose key or content key signature doesn't verify, as
|
||||
its new content would be encrypted with a key someone else may hold.
|
||||
A content key without a signature counts as one that doesn't verify;
|
||||
- create, upload, rename, move or copy anything into a folder whose key
|
||||
isn't verified, because what is written there would be encrypted to a
|
||||
key someone else may hold, or whose name hash key's signature doesn't
|
||||
verify, because whoever made that key could test guesses at the names
|
||||
written there;
|
||||
- rename, move or copy an item whose key isn't verified, because that
|
||||
re-signs it with your keys.
|
||||
|
||||
The errors say so and name the `ignore_signature_errors` advanced option.
|
||||
Data uploaded by some old or third-party clients, or shared with you by
|
||||
others, may need it. When it is set, rclone logs an `ERROR` for each
|
||||
item and operation it lets through and goes ahead; an item renamed,
|
||||
moved or copied this way is signed with your keys and trusted from then
|
||||
on. The option never turns off the block hash checks.
|
||||
|
||||
A file or folder name whose signature doesn't verify is listed anyway,
|
||||
with an `ERROR` logged, as Proton's apps do.
|
||||
|
||||
The modification time, size and SHA-1 of an item whose extended
|
||||
attributes' signature doesn't verify aren't used unless
|
||||
`ignore_signature_errors` is set: rclone logs an `ERROR` and shows the
|
||||
server's modification time and no SHA-1. A file's size then comes from
|
||||
its recorded block sizes, which reads check, or is the encrypted size if
|
||||
it has none. The file stays readable.
|
||||
|
||||
Two limits of the Proton Drive protocol, which Proton's SDK shares,
|
||||
can't be detected by rclone:
|
||||
|
||||
- A server which serves a whole earlier version of a file (its blocks,
|
||||
manifest and attributes, all validly signed) in place of the current
|
||||
one.
|
||||
- Below the root of your drive, an item signed by another Proton user,
|
||||
such as a member of a shared folder, is checked against public keys
|
||||
which the server itself supplies, so a malicious server could forge
|
||||
those.
|
||||
|
||||
### Restricted filename characters
|
||||
|
||||
Invalid UTF-8 bytes will be [replaced](/overview/#invalid-utf8), also left and
|
||||
right spaces will be removed ([code reference](https://github.com/ProtonMail/WebClients/blob/b4eba99d241af4fdae06ff7138bd651a40ef5d3c/applications/drive/src/app/store/_links/validation.ts#L51))
|
||||
Names may be at most 255 characters long, counted in UTF-16 code units
|
||||
as Proton's apps count them (so some emoji count as two), and can't be
|
||||
empty, `.` or `..`, or contain `/` or NUL. The default encoding replaces
|
||||
these characters:
|
||||
|
||||
| Character | Value | Replacement |
|
||||
| --------- |:-----:|:-----------:|
|
||||
| NUL | 0x00 | ␀ |
|
||||
| / | 0x2F | / |
|
||||
|
||||
File names can also not start or end with the following characters.
|
||||
These only get replaced if they are the first or last character in the
|
||||
name:
|
||||
|
||||
| Character | Value | Replacement |
|
||||
| --------- |:-----:|:-----------:|
|
||||
| SP | 0x20 | ␠ |
|
||||
|
||||
Names which are exactly `.` or `..` are replaced with `.` and `..`.
|
||||
|
||||
Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8). If
|
||||
the encoding is changed so that they are not, names which aren't valid
|
||||
UTF-8 are refused.
|
||||
|
||||
### Duplicated files
|
||||
|
||||
Proton Drive can not have two files with exactly the same name and path. If the
|
||||
conflict occurs, depending on the advanced config, the file might or might not
|
||||
be overwritten.
|
||||
Proton Drive can't have two files with exactly the same name in the same
|
||||
folder. Uploading a file whose name is already taken by a file adds a new
|
||||
revision to that file, keeping its history, rather than making a second
|
||||
file.
|
||||
|
||||
### [Mailbox password](https://proton.me/support/the-difference-between-the-mailbox-password-and-login-password)
|
||||
An upload which is cancelled or fails can leave a draft behind. A draft
|
||||
left by the same remote is replaced on the next upload of that file. A
|
||||
draft left by another client, which may still be uploading, makes the
|
||||
upload fail unless the `replace_existing_draft` advanced option is set.
|
||||
A remote set up with environment variables or a connection string,
|
||||
which can't save settings, is told apart by its name and account, so two
|
||||
machines using such a remote of the same name on the same account count
|
||||
as the same remote.
|
||||
|
||||
Please set your mailbox password in the advanced config section.
|
||||
If the final step of an upload fails in a way which leaves it unknown
|
||||
whether the file was saved, rclone reports "upload outcome unknown: it
|
||||
may have succeeded" and doesn't retry that upload at once; a later run of
|
||||
`rclone sync` or `rclone copy` checks the destination first. The same
|
||||
goes for "upload committed, but the server doesn't show it yet", which
|
||||
means Proton accepted the upload but didn't list it within a few
|
||||
seconds.
|
||||
|
||||
### Deleting files
|
||||
|
||||
Deleted files and folders (`rclone delete`, `rclone purge`,
|
||||
`rclone rmdir`) are moved to the Proton Drive trash, as Proton has no
|
||||
direct permanent delete. They count against your storage until the trash
|
||||
is emptied. `rclone cleanup remote:` empties the whole trash
|
||||
permanently, including items deleted with Proton's apps.
|
||||
|
||||
`rclone rmdir` asks the server whether the folder is empty, so it refuses
|
||||
to remove a folder which holds items rclone doesn't list, such as Proton
|
||||
Docs or items it can't decrypt.
|
||||
|
||||
### Items left out of listings
|
||||
|
||||
Proton Docs and Sheets have no file content for rclone to read, so they
|
||||
are left out of listings, with one notice logged. Items whose name or
|
||||
key can't be decrypted are left out too, each with an `ERROR` logged.
|
||||
|
||||
### Mailbox password
|
||||
|
||||
If your account uses a separate [mailbox
|
||||
password](https://proton.me/support/the-difference-between-the-mailbox-password-and-login-password),
|
||||
set it in the `mailbox_password` advanced option to sign in with a
|
||||
password. Browser sign-in doesn't need it.
|
||||
|
||||
### Caching
|
||||
|
||||
The cache is currently built for the case when the rclone is the only instance
|
||||
performing operations to the mount point. The event system, which is the proton
|
||||
API system that provides visibility of what has changed on the drive, is yet
|
||||
to be implemented, so updates from other clients won’t be reflected in the
|
||||
cache. Thus, if there are concurrent clients accessing the same mount point,
|
||||
then we might have a problem with caching the stale data.
|
||||
rclone keeps the decrypted metadata of the files and folders it has read
|
||||
in memory, to save requests and decryption. Changes made by other clients
|
||||
(Proton's apps, or another rclone) to folders it has already listed are
|
||||
not seen until rclone restarts, so a long-running `rclone mount` or
|
||||
`rclone serve` can show out of date listings. If other clients change
|
||||
the same files while rclone runs, as can happen with a mount, set the
|
||||
`enable_caching` advanced option to `false`.
|
||||
|
||||
### Logs and `--dump`
|
||||
|
||||
rclone's own log messages for this backend never contain passwords, keys
|
||||
or tokens, at any `-v` level. The output of the `--dump` flags does:
|
||||
`--dump headers` shows the `x-pm-uid` and `pm-storage-token` headers,
|
||||
and `--dump bodies` shows the access, refresh and upload tokens Proton
|
||||
sends in request and response bodies. Treat `--dump` output from this
|
||||
backend as secret, and sign the remote in again (with
|
||||
`rclone config reconnect remote:`) if you have shared it.
|
||||
|
||||
<!-- autogenerated options start - DO NOT EDIT - instead edit fs.RegInfo in backend/protondrive/protondrive.go and run make backenddocs to verify --> <!-- markdownlint-disable-line line-length -->
|
||||
### Standard options
|
||||
@@ -330,25 +555,21 @@ Properties:
|
||||
|
||||
## Limitations
|
||||
|
||||
This backend uses the
|
||||
[Proton-API-Bridge](https://github.com/henrybear327/Proton-API-Bridge), which
|
||||
is based on [go-proton-api](https://github.com/henrybear327/go-proton-api), a
|
||||
fork of the [official repo](https://github.com/ProtonMail/go-proton-api).
|
||||
|
||||
There is no official API documentation available from Proton Drive. But, thanks
|
||||
to Proton open sourcing [proton-go-api](https://github.com/ProtonMail/go-proton-api)
|
||||
and the web, iOS, and Android client codebases, we don't need to completely
|
||||
reverse engineer the APIs by observing the web client traffic!
|
||||
|
||||
[proton-go-api](https://github.com/ProtonMail/go-proton-api) provides the basic
|
||||
building blocks of API calls and error handling, such as 429 exponential
|
||||
back-off, but it is pretty much just a barebone interface to the Proton API.
|
||||
For example, the encryption and decryption of the Proton Drive file are not
|
||||
provided in this library.
|
||||
|
||||
The Proton-API-Bridge, attempts to bridge the gap, so rclone can be built on
|
||||
top of this quickly. This codebase handles the intricate tasks before and after
|
||||
calling Proton APIs, particularly the complex encryption scheme, allowing
|
||||
developers to implement features for other software on top of this codebase.
|
||||
There are likely quite a few errors in this library, as there isn't official
|
||||
documentation available.
|
||||
- Only the "My files" section of Proton Drive is available; computers
|
||||
backed up with Proton's desktop app, Photos and items shared with you
|
||||
are not.
|
||||
- Changes made by other clients are not seen while rclone runs (see
|
||||
[Caching](#caching)): the backend doesn't yet follow Proton's change
|
||||
events, so `rclone mount` doesn't pick them up.
|
||||
- rclone metadata (`--metadata`) is not supported, and the modification
|
||||
time of an existing file can't be changed without uploading it again.
|
||||
- A file may have at most 524288 blocks of 4 MiB (2 TiB); rclone refuses
|
||||
to read a file which lists more. Items more than 2048 folders deep
|
||||
can't be used.
|
||||
- `rclone about` reports the storage used by your whole Proton account,
|
||||
not only by Drive.
|
||||
- Files are uploaded in blocks of 4 MiB, each held in memory while it is
|
||||
uploaded: up to 5 blocks of one file at once (the `upload_concurrency`
|
||||
advanced option, default 4), and at most 10 blocks uploading and 10
|
||||
downloading at once per remote, so with a high `--transfers` some
|
||||
transfers wait their turn.
|
||||
@@ -13,6 +13,7 @@ The S3 backend can be used with a number of different providers:
|
||||
|
||||
{{< provider_list >}}
|
||||
{{< provider name="AWS S3" home="https://aws.amazon.com/s3/" config="/s3/#configuration" start="true" >}}
|
||||
{{< provider name="Abrha Storage" home="https://parspack.com/cloud-storage" config="/s3/#abrhastorage" >}}
|
||||
{{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
|
||||
{{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}}
|
||||
{{< provider name="China Mobile Ecloud Elastic Object Storage (EOS)" home="https://ecloud.10086.cn/home/product-introduction/eos/" config="/s3/#china-mobile-ecloud-eos" >}}
|
||||
@@ -5095,6 +5096,142 @@ endpoint = http://[IP of Snowball]:8080
|
||||
upload_cutoff = 0
|
||||
```
|
||||
|
||||
### Abrha Storage {#abrhastorage}
|
||||
|
||||
[Abrha Storage](https://parspack.com/cloud-storage) is S3-compatible object
|
||||
storage. Each space has its own endpoint, shown in the storage panel, of the
|
||||
form `https://c123456.parspack.net`. The bucket name in rclone paths is that
|
||||
same id (`c123456`).
|
||||
|
||||
Connection details are in the ParsPack S3 guide:
|
||||
|
||||
<https://parspack.com/blog/hosting/connecting-to-s3-bucket-with-cloud-storage>
|
||||
|
||||
Leave the region empty. The provider uses path-style access and automatically
|
||||
skips the bucket existence check, matching ParsPack's documented workaround
|
||||
for bucket check access errors. Create the bucket in the panel before use.
|
||||
|
||||
Here is an example of making a remote. First run:
|
||||
|
||||
```console
|
||||
rclone config
|
||||
```
|
||||
|
||||
This will guide you through an interactive setup process.
|
||||
|
||||
```text
|
||||
No remotes found, make a new one?
|
||||
n) New remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/s/q> n
|
||||
|
||||
Enter name for new remote.
|
||||
name> pspk
|
||||
|
||||
Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
[snip]
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, AbrhaStorage, ...
|
||||
\ (s3)
|
||||
[snip]
|
||||
Storage> s3
|
||||
|
||||
Option provider.
|
||||
Choose your S3 provider.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
[snip]
|
||||
XX / Abrha Storage (ParsPack)
|
||||
\ (AbrhaStorage)
|
||||
[snip]
|
||||
provider> AbrhaStorage
|
||||
|
||||
Option env_auth.
|
||||
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
|
||||
Only applies if access_key_id and secret_access_key is blank.
|
||||
Choose a number from below, or type in your own boolean value (true or false).
|
||||
Press Enter for the default (false).
|
||||
1 / Enter AWS credentials in the next step.
|
||||
\ (false)
|
||||
2 / Get AWS credentials from the environment (env vars or IAM).
|
||||
\ (true)
|
||||
env_auth> 1
|
||||
|
||||
Option access_key_id.
|
||||
AWS Access Key ID.
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
access_key_id> YOURACCESSKEY
|
||||
|
||||
Option secret_access_key.
|
||||
AWS Secret Access Key (password).
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
secret_access_key> YOURSECRETACCESSKEY
|
||||
|
||||
Option region.
|
||||
Region to connect to.
|
||||
Leave blank if you are using an S3 clone and you don't have a region.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
/ Use this if unsure.
|
||||
1 | Will use v4 signatures and an empty region.
|
||||
\ ()
|
||||
/ Use this only if v4 signatures don't work.
|
||||
2 | E.g. pre Jewel/v10 CEPH.
|
||||
\ (other-v2-signature)
|
||||
region>
|
||||
|
||||
Option endpoint.
|
||||
Endpoint for S3 API.
|
||||
Required when using an S3 clone.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
endpoint> https://c123456.parspack.net
|
||||
|
||||
Option bucket_object_lock_enabled.
|
||||
Enable Object Lock when creating new buckets.
|
||||
Enter a boolean value (true or false). Press Enter for the default (false).
|
||||
bucket_object_lock_enabled>
|
||||
|
||||
Edit advanced config?
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n> n
|
||||
|
||||
Configuration complete.
|
||||
Options:
|
||||
- type: s3
|
||||
- provider: AbrhaStorage
|
||||
- access_key_id: YOURACCESSKEY
|
||||
- secret_access_key: YOURSECRETACCESSKEY
|
||||
- endpoint: https://c123456.parspack.net
|
||||
Keep this "pspk" remote?
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
This will leave the config file looking like this.
|
||||
|
||||
```ini
|
||||
[pspk]
|
||||
type = s3
|
||||
provider = AbrhaStorage
|
||||
access_key_id = YOURACCESSKEY
|
||||
secret_access_key = YOURSECRETACCESSKEY
|
||||
endpoint = https://c123456.parspack.net
|
||||
```
|
||||
|
||||
`force_path_style` and `no_check_bucket` are applied automatically for
|
||||
this provider, so they do not need to be set. `no_check_bucket` can't be
|
||||
overridden with `no_check_bucket = false`, since buckets can only be
|
||||
created in the ParsPack panel.
|
||||
|
||||
Paths are `remote:bucket`, for example `pspk:c123456`.
|
||||
|
||||
### Alibaba OSS {#alibaba-oss}
|
||||
|
||||
Here is an example of making an [Alibaba Cloud (Aliyun) OSS](https://www.alibabacloud.com/product/oss/)
|
||||
|
||||
@@ -2,7 +2,7 @@ backend: protondrive
|
||||
name: Proton Drive
|
||||
tier: Tier 4
|
||||
maintainers: Community
|
||||
features_score: 4
|
||||
features_score: 5
|
||||
integration_tests: Passing
|
||||
data_integrity: Hash
|
||||
performance: High
|
||||
@@ -15,13 +15,16 @@ features:
|
||||
- About
|
||||
- CanHaveEmptyDirectories
|
||||
- CleanUp
|
||||
- Copy
|
||||
- DirCacheFlush
|
||||
- DirMove
|
||||
- Disconnect
|
||||
- ListP
|
||||
- Move
|
||||
- NoMultiThreading
|
||||
- Purge
|
||||
- PutStream
|
||||
- ReadMimeType
|
||||
- Shutdown
|
||||
hashes:
|
||||
- sha1
|
||||
precision: 1000000000
|
||||
@@ -1 +1 @@
|
||||
v1.76.0
|
||||
v1.77.0
|
||||
@@ -0,0 +1,460 @@
|
||||
# Handoff: Proton Drive backend rewrite (plan 001)
|
||||
|
||||
Written 2026-11-11 when development was paused at the owner's request.
|
||||
Everything a new agent needs is in this repository; nothing depends on the
|
||||
conversation that produced it. Fork-only document - never include it in a
|
||||
PR to rclone/rclone.
|
||||
|
||||
## 1. What this is
|
||||
|
||||
The owner's fork of rclone is rewriting `backend/protondrive` as a native Go
|
||||
client for Proton Drive's current API (volume-based `drive/v2` routes),
|
||||
replacing the abandoned `Proton-API-Bridge` / `go-proton-api` libraries.
|
||||
The goal is a working, maintainable, secure backend that rclone's
|
||||
maintainers could one day accept.
|
||||
|
||||
Read, in this order:
|
||||
|
||||
1. `AGENTS.md` - code, comment and commit rules; the "Fork Workflow" section.
|
||||
2. `docs/specs/001-protondrive-tier1.md` - the approved spec (binding).
|
||||
3. `docs/plans/001-protondrive-tier1.md` - the 21-task plan. Checkboxes
|
||||
show what is done.
|
||||
4. This file - state, rulings, and how to resume.
|
||||
5. `docs/plans/001-protondrive-tier1-workspace/progress.md` - the full
|
||||
execution ledger (every ruling, every deferred minor, every live probe).
|
||||
6. `docs/ARCHITECTURE.md`, `docs/DECISIONS.md`, `docs/TODO.md`.
|
||||
|
||||
## 2. Owner's standing instructions
|
||||
|
||||
- Nothing is posted upstream (no issue, PR or forum post) for now. The
|
||||
upstream path in spec section 13 is deferred.
|
||||
- The owner is not planning permanent maintenance of the backend.
|
||||
- The fake server (`backend/protondrive/protontest`) stays in the fork even
|
||||
though it is about a third of the code; it would not go upstream.
|
||||
- Implementers are Opus 5.5 subagents, one task at a time; each task gets
|
||||
an independent review (Opus for auth, crypto, concurrency and the swap;
|
||||
Sonnet for small diffs and scoped re-reviews).
|
||||
- "Once I approve a design, carry it through spec, plan and execution with
|
||||
no extra review stops. Report the decisions you made at the end."
|
||||
- Permission rules: `docs/PERMISSIONS.md` (no sudo, no force-push, no
|
||||
`git reset --hard`, no `git clean`, no recursive `rm`). Ask before adding
|
||||
or upgrading a dependency.
|
||||
- The push of this work to `master` on 2026-11-11 was an explicit one-off
|
||||
instruction from the owner. The standing rule in `AGENTS.md` (never push
|
||||
or commit directly to `master`) still applies to future work: branch
|
||||
first and ask before pushing.
|
||||
|
||||
## 3. State
|
||||
|
||||
Updated 2026-11-12. Branch `protondrive-tier1`; `master` on Gitea holds
|
||||
the work only up to the 2026-11-11 pause (`4282540e0`); everything after
|
||||
is local until the owner asks for a push. Plan base (for a whole-branch
|
||||
diff): `06700aaf2`. Rewritten history: see "Open questions" item 1.
|
||||
|
||||
| Work | State |
|
||||
|---|---|
|
||||
| Tasks 1-21 | Implemented and independently reviewed; Task 21 as far as the test accounts allow (section 6) |
|
||||
| Final whole-branch review | Run 2026-11-12 (four reviewers by area); one fix wave (`e95317b52`..`944b36005`); scoped re-review ALL CLOSED |
|
||||
|
||||
What exists now:
|
||||
|
||||
- `backend/protondrive` is a native client of Proton's Drive API
|
||||
(`session.go`, `account.go`, `crypto.go`, `nodes.go`, `cache.go`,
|
||||
`download.go`, `upload.go`) behind the rclone `Fs` in `protondrive.go`.
|
||||
The old bridge libraries are gone from `go.mod`.
|
||||
- `backend/protondrive/protontest` is the fake Proton server (fork-only),
|
||||
`protontest/cmd/protonfake` its standalone binary, started for
|
||||
`fstest/test_all` by `fstest/testserver/init.d/TestProtonDriveFake`.
|
||||
- Tests: `*_internal_test.go` per client file, `fake_test.go`
|
||||
(`TestIntegrationFake`: rclone's contract suite against the fake),
|
||||
`regression_test.go`, `security_test.go` (hostile server),
|
||||
`budget_test.go` (request budgets), `fuzz_internal_test.go`,
|
||||
`golden_internal_test.go` (re-keyed files written by Proton's web app
|
||||
and official CLI, `testdata/golden/`), frozen old-library fixtures in
|
||||
`testdata/oldlib/`.
|
||||
- Fork CI: `.gitea/workflows/ci.yml` (the owner's Gitea has a slow
|
||||
runner; the workflow has never run - the first push will show).
|
||||
- User docs: `docs/content/protondrive.md` describes the new backend; its
|
||||
generated options block is regenerated only for upstream.
|
||||
- Live: see section 6.
|
||||
|
||||
Not done, and easy to assume otherwise:
|
||||
|
||||
- `ChangeNotify`, the persistent cache, Metadata and the small-file upload
|
||||
path (M5, M6). Without events, a mount never sees other clients'
|
||||
changes to folders it has listed.
|
||||
- Five `test_all` tests fail against both the fake and the real service
|
||||
because each `Fs` has its own listing cache (open question 5).
|
||||
- Ranged reads check each block's hash and the content key's signature
|
||||
but not the signed manifest (`docs/TODO.md`).
|
||||
- 2FA, CAPTCHA, Proton's desktop and mobile clients, and a drive created
|
||||
by Proton's own apps are unverified live.
|
||||
|
||||
## 4. How to resume
|
||||
|
||||
Development used the `superpowers:subagent-driven-development` skill
|
||||
(obra's superpowers plugin, version 7.0.0; install with
|
||||
`/plugin marketplace add obra/superpowers-marketplace` then
|
||||
`/plugin install superpowers@superpowers-marketplace`). The skill is a
|
||||
convenience, not a requirement: the per-task loop below can be followed by
|
||||
hand. Its workspace lives at `.superpowers/sdd/001-protondrive-tier1/`,
|
||||
which `.gitignore` excludes; a copy of every Markdown file
|
||||
from it (ledger, per-task briefs, implementer reports, reviews, the shared
|
||||
implementer/reviewer rules and the global constraints) is committed at
|
||||
`docs/plans/001-protondrive-tier1-workspace/`. To resume with the skill:
|
||||
|
||||
```bash
|
||||
mkdir -p .superpowers/sdd/001-protondrive-tier1
|
||||
cp docs/plans/001-protondrive-tier1-workspace/*.md .superpowers/sdd/001-protondrive-tier1/
|
||||
git switch -c protondrive-tier1-resume # never work on master
|
||||
```
|
||||
|
||||
The ledger's first line names the plan; tasks with a `Task N: complete`
|
||||
line are done. Paths inside `implementer-common.md` and
|
||||
`reviewer-common.md` assume the checkout is `/home/user/rclone` and the
|
||||
branch `protondrive-tier1` - adjust both if they differ. One rule in
|
||||
`implementer-common.md` is obsolete: "the old `protondrive.go` must keep
|
||||
compiling until Task 14" (Task 14 is done; there is no old implementation
|
||||
left in the package).
|
||||
|
||||
Per task: extract the brief (`task-N-brief.md` holds the plan text for
|
||||
task N; later ones are not extracted yet - copy the task's section from
|
||||
the plan), record `BASE=$(git rev-parse HEAD)`, dispatch one implementer
|
||||
with the brief plus `implementer-common.md`, then a reviewer with the
|
||||
brief, the report and `git diff -U10 BASE..HEAD`, fix until clean, and
|
||||
record it in the ledger. After Task 21: one whole-branch review on the
|
||||
most capable model (merge base `06700aaf2`), one fix wave, then
|
||||
`git rm` the plan and this workspace copy before any upstream-bound merge.
|
||||
|
||||
### Remaining work
|
||||
|
||||
All 21 plan tasks are done. What is left:
|
||||
|
||||
- The owner's open questions (below), especially question 5.
|
||||
- Later milestones, each needing its own plan: M5 (events and
|
||||
`ChangeNotify`), M6 (persistent metadata cache, rclone Metadata
|
||||
support, small-file single-request upload), M7 (upstream path,
|
||||
deferred by the owner; spec section 13 has the recipe).
|
||||
- Live checks still open (`docs/TODO.md`): 2FA, CAPTCHA, desktop and
|
||||
mobile clients, a drive made by Proton's apps.
|
||||
|
||||
Notes for live work:
|
||||
|
||||
- Live probes run from a scratch `git worktree` with a binary built from
|
||||
it, never with `go test` in the main checkout without
|
||||
`RCLONE_CONFIG=/notfound` unless the live test is the point.
|
||||
- Proton's official CLI 0.9.0 runs in Docker: image
|
||||
`proton-drive-cli:0.9.0` (debian:trixie-slim, non-root, built from the
|
||||
checksum-verified release binary from
|
||||
`https://proton.me/download/drive/cli/0.9.0/linux-x64/proton-drive`),
|
||||
its free-account session in volume `pdcli-free`, e.g.
|
||||
`docker run --rm -v pdcli-free:/home/drive proton-drive-cli:0.9.0 filesystem list /my-files`.
|
||||
Its sign-in is browser-only (`auth login` prints a URL the owner opens)
|
||||
and gives up after about 10 minutes; a late sign-in shows "Invalid
|
||||
code".
|
||||
- `rclone config create` prints the whole new remote, session secrets
|
||||
included, for every backend; never run it where the output is captured.
|
||||
- Golden fixtures: `TestGoldenCapture` with `-capture DIR
|
||||
-capture-label web|cli -capture-remote NAME:PATH` (never `-remote`),
|
||||
into a scratch directory first; inspect for account data before
|
||||
copying into `testdata/golden/` (the ledger records how it was done).
|
||||
- Do not delete the evidence folders listed in section 6.
|
||||
|
||||
## Open questions for the owner
|
||||
|
||||
Answered by the owner on 2026-11-12:
|
||||
|
||||
1. **Usernames in git history:** remove them. Done: commit `901d7fd5b`
|
||||
was rewritten without the usernames (as `31f6aafd9`), the later
|
||||
commits replayed unchanged, and `master` on Gitea force-pushed once
|
||||
(`85f6b2f0e` -> `4282540e0`). A one-off; the no-force-push rule stands.
|
||||
Old hashes in this file and the ledger written before that date
|
||||
(`901d7fd5b`, `85f6b2f0e`, `d211b45e6`, `28f1aaf8a`) now map to
|
||||
`31f6aafd9`, `4282540e0`, `65840bdc5`, `5afbe9e4f`.
|
||||
2. **Gitea Actions:** a runner exists but is slow and limited; keep
|
||||
`.gitea/workflows/ci.yml` light (no `test_all` in CI).
|
||||
3. **Official client interop (Task 21):** allowed - install a released
|
||||
Proton client and use the web app, preferably inside a Docker
|
||||
container, signed in only to the test accounts.
|
||||
|
||||
Still open:
|
||||
|
||||
4. **Rulings.** Section 7 lists 33 decisions made on the owner's behalf;
|
||||
they were reported in summary and not confirmed one by one.
|
||||
5. **Cross-`Fs` cache staleness (Task 18).** Five `test_all` tests are
|
||||
ignored because each `Fs` has its own node cache; a mount never sees
|
||||
other clients' changes. Fix options (shared per-account cache = package
|
||||
state the spec forbids; listing TTL; events in M5) are in
|
||||
`docs/TODO.md`. Current ruling: leave for M5.
|
||||
|
||||
## 5. Test accounts and machine setup
|
||||
|
||||
Machine-local, not in the repository. A new machine needs them recreated.
|
||||
|
||||
- Go 1.27.4 is in `~/.local/opt/go` (symlinked into `~/.local/bin`),
|
||||
golangci-lint 2.15.0 in `~/.local/bin`.
|
||||
- Two Proton accounts the owner created for testing, configured as rclone
|
||||
remotes in `~/.config/rclone/rclone.conf` (mode 600, passwords obscured):
|
||||
- `TestProtonDrive:` - free account; password file
|
||||
`~/proton_rclone_test_acc_password`.
|
||||
- `TestProtonDrivePaid:` - paid account; password file
|
||||
`~/proton_paid_acc_password`. **Never signed in to yet.**
|
||||
Where to find the credentials on this machine (never copy either value
|
||||
into the repository, a commit message, a log or a subagent prompt):
|
||||
- Username: the `username` key of each remote in
|
||||
`~/.config/rclone/rclone.conf`; `rclone config show TestProtonDrive`
|
||||
and `rclone config show TestProtonDrivePaid` print it. That output also
|
||||
contains the obscured password and, for the free account, session
|
||||
tokens, so do not paste it anywhere.
|
||||
- Password: plain text in the two files named above, one per account.
|
||||
The same password is stored obscured in the rclone config, so rclone
|
||||
itself needs nothing more; read the files only if a test must supply
|
||||
the password directly, and then pass it by file or environment, never
|
||||
on a command line that gets logged.
|
||||
Both remotes are set up for password sign-in, which is what every live
|
||||
probe used; browser sign-in has not been exercised against the real
|
||||
service. On a different machine, ask the owner for the accounts.
|
||||
- **Every `go test` must set `RCLONE_CONFIG=/notfound`**, e.g.
|
||||
`RCLONE_CONFIG=/notfound go test -race ./backend/protondrive/...`.
|
||||
Without it the live `TestIntegration` signs in to the real service on
|
||||
every run. Only Task 21 talks to Proton deliberately.
|
||||
- Live probes were run from a scratch `git worktree` so nothing temporary
|
||||
could be committed. Proton's guidelines for third-party clients apply:
|
||||
honest `x-pm-appversion` (`external-drive-rclone@<semver>-<channel>`),
|
||||
official endpoints only, no hammering.
|
||||
- The reference checkout of Proton's official SDK used during development
|
||||
was `~/.cache/protondrive-sdk-ref` (github.com/ProtonDriveApps/sdk at
|
||||
commit `05c9115`, tag `js/v0.23.0`). Read-only; never built or run.
|
||||
- Live state on the free account: folder
|
||||
`rclone-live-probe-20261111-130916` (files `empty.bin`, `five-mib.bin`,
|
||||
`hello.txt` with two revisions, and `my-little-folder/Readme.txt` made by
|
||||
the owner in Proton's web app). Leave it: it is the interop evidence.
|
||||
|
||||
## 6. What has been proven live
|
||||
|
||||
- Password (SRP) sign-in with the new code; a fresh account's Drive volume
|
||||
is created by the client.
|
||||
- Uploads of 0 B, 25 B, 5 MiB+3 and 9 MB files and a second revision are
|
||||
accepted by Proton and read back byte-identical with signature and
|
||||
manifest verification.
|
||||
- Proton's web app lists the files, opens them, and shows "Digital
|
||||
signature verified", the right size, modification time and SHA1.
|
||||
- A folder and a file created in Proton's web app are listed, downloaded
|
||||
and verified by the new code.
|
||||
- With the rewritten backend (binary built at `c00162ed2`): `lsd`, `lsl`,
|
||||
`cat`, `sha1sum`, `about`, `sync` up, `check` (0 differences), new
|
||||
revision, server-side `moveto` and `copyto`, `sync` back byte-identical,
|
||||
`rmdir` of a non-empty folder refused, `purge`.
|
||||
|
||||
Proven on 2026-11-12 (binaries at `502a32b5f` and `b440df2e1`):
|
||||
|
||||
- `TestIntegration` (`fstests`) against `TestProtonDrive:`: 98 pass,
|
||||
29 skipped by the suite's feature checks, 0 fail.
|
||||
- `test_all -n 1` against the real service: only the five cross-`Fs`
|
||||
cache tests fail (owner question 5); `fs/operations`, `fs/sync` and
|
||||
`vfs` otherwise pass. The first run found two real bugs (read-after-
|
||||
write lag after a commit), fixed in `af6fdd789` and `be0053607`, and
|
||||
the re-run confirmed the fix.
|
||||
- Browser (session-fork) sign-in with `rclone config create` against the
|
||||
real service, then listing, a verified read and `config disconnect`.
|
||||
- The paid account: first password sign-in, `about` (64 GiB), mkdir and
|
||||
rmdir. Its drive was most likely created by rclone at that moment.
|
||||
- Interop with Proton's official CLI 0.9.0 (run in Docker, signed in to
|
||||
the free account): each reads, verifies and modifies what the other
|
||||
wrote, both ways, including multi-block, empty and Unicode-named files.
|
||||
- Upgrade/downgrade with released rclone v1.76.0: each reads what the
|
||||
other wrote, byte-identical.
|
||||
- Re-keyed fixtures of files written by the web app and the CLI are in
|
||||
`backend/protondrive/testdata/golden/` (`TestGoldenFixtures`).
|
||||
|
||||
Still not proven: 2FA; CAPTCHA; Proton's desktop and mobile clients; the
|
||||
root-of-trust rule on a drive made by Proton's own apps; the items marked
|
||||
"confirm" in `docs/TODO.md`.
|
||||
|
||||
Folders on the free account to leave in place (interop evidence):
|
||||
`rclone-live-probe-20261111-130916`, `rclone-interop-20261112`,
|
||||
`rclone-golden-cli-20261112`. The official CLI's session lives in the
|
||||
Docker volume `pdcli-free` (image `proton-drive-cli:0.9.0`, built from the
|
||||
checksum-verified release binary).
|
||||
|
||||
The security changes made after `c00162ed2` were re-checked live on the
|
||||
free account without `ignore_signature_errors`: at `e7e43095d` (root of
|
||||
trust, subtree distrust, manifest-first reads: listing, a verified full
|
||||
read of the web-app file, an upload, a `Range: bytes=0-` read, purge) and
|
||||
at `9c90998f7`, the last code commit (sync up, `check` with 0 differences,
|
||||
a 4-stream multi-thread download byte-identical, verified read of the
|
||||
web-app file, server-side move, purge). That drive's volume was created by
|
||||
this client, so the root-of-trust rule is still unchecked on a drive made
|
||||
by Proton's own apps. The hostile cases themselves can only be exercised
|
||||
against the fake.
|
||||
|
||||
## 7. Rulings made during execution
|
||||
|
||||
Decisions the controller made where the plan or spec was silent or wrong.
|
||||
Where this list and the plan or spec disagree, this list is what the code
|
||||
does. Most are in the ledger as `Ruling:` lines with their reason and what
|
||||
it costs if wrong; 16, 17 and 24 to 28 came out of review fix rounds and
|
||||
their reasoning is in `task-14-review.md`, `task-16-review.md`,
|
||||
`task-16-rereview.md`, `task-16-rereview2.md` and `task-17-review.md` in
|
||||
the workspace copy, and in `docs/DECISIONS.md`. The owner was told of them
|
||||
in summary but has not confirmed them one by one.
|
||||
|
||||
1. No separate worktree: work happened on branch `protondrive-tier1` in
|
||||
the main checkout.
|
||||
2. Task 8's retry test is `TestClientShouldRetry` (name collision).
|
||||
3. The fake keeps Proton's published app-version grammar (channel
|
||||
optional); the client always sends a channel.
|
||||
4. Two sessions sharing one config: after a rejected refresh the session
|
||||
re-reads the credential store for up to 3 s to adopt the other's new
|
||||
tokens. No cross-`Fs` lock (the spec allows no package state beyond
|
||||
`unlockMu`).
|
||||
5. The key salt is chosen by the primary user key's ID (old library
|
||||
behaviour), first entry as fallback.
|
||||
6. A fresh account has no Drive volume (`my-files` answers 422 code 2501):
|
||||
the client creates it, as the SDK does.
|
||||
7. Written content follows the SDK (v4 node key advertising AEAD, v6 PKESK
|
||||
content key, SEIPDv2 blocks) even where the old library's comments
|
||||
disagree; confirmed accepted by the real server and web app.
|
||||
8. The intended-recipient check on signatures is kept (stricter than the
|
||||
SDK may be).
|
||||
9. The fake rejects AEAD-encrypted names, passphrases, hash keys and
|
||||
extended attributes written by the client.
|
||||
10. An honoured `Retry-After` is capped at 60 s.
|
||||
11. Upload commit outcome unknown: delete nothing, return
|
||||
`errUploadOutcomeUnknown`. A draft is deleted only when the server
|
||||
explicitly refused the commit (a 4xx with a Proton code, not 408) on
|
||||
a single attempt the pacer did not resend; no read-back decides
|
||||
deletion, as reads may lag (Task 21f fix round, superseding "positive
|
||||
proof" from a read).
|
||||
12. The fake reports a draft's owner in conflict details but does not gate
|
||||
routes by `ClientUID`.
|
||||
13. `Proton-API-Bridge` stays in `go.mod` until Task 19 even though no Go
|
||||
file imports it any more.
|
||||
14. In `rclone config`, browser sign-in is the default only when username
|
||||
and password are not both set.
|
||||
15. `Shutdown` leaves the `Fs` usable (it only drops the node cache); keys
|
||||
are not cleared. This supersedes the plan's "Shutdown calls
|
||||
`client.close()` once". `Disconnect` logs out; later use returns a
|
||||
clear "not signed in" error naming the remote.
|
||||
16. `Rmdir` asks the server, uncached, whether the folder has any child at
|
||||
all and refuses if so.
|
||||
17. `rclone config` on a remote with a saved session asks before signing in
|
||||
again (default no) and logs the old session out when it does.
|
||||
18. A file whose node key is not signed by its owner is refused unless
|
||||
`ignore_signature_errors` is set. A key verified against an unverified
|
||||
folder key is itself unverified, all the way down; creating, uploading,
|
||||
renaming, moving or copying into such a folder is refused without the
|
||||
flag. The flag never disables integrity checks.
|
||||
19. A full read fetches the whole block list and verifies the signed
|
||||
manifest before releasing any byte. Ranged reads do not verify the
|
||||
manifest yet (`docs/TODO.md`). First-byte latency on very large files
|
||||
grows by about one request per 80 MiB.
|
||||
A re-fetched block list that changes a hash is an integrity error.
|
||||
20. `TestBudgetNewObject` allows 8 requests, not the plan's 7 (four folder
|
||||
levels at two requests each; Proton has no lookup-by-name route).
|
||||
21. Every node beneath a folder whose key is unverified is unverified,
|
||||
reads included, even when signed by the user's own address (the
|
||||
implementer went beyond the ruling; a reviewer judged it justified and
|
||||
it was kept). A genuine folder signed by a since-deleted address locks
|
||||
its subtree behind `ignore_signature_errors`.
|
||||
22. Moving, copying or renaming a node whose key is unverified is refused
|
||||
without the flag. With the flag it proceeds, logs an ERROR once per
|
||||
node and operation, and the node is trusted from then on.
|
||||
23. Root of trust: the My Files share's creator and the root folder's
|
||||
signer must be one of the account's own addresses, else the whole
|
||||
drive is unverified. Addresses are compared after lowercasing ASCII
|
||||
letters only; a match is always verified with the account's own keys
|
||||
and never looked up on the server.
|
||||
24. A file may list at most 2^19 blocks (2 TiB at 4 MiB). This bounds the
|
||||
count, not the bytes: about 250-350 MiB of block metadata at the cap.
|
||||
25. Parent chains deeper than 2048, or cyclic, are an integrity error.
|
||||
26. A range covering the whole file takes the verified full-read path and
|
||||
returns only the bytes asked for.
|
||||
27. A folder listing is cached only if the folder's children did not
|
||||
change while it was being read (per-folder change count).
|
||||
28. Names that are not valid UTF-8 are refused on write.
|
||||
|
||||
29. A file whose content key packet signature is missing or invalid is
|
||||
unverified (as 18 and 22): full reads, ranged reads and `Update`
|
||||
refused without `ignore_signature_errors`, ERROR once per node and
|
||||
operation with it. Stricter than Proton's SDK.
|
||||
30. Proton codes 5001 and 5003 are the fatal "upgrade rclone" error; 2000
|
||||
is a plain no-retry error keeping Proton's message; 2028 is retried
|
||||
through the pacer. Supersedes the spec 6.6 mapping of 2000 and 2028.
|
||||
31. Keys from the server or another user whose Argon2 parameters exceed 4
|
||||
passes, parallelism 8 or 128 MiB of memory are refused before the KDF
|
||||
runs (integrity error, not overridable).
|
||||
32. A folder whose name hash key signature fails is unverified for writes
|
||||
(as 18); a rename or move gets a fresh name session key unless the old
|
||||
name verified; extended attributes whose signature fails are not used
|
||||
without the flag (size then comes from the verified block sizes, the
|
||||
modification time from the server, no SHA-1).
|
||||
33. A remote whose config can't be saved (environment variables,
|
||||
connection strings) derives its draft client ID from the volume ID, so
|
||||
it is stable across runs; two machines using such a remote of the same
|
||||
name on one account share it.
|
||||
|
||||
Known protocol limits, documented rather than fixed: a server that serves
|
||||
a whole older, validly signed revision under the current revision ID
|
||||
cannot be detected (Proton's SDK cannot either); below the root, something
|
||||
signed under another user's email is verified against public keys the
|
||||
server itself supplies.
|
||||
|
||||
## 8. Deferred findings
|
||||
|
||||
The final whole-branch review (2026-11-12) left its Minor findings in
|
||||
`docs/TODO.md` under "final review minors (2026-11-12)"; its reports are
|
||||
`final-review-*.md`, `final-fix-report.md` and `final-rereview.md` in the
|
||||
workspace copy. The list below predates it and some items are now fixed.
|
||||
|
||||
Every review's minor findings were deferred to the final whole-branch
|
||||
review. They are listed per task in the ledger (`progress.md`, lines
|
||||
beginning `Task N: minor (deferred)` and "Deferred minors"). Only Tasks
|
||||
14, 16 and 17 have review files in the workspace copy with file and line;
|
||||
for Tasks 1 to 13 and 15 the reviewer's text was not saved and the ledger's
|
||||
one-line entry (plus the implementer's report for that task) is all there
|
||||
is, so expect to re-locate those by reading the code. The whole-branch
|
||||
review will find most of them again. Items that need a decision or a live
|
||||
check are in `docs/TODO.md`. The ones most worth
|
||||
a new agent's attention:
|
||||
|
||||
- Ranged reads do not verify the signed manifest. Planned fix: a bounded
|
||||
per-revision cache of verified block hashes.
|
||||
- With `ignore_signature_errors` set, `Update` of an unverified file logs
|
||||
"reading", not "updating" (`upload.go`, `download.go` `contentKey`).
|
||||
- The depth-limit error is re-wrapped at every level and reaches about
|
||||
88 KB; a hostile server can flood the log with it (`nodes.go`).
|
||||
- `TestListingRacingChange` can hang until the package timeout if it
|
||||
fails before releasing its held request; `protonfake`'s `main_test.go`
|
||||
leaks its `run` goroutine on a failed assertion.
|
||||
- `TestUploadShortSource` failed its goroutine-leak check once under heavy
|
||||
parallel load and never again; `TestListCancel` uses the same
|
||||
goroutine-count approach.
|
||||
- `ModTime` is returned with millisecond precision while `Precision()`
|
||||
says one second.
|
||||
- Move, Copy and DirMove between two different accounts are not refused
|
||||
with `fs.ErrorCant*` (rclone only calls them for the same remote name).
|
||||
- `apiCall` retries non-idempotent mutations after 5xx or network errors.
|
||||
- An extended-attribute `"Size": null` is read as size 0.
|
||||
- API responses have no body-size limit; API error messages are neither
|
||||
bounded nor sanitised before logging.
|
||||
- Two processes sharing tokens through environment variables cannot see
|
||||
each other's refresh.
|
||||
- The race test for volume creation waits 2 s for no benefit.
|
||||
- Real-server behaviours the fake only assumes: error codes for a bad
|
||||
verifier, a missing block and a stale revision ID; whether drafts are
|
||||
hidden from children; what `delete_multiple` does to an active link.
|
||||
|
||||
## 9. Before any upstream work
|
||||
|
||||
The spec's upstream path (section 13) is deferred. If it is ever taken
|
||||
up: rclone's maintainers refuse cgo; Proton's SDK README says the SDK "is
|
||||
not yet ready for third-party production use" and allows personal,
|
||||
non-commercial projects under its guidelines; a Proton engineer wrote that
|
||||
Proton "does not prohibit the usage of rclone, but does not offer formal
|
||||
support for it either" (rclone issue #9252, rclone forum thread 53609).
|
||||
Fork-only paths (`docs/specs/`, `docs/plans/`, `docs/ARCHITECTURE.md`,
|
||||
`docs/DECISIONS.md`, `docs/TODO.md`, `docs/PERMISSIONS.md`, `GEMINI.md`,
|
||||
`.claude/`, `.codex/`, `.gemini/`, the "Fork Workflow" section of
|
||||
`AGENTS.md`, and `backend/protondrive/protontest`) never go into an
|
||||
upstream PR.
|
||||
@@ -0,0 +1,99 @@
|
||||
# Final fix wave — brief (controller rulings)
|
||||
|
||||
Findings files (read the cited entries in full; file:line refers to HEAD 29ad26944):
|
||||
- final-review-crypto.md (C1, C2, I1, I2, I3)
|
||||
- final-review-transfer.md (I-1, I-2)
|
||||
- final-review-auth.md (I-1, I-2)
|
||||
- final-review-tests.md (I-1, I-2 only; I-3..I-5 are the controller's)
|
||||
All in /home/user/rclone/.superpowers/sdd/001-protondrive-tier1/.
|
||||
|
||||
Work order: Critical first, one commit per finding (or tightly related group), each
|
||||
with a regression test written first and seen failing (the reviewers' reproductions
|
||||
are described in the files; reuse them). Commit prefix `protondrive:` (CI: `ci:` is
|
||||
fine), messages user-readable per AGENTS.md.
|
||||
|
||||
## Rulings
|
||||
|
||||
**crypto C1 — unsigned/invalid content key packet signature.** Verify
|
||||
ContentKeyPacketSignature (as the SDK defines it; check the reference checkout
|
||||
~/.cache/protondrive-sdk-ref read-only) on every use of a file's content key: reads
|
||||
(full and ranged) and Update. A missing or invalid signature makes the file
|
||||
unverified, exactly like an unverified node key (ruling 18/22): reads and writes
|
||||
refused unless `ignore_signature_errors`; with the flag, proceed and log an ERROR
|
||||
once per node and operation. Integrity checks are never disabled. This is stricter
|
||||
than the SDK; record it in docs/DECISIONS.md and as handoff ruling 29.
|
||||
|
||||
**crypto C2 — Argon2 parameters from the server.** Before running any S2K/KDF on
|
||||
key material that came from the server or another user (node keys, share keys,
|
||||
anything not the account's own locally verified keys), check the S2K parameters
|
||||
against fixed caps and refuse (integrity error, not overridable by the flag) above
|
||||
them. Choose the caps from what Proton's clients actually generate (gopenpgp v3 and
|
||||
go-crypto defaults; the SDK) with modest headroom, and document them. Also bound
|
||||
iterated-salted S2K counts if the library does not. Add a fuzz/unit test.
|
||||
|
||||
**crypto I1 — unverified hash key used for writes.** A hash key whose signature is
|
||||
missing/invalid makes the folder unverified for writes (same as ruling 18: creating,
|
||||
uploading, renaming, moving into it refused without the flag; ERROR with the flag).
|
||||
|
||||
**crypto I2 — rename/move reusing a name session key whose signature failed.** Never
|
||||
reuse a session key from a name that did not verify; generate a fresh one (if the
|
||||
name verified, keep current behaviour).
|
||||
|
||||
**crypto I3 — unsigned xattrs trusted.** Follow spec 6.4: xattrs whose signature is
|
||||
missing/invalid are not used for modtime/size/SHA1 unless `ignore_signature_errors`
|
||||
(then used with an ERROR once per node). Without them, fall back as the code already
|
||||
does when xattrs are absent (check what that fallback is and that rclone behaves
|
||||
sanely, e.g. ModTime from the revision create time, no SHA1). Do not make the file
|
||||
unreadable just for this.
|
||||
|
||||
**transfer I-1 — draft_client_uid not persisted for env/connection-string remotes.**
|
||||
When the value cannot be persisted (or no value exists), derive a stable one
|
||||
deterministically instead of random per run, e.g. from the account's user ID and the
|
||||
remote name through a hash, so the same remote on the same account gets the same
|
||||
UID across runs; keep saving it when the config can store it. No package state.
|
||||
|
||||
**transfer I-2 — quadratic multi-thread reads of files without recorded block
|
||||
sizes.** Find a fix that keeps returned bytes always correct and every block
|
||||
verified. Preferred: compute each block's plaintext size from the block list
|
||||
without downloading, if and only if it can be done exactly for the packet formats
|
||||
involved (verify against the golden fixtures and the fake's shapes; if compression
|
||||
or variable framing makes it inexact, do not use it). Otherwise a bounded per-`Fs`
|
||||
cache (not package state) of plaintext block sizes learned per revision as blocks
|
||||
are decrypted, so later chunks seek directly. Measure with a test counting block
|
||||
fetches for an N-chunk multi-thread read. Update the docs page and TODO.
|
||||
|
||||
**auth I-1 — error-code mapping.** Map Proton's bad-app-version codes (5003, 5001;
|
||||
confirm in the SDK reference) to the fatal "upgrade rclone" error. Code 2000 becomes
|
||||
a plain `fserrors.NoRetryError` keeping Proton's message (not fatal). Code 2028:
|
||||
treat as Proton's documented meaning (check the SDK); if it can mean rate limiting,
|
||||
make it retryable via the pacer, not fatal. Update spec section 6.6 note via a
|
||||
DECISIONS entry and handoff ruling 30. Tests for each code.
|
||||
|
||||
**auth I-2 — x-pm-drive-requirements.** Log those response headers at debug level
|
||||
when present, as spec 6.6 says (once per value change, not per request).
|
||||
|
||||
**tests I-1 — CI race timeout.** Add `-timeout 45m` (or fitting the job timeout) to
|
||||
the CI `go test -race` step; consider `-p`/`-parallel` limits for the slow runner.
|
||||
|
||||
**tests I-2 — CI Go version / govulncheck.** Pin Go to the 1.27 line the code was
|
||||
verified with (go-version: '1.27.x' or exact 1.27.4) instead of go.mod's minimum,
|
||||
and pin govulncheck to a fixed version instead of @latest.
|
||||
|
||||
Minors from all four files: fix the ones that are cheap and local to code you are
|
||||
already touching; list every other Minor in docs/TODO.md under a single
|
||||
"final review minors (2026-11-12)" heading with file references, so nothing is lost.
|
||||
|
||||
## Rules
|
||||
|
||||
Read implementer-common.md (its rule about the old protondrive.go is obsolete) and
|
||||
global-constraints.md first. RCLONE_CONFIG=/notfound on every go test; never contact
|
||||
Proton; never read ~/.config/rclone/rclone.conf or ~/proton_*password; never pass
|
||||
-remote/-capture-remote. No dependency changes. Do not edit
|
||||
docs/plans/001-protondrive-tier1-HANDOFF.md, docs/plans/001-protondrive-tier1-workspace/,
|
||||
or docs/specs/001-protondrive-tier1.md section 13 (the controller edits those in
|
||||
parallel); you may append rulings text to the controller via your report instead.
|
||||
Done = go build ./..., go vet ./backend/protondrive/..., golangci-lint run
|
||||
./backend/protondrive/... (0 issues), full race suite once at the end, TestGoldenFixtures
|
||||
passing. No sudo, no force-push, no git reset --hard, no git clean, no recursive rm,
|
||||
never push, never switch branches. End commits with:
|
||||
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
|
||||
@@ -0,0 +1,267 @@
|
||||
# Final fix wave: implementer report
|
||||
|
||||
Branch `protondrive-tier1`, starting at 29ad26944. One commit per finding.
|
||||
The controller's `ea8d09ce4` landed between mine and is not part of this
|
||||
work. Every test ran with `RCLONE_CONFIG=/notfound`. No dependency
|
||||
changes, nothing pushed, and no edits to the handoff, the workspace copy
|
||||
or spec section 13.
|
||||
|
||||
## Commits
|
||||
|
||||
| Commit | Finding |
|
||||
|---|---|
|
||||
| e95317b52 | crypto C1: content key signature |
|
||||
| db7afafc0 | crypto C2: Argon2 caps on node and share keys |
|
||||
| b3f5685e6 | crypto I1: unverified hash key blocks writes |
|
||||
| 67e4cfeb7 | crypto I2: no session key reuse for an unverified name |
|
||||
| 94eedc942 | crypto I3: unsigned xattrs not used |
|
||||
| 674b538ca | transfer I-1: stable draft client ID when it can't be saved |
|
||||
| 788a711dc | transfer I-2: learnt block sizes make multi-thread reads linear |
|
||||
| 5dbf5b04c | auth I-1: error-code mapping (5001/5003 fatal, 2000 no-retry, 2028 retried) |
|
||||
| b4a23a31e | auth I-2: x-pm-drive-requirements logged |
|
||||
| b5da26169 | tests I-1 and I-2: CI timeout, Go 1.27.x, govulncheck pinned |
|
||||
| 634a2423e | (found while working) tests honour RCLONE_CONFIG whichever tests run |
|
||||
| 30cfcac41 | auth M-1 and M-2: refresh 5xx/429/408 retried |
|
||||
| d3aee2af1 | cheap minors: crypto M5 and M6, transfer M-3 and M-8, fake gofmt |
|
||||
| bfa9b7122 | test minors 1, 6, 8 and 9, tests minors 13 and 14 (docs), every other minor logged in TODO |
|
||||
| ad3552e2e | ARCHITECTURE line for the block size cache |
|
||||
| 944b36005 | fake test follows the renamed code constant (found by the final race run) |
|
||||
|
||||
## Per finding
|
||||
|
||||
### crypto C1: unsigned or invalid content key signature (e95317b52)
|
||||
- **Failing test first:** two new `hostileContents` cases in `TestHostileServer`
|
||||
("unsigned content key the server made", "content key signed by another
|
||||
key"), and "ranged reads and updates of a file with content-key-unsigned /
|
||||
content-key-signature". Before the fix the read of an unsigned key returned
|
||||
content (`security_test.go:204 An error is expected but got nil`), the Update
|
||||
went through (`:570`), and the per-operation ERROR line was missing.
|
||||
- **Fake:** the new corruptions `content-key-unsigned` and `content-key-signature`
|
||||
make a new content key and re-encrypt every block under it. The blocks and
|
||||
manifest are re-signed with the user's key, which is more than a real server
|
||||
can do, so only the content key signature can catch it.
|
||||
- **Change:** `decryptContentKey` always verifies, so a missing signature is
|
||||
`errSignatureMissing`. `contentKey(n, op)` takes "reading" or "updating". Without
|
||||
the flag it refuses with errIntegrity and an op-specific hint. With the flag it
|
||||
logs an ERROR once per node and operation (`reportOnce("content key "+op)`). The
|
||||
pre-2023 all-addresses fallback now applies only to a signature that is present
|
||||
but invalid, as in the SDK. This also fixes transfer M-7 (Update logged
|
||||
"reading").
|
||||
- **Existing tests changed:** `TestContentKeyRoundTrip/NoSignature` (now expects
|
||||
`errSignatureMissing`) and `TestDownloadContentKey` ("unsigned" is now refused,
|
||||
plus a new "unsigned before 2023" case).
|
||||
- **Deviation:** none. There is no date exemption for unsigned keys, because the
|
||||
date comes from the server. Every golden fixture has a signed content key
|
||||
(`TestGoldenFixtures` passes). Recorded in DECISIONS (ruling 29) and on the docs page.
|
||||
|
||||
### crypto C2: Argon2 parameters from the server (db7afafc0)
|
||||
- **Failing test first:** `TestNodeKeyS2KLimits`. The patched keys (memory 2^18,
|
||||
passes 255, parallelism 255, and memory 2^31 at the library limit) got no
|
||||
errIntegrity. With a stub check, the memory case ran a 256 MiB KDF in 0.2 s.
|
||||
- **Change:** `decryptNodeKeys` (used for node and share keys) calls
|
||||
`unlockForeignKey`. That function unarmors once, runs `checkKeyS2K` on every
|
||||
secret key and subkey packet before the KDF can run, then unlocks. Neither
|
||||
go-crypto nor gopenpgp exposes a key's S2K parameters, so `s2kSpecifier`
|
||||
finds them by hand. It parses the packet with go-crypto (cheap: no KDF), takes
|
||||
the public key's serialized body as the prefix, then walks the usage, count,
|
||||
cipher, AEAD and length octets the way go-crypto does.
|
||||
- **Caps:** Argon2 may use at most 4 passes, parallelism 8 and 2^17 KiB (128 MiB).
|
||||
The defaults of gopenpgp and OpenPGP.js are 3, 4 and 64 MiB, and Proton's node
|
||||
keys use iterated-and-salted S2K anyway. Going over a cap is errIntegrity, and
|
||||
the flag can't override it.
|
||||
- **Iterated-salted:** not capped further. go-crypto's one-byte count encoding
|
||||
already bounds it to 65011712 bytes hashed (4 times the default), and a lower
|
||||
cap could refuse genuine old keys.
|
||||
- **Fuzz:** `FuzzCheckKeyS2K` ran 20 s locally (1.3M execs, no panic).
|
||||
- **Deviation:** the account's own user keys are not checked. The ruling scopes
|
||||
this to foreign keys, and a genuine user key's parameters aren't known. This is
|
||||
logged in TODO. Address keys are safe, because their token's signature is checked
|
||||
before the unlock. DECISIONS entry added.
|
||||
|
||||
### crypto I1: unverified hash key used for writes (b3f5685e6)
|
||||
- **Failing test first:** "writing into a folder whose hash key the server made".
|
||||
The upload into the folder succeeded before the fix (`:272`).
|
||||
- **Change:** `node.hashKeyErr` is set when the hash key signature fails and isn't
|
||||
date-exempt. `targetFolder` refuses without the flag, which covers create,
|
||||
upload, rename (within the folder), move into and copy into. With the flag it
|
||||
logs an ERROR once per folder. Reads are not affected. A comment notes that the
|
||||
date exemptions use the server's CreateTime (crypto M4). Docs page updated.
|
||||
- **Deviation:** not folded into `keys.unverified`. Doing that would make every
|
||||
child unreadable, and the ruling covers writes only.
|
||||
|
||||
### crypto I2: rename/move reusing an unverified name's session key (67e4cfeb7)
|
||||
- **Failing test first:** "renaming or moving a node whose name the server
|
||||
encrypted". The new stored name's session key equalled the server's for both
|
||||
the rename and the move. The fake gained `User.NameSessionKey` and the
|
||||
`name-unsigned` corruption.
|
||||
- **Change:** `node.nameSigned` records whether the name signature actually
|
||||
verified, taken before the pre-2021 exemption so that a back-dated name doesn't
|
||||
count. `renameSessionKey` returns nil, which means a fresh key, unless the name
|
||||
is signed. The same test checks that a verified name keeps its key.
|
||||
- **Deviation:** a pre-2021 unsigned (exempt) name also gets a fresh key. This is
|
||||
the safe reading of "did not verify".
|
||||
|
||||
### crypto I3: unsigned xattrs trusted (94eedc942)
|
||||
- **Failing test first:** "unsigned extended attributes". The forged modtime
|
||||
(2099), the forged SHA-1 and the forged 1 TiB size were all used, and the
|
||||
folder's modtime too.
|
||||
- **Change:** without the flag, when the xattr signature fails, `n.X` becomes
|
||||
`blockSizesOnly()`. The ERROR line says the modification time, size and SHA-1
|
||||
are not used. With the flag the attributes are used, logged once per node by the
|
||||
existing verify report.
|
||||
- **Deviation:** I did not use the plain "xattrs absent" fallback. That fallback
|
||||
reports the *encrypted* size, so every `rclone copy` of the file would fail its
|
||||
size check, which in practice makes the file unreadable. The ruling forbids
|
||||
that. Instead I keep only the recorded block sizes, which every read checks
|
||||
block by block, and take Size from their sum. ModTime falls back to the server's
|
||||
ModifyTime and there is no SHA-1. If there are no block sizes, Size is the
|
||||
encrypted size as before. The test checks that full and ranged reads still work.
|
||||
Docs page updated.
|
||||
|
||||
### transfer I-1: draft_client_uid for env/connection-string remotes (674b538ca)
|
||||
- **Failing test first:** `TestDraftClientUIDStableWhenUnsaved`, using a mapper
|
||||
whose `Set` does nothing. Two runs of the same remote got different IDs.
|
||||
- **Change:** `NewFs` sets a random ID through `m.Set`, then reads it back with
|
||||
`m.Get`. If it didn't stick (env vars or a connection string), it uses
|
||||
`client.derivedDraftClientUID(name)` = hex(SHA-256("rclone draft client\0" +
|
||||
volumeID + "\0" + name))[:32].
|
||||
- **Deviation:** I used the account's volume ID rather than the user ID. The
|
||||
account struct doesn't keep the user ID, and the volume ID is just as stable
|
||||
per account. Config-file remotes still get a random saved ID, so two machines
|
||||
with their own config files don't collide. The docs say that two machines
|
||||
running an env-configured remote of the same name count as one remote.
|
||||
|
||||
### transfer I-2: quadratic multi-thread reads (788a711dc)
|
||||
- **Option chosen:** a bounded per-client cache, not computing sizes from the
|
||||
listing. The packet framing (partial body lengths, the literal packet header and
|
||||
any compression) depends on the writer, so plaintext size can't be computed
|
||||
exactly from encrypted size.
|
||||
- **Failing test first:** `TestDownloadChunksUnrecordedBlockSizes`, 32 blocks read
|
||||
as 8 sequential range opens. It fetched 145 blocks before the fix and 39 after.
|
||||
A range read afterwards fetches exactly its 3 blocks, and a block list changed
|
||||
since the sizes were learnt is errIntegrity.
|
||||
- **Change:** `blockSizeCache` lives on `client`, so it is per Fs and not package
|
||||
state. It holds at most 16 revisions and 65536 blocks (about 2.5 MiB). The
|
||||
reader learns sizes in order in `next()`, as a contiguous prefix with each
|
||||
block's hash. `openRead` uses `locate` to start at the right block, and to stop
|
||||
at the last block when the prefix covers the range, which partly fixes transfer
|
||||
M-5.
|
||||
- **Correctness:** every block is still hash-checked. If the start block's listed
|
||||
hash differs from the learnt hash, or a re-read block's hash or size differs,
|
||||
that is errIntegrity.
|
||||
- **Limit:** chunks that start together still each read from block 1. The docs
|
||||
page, the TODO entry and the conformance checklist are updated.
|
||||
- **Existing tests changed:** `TestDownloadRanges` (the unrecorded case now
|
||||
benefits from learnt sizes) and the always-3 nit in it.
|
||||
|
||||
### auth I-1: error codes (5dbf5b04c)
|
||||
- **Codes checked in the SDK reference:**
|
||||
- 5003 is "You need to update this app" (`coreTypes.ts`).
|
||||
- 5001 is from go-proton-api.
|
||||
- 2028 is "feature disabled, rate-limited or blocked because of abuse"
|
||||
(`driveTypes.ts`).
|
||||
- **Failing tests first:** `TestTranslateError` (5001 fatal, 2000 no-retry),
|
||||
`TestClientShouldRetry` (2028 retried, and with Retry-After honoured),
|
||||
`TestRegressionFeatureBlockedRetried`, `TestRegressionRenameResendNotFatal`
|
||||
(the reviewer's lost-rename reproduction, which before the fix was fatal with
|
||||
"upgrade rclone"). `TestRegressionAppVersionRejected` now uses 5003.
|
||||
- **Change:**
|
||||
- api: `CodeFeatureBlocked=2028`, `CodeAppVersionMissing=5001` and
|
||||
`CodeAppVersionBad=5003` (renamed; it used to mean 2028).
|
||||
- `translateError`: 5001/5003 are fatal with "upgrade rclone". 2000 is a
|
||||
`NoRetryError` that keeps Proton's message. 2028 falls through as a plain
|
||||
error.
|
||||
- `shouldRetry`: 2028 is retried, through `RetryAfterError` when a Retry-After
|
||||
header is present.
|
||||
- **Deviation:** 2064 is left unmapped, because the user can set `app_version`
|
||||
wrong themselves. DECISIONS entry added (ruling 30).
|
||||
|
||||
### auth I-2: x-pm-drive-requirements (b4a23a31e)
|
||||
- **Failing test first:** `TestDriveRequirementsLogged`. The fake gained
|
||||
`SetDriveRequirements`.
|
||||
- **Change:** `session.do` calls `observeRequirements`, which logs at debug, `%q`,
|
||||
truncated to 1 KiB, only when the joined value changes.
|
||||
|
||||
### tests I-1 and I-2: CI (b5da26169)
|
||||
- `go test -race -timeout 40m`, and `timeout-minutes` raised from 45 to 60 to make
|
||||
room for the fuzz step.
|
||||
- `go-version: '1.27.x'`.
|
||||
- `govulncheck@v1.9.0`, the latest release on proxy.golang.org at the time of
|
||||
writing (queried with `go list -m -versions`).
|
||||
- No `-p`/`-parallel` limits: the race suite is one big package, so `-p` doesn't
|
||||
help, and limiting `-parallel` would make it slower.
|
||||
|
||||
### Extra: tests read the real rclone.conf (634a2423e)
|
||||
- While working I saw "Using config file from ~/.config/rclone/rclone.conf" in a
|
||||
captured test log. `RCLONE_CONFIG` is only applied by `fstest.Initialise`, which
|
||||
only `TestIntegration` runs, so any filtered run (`-run X`) read the real config
|
||||
holding the live `TestProtonDrive:` remote. Nothing contacted Proton, since the
|
||||
tests use env-configured fake remotes, but the file was read.
|
||||
- `TestMain` now calls `config.SetConfigPath(RCLONE_CONFIG)` first; `/notfound`
|
||||
means memory-only. Earlier runs in this and previous sessions read that file.
|
||||
|
||||
## Minors
|
||||
|
||||
**Fixed:**
|
||||
- crypto M4: code comment, plus a TODO line.
|
||||
- crypto M5: `decryptBlock` moved to the tests; the `reencryptPassphrase` alias
|
||||
removed.
|
||||
- crypto M6: `truncateName` cuts on a rune boundary, with a test.
|
||||
- transfer M-3: godoc rewrapped.
|
||||
- transfer M-5: partly; the range end is known once sizes are learnt.
|
||||
- transfer M-7: covered by C1.
|
||||
- transfer M-8: the superseded message gives the state and revision.
|
||||
- auth M-1 and M-2: refresh 5xx/429/408 retried, and 408 no longer kills the
|
||||
session; `TestRefreshTransientErrorRetried` failed first for 503 and for 408.
|
||||
- tests 1: timing bounds raised to 3 s.
|
||||
- tests 6: the 502-once test counts the uploads.
|
||||
- tests 8: `TestRefusedCommit` pins the 408 exclusion.
|
||||
- tests 9: "assumed" renamed and the always-3 bound fixed.
|
||||
- tests 13: conformance checklist.
|
||||
- tests 14: the docs explain errUploadUnconfirmed.
|
||||
|
||||
**Logged** in docs/TODO.md under "final review minors (2026-11-12)":
|
||||
- crypto M1, M2, M3, M7 and M8.
|
||||
- transfer M-1, M-2, M-4, M-6 and M-9.
|
||||
- auth M-3 to M-11.
|
||||
- tests 2, 3, 4, 5, 7, 10, 11, 12, 15, 16 and 17.
|
||||
- tests 17 (.codex rules) is a permission-file change, which I left to the owner.
|
||||
|
||||
**Not mine:** tests 18 and 19 (handoff) and tests I-3 to I-5 (the controller's).
|
||||
|
||||
## Verification (at ad3552e2e)
|
||||
- `go build ./...`: OK.
|
||||
- `go vet ./backend/protondrive/...`: OK.
|
||||
- `golangci-lint run ./backend/protondrive/...`: 0 issues.
|
||||
- Full race suite at ad3552e2e: `RCLONE_CONFIG=/notfound go test -race -count=1 -timeout 40m ./backend/protondrive/...`
|
||||
- `backend/protondrive`: ok, 332.6 s.
|
||||
- `protonfake`: ok.
|
||||
- `protontest`: failed `TestFaultInjection/Skip`. It still used the
|
||||
`api.CodeAppVersionBad` name for 2028, and my rename had missed it. Fixed in
|
||||
944b36005; the `protontest` race run then passed (39.1 s), and lint was
|
||||
re-run with 0 issues.
|
||||
- `TestGoldenFixtures`: PASS (also run alone with -v).
|
||||
|
||||
## Suggested handoff ruling text
|
||||
|
||||
29. A file whose content key signature is missing or invalid is
|
||||
unverified, like an unverified node key (rulings 18, 22): reads, full
|
||||
and ranged, and updates are refused unless `ignore_signature_errors`
|
||||
is set, when an ERROR is logged once per file and operation. The
|
||||
pre-2023 fallback to the account's other address keys applies only to
|
||||
a signature that is present. Stricter than the SDK, which skips the
|
||||
check when the signature is absent. In the same wave: a folder whose
|
||||
hash key signature fails can't be written into without the flag;
|
||||
extended attributes whose signature fails aren't used without it
|
||||
(modtime from the server, no SHA-1, size from the recorded block sizes);
|
||||
a rename or move never reuses the session key of a name whose signature
|
||||
didn't verify; node and share keys with an Argon2 S2K beyond 4 passes,
|
||||
parallelism 8 or 128 MiB are an integrity error the flag can't override.
|
||||
|
||||
30. Error codes: 5001 and 5003 (missing or rejected app version) are a
|
||||
fatal error telling the user to upgrade rclone; 2000 (requirements not
|
||||
met, e.g. a stale `OriginalHash` after a resent rename) is a no-retry
|
||||
error keeping Proton's message; 2028 (feature disabled, rate-limited or
|
||||
blocked) is retried through the pacer, honouring `Retry-After`. This
|
||||
supersedes spec 6.6's 2000/2028 fatal mapping. `x-pm-drive-requirements`
|
||||
is logged at debug level when it changes.
|
||||
@@ -0,0 +1,99 @@
|
||||
# Final fix wave: scoped re-review (29ad26944..944b36005)
|
||||
|
||||
Verdict: **ALL CLOSED**. No open or new Critical or Important findings. There are 3 new Minors.
|
||||
|
||||
How I checked: I read the diff (I skipped ea8d09ce4, the controller's spec section 13 commit) and the code at HEAD where needed. I compared it with the SDK reference (`~/.cache/protondrive-sdk-ref`, read-only) and with gopenpgp v3.4.1 and go-crypto v1.5.2 in the module cache. I worked in a scratch worktree, `fr-rereview` at 944b36005, which is now removed with `git worktree remove` (no --force). Every `go test` ran with `RCLONE_CONFIG=/notfound`. I did not touch `/home/user/rclone`, apart from this file.
|
||||
|
||||
## Findings: closed?
|
||||
|
||||
| Finding | Status | Test that fails without the fix (mutation run here) |
|
||||
|---|---|---|
|
||||
| crypto C1, unsigned content key | Closed | I put back `if sig == "" { return sk, nil, nil }` in `decryptContentKey`. That fails `TestContentKeyRoundTrip/NoSignature`, `TestHostileServer/unsigned_content_key_the_server_made`, `.../ranged_reads_and_updates_of_a_file_with_content-key-unsigned` and `TestDownloadContentKey/unsigned{,_before_2023}`. Reads (full and ranged) and Update (`upload.go:319`, op "updating") go through `contentKey(n, op)`. It has no other callers. |
|
||||
| crypto C2, Argon2 from the server | Closed | Mutation 1: removed the `checkKeyS2K` call. `TestNodeKeyS2KLimits/{Memory,Passes,Parallelism}` fail, and the KDF runs. Mutation 2: shifted the specifier offsets by one. `.../Accepted` fails. `decryptNodeKeys` is the only unlock path for node and share keys (`nodes.go:367,530`). The offsets match go-crypto's `PrivateKey.parse` (`private_key.go:219-271`): usage, the v5/v6 count, cipher, AEAD for 253, the v6 length. |
|
||||
| crypto I1, unverified hash key used for writes | Closed | I made the `hashKeyErr` refusal unreachable. `TestHostileServer/writing_into_a_folder_whose_hash_key_the_server_made` fails. |
|
||||
| crypto I2, rename/move reusing the session key of an unverified name | Closed | I made `renameSessionKey` ignore `nameSigned`. `TestHostileServer/renaming_or_moving_a_node_whose_name_the_server_encrypted` fails. |
|
||||
| crypto I3, unsigned xattrs | Closed (accepted deviation) | I removed `n.X = n.X.blockSizesOnly()`. `TestHostileServer/unsigned_extended_attributes` fails. |
|
||||
| transfer I-1, draft_client_uid | Closed (accepted deviation) | I made the derived-UID branch a no-op. `TestDraftClientUIDStableWhenUnsaved` fails. `m.Get` after `m.Set` works as a test of whether the value was saved: `fs.setConfigFile.Set` saves only when the section exists (`fs/configmap.go:90`), and `getConfigFile` reads the value back. |
|
||||
| transfer I-2, quadratic multi-thread reads | Closed | I made `openRead` skip `learnt.locate`. `TestDownloadChunksUnrecordedBlockSizes` fails with 146 blocks fetched for 8 chunks (the limit is 64). |
|
||||
| auth I-1, error codes | Closed | I mapped 2000/2028 back to fatal. `TestTranslateError` and `TestRegressionRenameResendNotFatal` fail. 5001 and 5003 are confirmed fatal in the code and the tests. |
|
||||
| auth I-2, x-pm-drive-requirements | Closed | `TestDriveRequirementsLogged`. The header is logged once per change and truncated to 1 KiB. |
|
||||
| tests I-1, CI race timeout | Closed | `-timeout 40m`, inside the job's `timeout-minutes: 60`. |
|
||||
| tests I-2, CI Go version and govulncheck | Closed | `go-version: '1.27.x'` (local Go is 1.27.4). `govulncheck@v1.9.0`, which `go list -m -versions golang.org/x/vuln` shows is the latest. |
|
||||
|
||||
## Regression checks
|
||||
|
||||
**(a) Reading files that real Proton clients wrote.** `TestGoldenFixtures`, `TestGoldenRoundTrip` and `TestGoldenRefusesForeignKeys` pass at HEAD. Re-made signatures could hide a real-world mismatch, so I also checked two things:
|
||||
|
||||
- **The fixtures keep each signature's role and keep missing signatures missing.**
|
||||
- `goldenCapture.verified` (`golden_internal_test.go:864`) made the original capture fail on any signature failure the backend reported against the real account.
|
||||
- `resignDetached` keeps `""` as `""` and re-signs with the replacement for the same signer role.
|
||||
- The rekey notes record which key signed what in the captured web and CLI files:
|
||||
- `ContentKeyPacketSignature`: by the node key, 4 files.
|
||||
- `NodeHashKey`: by the node key, including both roots.
|
||||
- `XAttr`: by the address key.
|
||||
- The oldlib fixture also has a `ContentKeyPacketSignature`.
|
||||
- **rclone verifies with the same key sets as the SDK:**
|
||||
- The content key with node key plus signature-email keys, or the parent key for an anonymous node. That is `[key, ...keyVerificationKeys]`, `cryptoService.ts:637-644`.
|
||||
- The fallback to all of the account's addresses only applies to a signature that is present but invalid (`:652`).
|
||||
- The hash key with node key plus signature-email keys. That is `decryptNodeHashKey`, `driveCrypto.ts:534-537`.
|
||||
- Revision and folder xattrs with the signature-email keys, or the node key when there is no email (`:582-584`, `:177-179`).
|
||||
|
||||
So the new refusals hit only signatures that the SDK would also report as failed. The one place rclone is stricter than the SDK is an absent content key signature. That is the recorded ruling 29.
|
||||
|
||||
**(b) Argon2 caps.**
|
||||
- The defaults are 3 passes, parallelism 4 and memory exponent 16:
|
||||
- go-crypto `Argon2Config` (`s2k_config.go:98-115`).
|
||||
- gopenpgp v3.4.1's RFC9580 preset (`preset.go:72-78`), which uses an empty `Argon2Config`, so it gets those defaults.
|
||||
- OpenPGP.js writes the same.
|
||||
- The caps are 4 passes, parallelism 8 and exponent 17, so they leave headroom.
|
||||
- The SDK checkout doesn't mention S2K or Argon2 anywhere, so it sets none of its own.
|
||||
- Iterated-and-salted S2K is not checked. Its one-octet count encoding bounds it already.
|
||||
- `checkKeyS2K` parses packets without running the KDF.
|
||||
- I noted no new problem here.
|
||||
|
||||
**(c) Block-size cache.**
|
||||
- It is per Fs: a `blockSizeCache` value on `client`, with no package state.
|
||||
- It is bounded: at most 16 revisions and 65536 blocks.
|
||||
- The eviction loops leave the counters consistent.
|
||||
- A new revision can't be left in `order` without its entry in `revs`, as long as `learntBlocks > 0`.
|
||||
- It is safe for concurrent use: every access holds the mutex, and `locate` clones the start hash before unlocking.
|
||||
- The focused `-race` run of `TestDownloadChunksUnrecordedBlockSizes`, `TestDownloadRanges` and `TestDownloadRangesUnrecordedBlockSizes` passed.
|
||||
- I also wrote and ran a scratch `-race` test: 8 to 9 concurrent chunk readers per round, 3 rounds, at offsets that don't line up with blocks. All bytes were correct and no race was reported.
|
||||
- It never returns wrong bytes from a genuine server:
|
||||
- Sizes are learnt only from blocks that were decrypted and checked against their hash, and only as a contiguous prefix.
|
||||
- The start block's listed hash must equal the learnt one.
|
||||
- A re-read block must match in both hash and size.
|
||||
- If eviction happens between `locate` and `learn`, learning just stops; it never corrupts.
|
||||
- It is invalidated on a new revision, because entries are keyed by revision ID and a committed revision's blocks are immutable.
|
||||
|
||||
**(d) Error-code remap.**
|
||||
- 2028 is retried only by `shouldRetry` inside the pacer, so the low-level retry count bounds it. A `Retry-After` value is capped by `maxRetryAfter`. After the retries, the plain error goes back to rclone.
|
||||
- 5001 and 5003 are 4xx, so they are not retried, and are fatal.
|
||||
- 2000 is `NoRetryError`.
|
||||
- The refresh path now returns `nil, refreshErr`. A rejected refresh is still `errSessionExpired`, which is not retried. A 5xx, 408 or 429 on the refresh is retried within the pacer's limit.
|
||||
- Nothing loops without a bound.
|
||||
|
||||
**(e) RCLONE_CONFIG in tests.**
|
||||
- `TestMain` lives in `package protondrive_test`, and Go builds the internal and external test files into one binary, so it covers every test in `backend/protondrive`.
|
||||
- I proved it with a fake `HOME`/`XDG_CONFIG_HOME` holding a sentinel `rclone.conf`, so the user's file was never at risk. I ran a scratch test that calls `configfile.Install()` with `RCLONE_CONFIG=/notfound`:
|
||||
- At HEAD it got path `""` (memory only) and no sections.
|
||||
- With the `TestMain` lookup disabled, it got the sentinel path and `[sentinelremote]`.
|
||||
- `protontest` and `protonfake` don't import `fs/config`.
|
||||
|
||||
**(f) CI file.** YAML structure is fine. The fuzz step has no `-timeout`, but each target runs for only 20 s. `go vet` passes. `golangci-lint run ./backend/protondrive/...` gives 0 issues (run here).
|
||||
|
||||
## New Minors (fold into docs/TODO.md "final review minors")
|
||||
|
||||
- **M-a. Ranged reads use block sizes from unsigned xattrs to find their start** (`nodes.go` decryptNode → `blockSizesOnly`; `download.go:116-151`).
|
||||
- Each fetched block's size is checked, but the blocks before the range's first block are never fetched.
|
||||
- So a server that forges unsigned xattrs with block 1 at 4 MiB-100 (and true sizes after it) makes a read at offset 4 MiB-100 start at block 2 and silently return genuine bytes 100 bytes out of place. A full read still fails.
|
||||
- This is the same class as the recorded "ranged read can be fed swapped blocks" limit, and it is no worse than before the fix, when all unsigned xattrs were used. But the docs page says the sizes are ones "which reads check".
|
||||
- Cheapest fix: keep the summed Size, but read such a file as if it recorded no block sizes, so the checked block-size cache places ranges. Or log it in TODO next to the swapped-blocks entry.
|
||||
- **M-b. `TestMain` applies `RCLONE_CONFIG` only when it is set** (`budget_test.go:39-45`).
|
||||
- A developer who forgets the variable still reads the default config.
|
||||
- AGENTS.md and CI always set it, so this is a gap in a safety net, not a defect.
|
||||
- Fix: use memory-only config when the variable is unset.
|
||||
- **M-c. Draft UID when the config can't be written** (`protondrive.go` NewFs).
|
||||
- If the remote's section exists but saving the config file fails (for example a read-only file), `ConfigFileSet` still keeps the value in memory. `m.Get` then returns the random UID, so it stays random on every run, like the original I-1.
|
||||
- This is rare and logged by rclone as "Failed saving config".
|
||||
- Fix: always use the derived UID unless the value is already in the config.
|
||||
@@ -0,0 +1,193 @@
|
||||
# Final review: authentication, sessions, rclone-facing layer
|
||||
|
||||
Reviewer area: `session.go`, `account.go`, `protondrive.go`, `api/types.go`
|
||||
(as used by these). HEAD 29ad26944.
|
||||
|
||||
Counts: Critical 0, Important 2, Minor 11.
|
||||
|
||||
Scratch proof tests (worktree since removed) are saved at
|
||||
`/tmp/claude-1000/-home-user-rclone/8cc17642-4f33-4045-ad2c-177bf869b82a/scratchpad/zz_review_auth_test.go.txt`.
|
||||
|
||||
## Important
|
||||
|
||||
### I-1. Error-code table: generic code 2000 is fatal ("upgrade rclone"), and the real bad-app-version code 5003 is not mapped
|
||||
|
||||
`protondrive.go:396-397`, `api/types.go:17,37-39`.
|
||||
|
||||
`translateError` maps `CodeInvalidRequirements` (2000) and `CodeAppVersionBad`
|
||||
(2028) to `fserrors.FatalError("this version of rclone is no longer accepted
|
||||
by Proton: upgrade rclone")`. The spec (6.6) asks for this, so this re-opens
|
||||
a recorded ruling. The reason is a defect the ruling didn't foresee:
|
||||
|
||||
- 2000 is Proton's generic "requirements not met" code. The fake, which
|
||||
models the SDK, returns it for a stale `OriginalHash` on rename/move
|
||||
(`protontest/mutate.go:527,574`), a stale `CurrentRevisionID` on revision
|
||||
creation (`protontest/upload.go:365`), a folder moved into itself, and
|
||||
commit validation (`upload.go:562`). Each of these then aborts the whole
|
||||
rclone run with a false "upgrade rclone" message.
|
||||
- A realistic trigger is a lost response. `apiCall` resends PUT rename/move
|
||||
after a 5xx or network error (the recorded Task 9 deferral). The first
|
||||
request has already landed, so the resend fails its `OriginalHash` check.
|
||||
**Reproduced** (scratch test `TestReviewRenameResendFatal`): inject 502
|
||||
AfterHandler once on `PUT .../rename`. The rename lands on the server
|
||||
(`new.txt` present), but `c.rename` returns `OriginalHash doesn't match
|
||||
(Code=2000)`, and `translateError(err, opMove)` gives
|
||||
`IsFatalError == true` with "upgrade rclone". A `rclone move` therefore
|
||||
aborts the whole sync over a rename that succeeded. A second trigger:
|
||||
another client uploads a new revision between `activeNode` and
|
||||
`createRevision` in `Object.Update`.
|
||||
- 2028, by our own comment in `api/types.go:37-38`, covers "a feature is
|
||||
disabled, rate-limited or blocked for abuse". A rate-limited feature
|
||||
should not give a fatal "upgrade rclone".
|
||||
- The code Proton actually returns for a rejected app version is **5003**
|
||||
(`AppVersionBadCode`), and **5001** for a missing one. That is Proton's own
|
||||
go-proton-api v1.0.4, `response.go:24-25`, still in the module cache. Neither
|
||||
is mapped, so a blocked rclone version gives a plain retried error on every
|
||||
call, without the upgrade hint the spec intends. The fake answers a bad
|
||||
`x-pm-appversion` with 2064 (`CodeInvalidSection`, `protontest/server.go:319`),
|
||||
which the client doesn't map either.
|
||||
|
||||
Fix: map 5001/5003 (and arguably 2064) to the fatal upgrade error. Drop 2000
|
||||
from the fatal list, leaving it a plain (non-fatal) error, or
|
||||
`NoRetryError`. Reconsider 2028: NoRetry rather than Fatal, with Proton's
|
||||
message kept. Update spec 6.6 and the `TestTranslateError` code list, and
|
||||
the `AppVersionRejected` regression test (`task-15`) to use 5003.
|
||||
|
||||
### I-2. Spec 6.6: `x-pm-drive-requirements` response headers are never logged
|
||||
|
||||
Spec `docs/specs/001-protondrive-tier1.md:363`: "The `x-pm-drive-requirements`
|
||||
response headers are logged at debug level so a pending server-side
|
||||
enforcement is visible in bug reports." Nothing in `backend/protondrive`
|
||||
reads the header (grep: no match outside the spec). It is not in
|
||||
`docs/TODO.md`, `DECISIONS.md` or the handoff rulings.
|
||||
|
||||
Failure scenario: Proton announces an upcoming requirement, such as a minimum
|
||||
client version or crypto, through the header. Bug reports with `-vv` don't
|
||||
show it, which is exactly what the spec item exists to prevent.
|
||||
|
||||
Fix: in `session.do` (`session.go:253-255`, next to `observeDate`), log
|
||||
`resp.Header.Values("x-pm-drive-requirements")` at debug level when present,
|
||||
ideally once per distinct value. The alternative is to record a ruling
|
||||
dropping it.
|
||||
|
||||
## Minor
|
||||
|
||||
### M-1. A 5xx or 429 on the token refresh is never retried
|
||||
`session.go:214-216`. `call` returns the original 401 `resp` together with
|
||||
the refresh error. `shouldRetry` (`nodes.go:256`) then takes status 401 from
|
||||
`resp` and falls through to `fserrors.ShouldRetry` on an `*api.Error`, which
|
||||
is false. **Reproduced** (`TestReviewRefresh503NotRetried`): expire the access
|
||||
token and inject one 503 on `/auth/v4/refresh`. `about` fails after 1
|
||||
refresh request. A 429 with Retry-After on refresh is ignored the same way.
|
||||
Every operation in flight at token expiry can fail once (EIO on a mount).
|
||||
Fix: `return nil, refreshErr` so `shouldRetry` reads the status from the
|
||||
wrapped `api.Error` (5xx gets retried, 429 gets the default Retry-After).
|
||||
|
||||
### M-2. A refresh that answers 408 permanently kills the session in-process
|
||||
`session.go:331`. Every 4xx except 429 counts as "refresh token rejected"
|
||||
(`deadRefresh` set, errSessionExpired, fatal), and that includes 408 Request
|
||||
Timeout, a transient status that `shouldRetry` itself retries elsewhere.
|
||||
The SDK precedent covers rejections, not timeouts. Fix: exclude 408 as well.
|
||||
|
||||
### M-3. Two `Fs` in the same process with an env-configured remote can't see each other's refresh
|
||||
`protondrive.go:278-339`, `fs/configmap.go:90-94`. For a remote defined
|
||||
only by `RCLONE_CONFIG_<NAME>_*`, `Save` writes nowhere: the config file
|
||||
has no section. Each `configStore` falls back to its own `saved`, so a
|
||||
second `Fs` (for example `rclone sync remote:a remote:b`) still reads the
|
||||
spent env refresh token. It POSTs that token, is rejected, waits 3 s, and
|
||||
fails fatally with "session expired". `docs/TODO.md` records this for
|
||||
*two processes* only, and the same-process case is worse than recorded.
|
||||
Fix: document it, or share a per-remote-name in-memory overlay. That overlay
|
||||
would be package state, which needs a ruling, so documenting is likely the
|
||||
answer.
|
||||
|
||||
### M-4. Re-sign-in prompt defaults to "no", including when the user follows our own reconnect hint
|
||||
`protondrive.go:581-589` (ruling 17). An expired session's error says
|
||||
`run "rclone config reconnect remote:"`. Credentials are never removed from
|
||||
the store, so reconnect asks "Already signed in - sign in again?" with
|
||||
default No. The statement is false (the session is dead), and pressing Enter
|
||||
does nothing and exits successfully. Fix: reword the prompt ("A session is
|
||||
saved - sign in again?"). Optionally default to yes, as `oauthutil`'s
|
||||
"Token already configured - replace it?" does. That would need the ruling
|
||||
revisited.
|
||||
|
||||
### M-5. After a CAPTCHA in Config, the method question defaults back to password
|
||||
`protondrive.go:630-631`. `ConfigError("method", ...)` returns to `method`,
|
||||
whose default is `!hasPassword`, which is false. Pressing Enter goes through
|
||||
password sign-in again and hits the CAPTCHA again. Fix: go straight to
|
||||
`browser`, or make the default true after a CAPTCHA.
|
||||
|
||||
### M-6. Wrong key password on a saved session: misleading message, no reconnect hint (deferred Task 14, still present)
|
||||
`account.go:123` says "no user key unlocks, is the password right?". This is
|
||||
returned from `connect` unwrapped, without `s.reconnect`. A browser-signed-in
|
||||
remote has no password, and the fix is to reconnect (for example after a
|
||||
password change re-keys the salts). Fix: wrap with the reconnect hint in
|
||||
`connect` when `loadAccount` fails to unlock.
|
||||
|
||||
### M-7. Refresh not proven against the real server; request differs from go-proton-api
|
||||
`api/types.go:73-77`, `session.go:322-326`. The body has no `UID` and no
|
||||
`RedirectURI`, both of which go-proton-api sends
|
||||
(`manager_auth_types.go:93-100`). Handoff section 6 lists no live refresh,
|
||||
and a broken refresh would break every remote after one access-token
|
||||
lifetime. Fix: add "token refresh after expiry" to the live checks in
|
||||
`docs/TODO.md`. If it fails live, add `RedirectURI`.
|
||||
|
||||
### M-8. Deferred Task 2/3/4 session minors, re-checked, all still present and minor
|
||||
- An adopted stored token whose access token is stale fails that call once
|
||||
with a plain 401 (`session.go:308-311` then `:231`).
|
||||
- The refresh HTTP call uses the first waiter's ctx (`session.go:322`).
|
||||
- `serverTime` can step back with out-of-order responses (`session.go:280-289`).
|
||||
- The half-signed-in cleanup DELETE uses the caller's possibly cancelled ctx
|
||||
(`session.go:462-467`). An empty SRP salt is not rejected up front
|
||||
(`session.go:430`).
|
||||
- `publicKeys` waiters get the leader's ctx error (`account.go:288`).
|
||||
`GET /core/v4/users` is fetched in `loginSRP` and again in `loadAccount`.
|
||||
|
||||
### M-9. Own-address public key ring rebuilt on every call
|
||||
`account.go:278-280`. `publicKeyRing` copies every address key (`ToPublic`)
|
||||
on each call. `decryptNode` calls it 2-3 times per node through `verifyKeys`
|
||||
(`nodes.go:494,500,573`), so large listings pay thousands of key copies.
|
||||
Fix: build each address's public ring once in `loadAccount`.
|
||||
|
||||
### M-10. Server-side Copy onto an existing destination always falls back to download and re-upload
|
||||
`protondrive.go:916-918`. `copyNode` refuses an existing name (errExists maps
|
||||
to `fs.ErrorCantCopy`). protondrive has no `PartialUploads`, so rclone copies
|
||||
in place to the existing name. Every server-side copy over a changed file
|
||||
then streams the whole file through the client. It is correct but costly.
|
||||
Fix (later): on errExists, copy to a temporary name and move over, or upload
|
||||
a new revision from the source node. At minimum, record it in
|
||||
`docs/TODO.md`.
|
||||
|
||||
### M-11. Known items re-checked, still minor and recorded
|
||||
- `Precision()` is 1 s while `ModTime` carries milliseconds (handoff 8).
|
||||
- Cross-account Move/Copy/DirMove are not refused (handoff 8). The rclone
|
||||
name check makes it practically unreachable.
|
||||
- The on-the-fly remote hint names the bare backend.
|
||||
- `fs.Debugf(nil, "protondrive: ...")` is used throughout instead of
|
||||
passing the `Fs`. This is style only: maintainers usually prefer the `Fs`
|
||||
as the log object, and it is worth a sweep before any upstream PR.
|
||||
|
||||
## Checked and found sound
|
||||
- Secrets: the password is never sent (SRP) or logged. The key password,
|
||||
tokens and fork key are never in log lines or error strings. The fork URL
|
||||
holding the AES key is printed, not logged. `--dump` behaviour is as
|
||||
documented in `docs/TODO.md`.
|
||||
- Refresh: single-flight per session. It re-reads the store first, and after
|
||||
a rejection it polls up to 3 s for a token rotated elsewhere. It never
|
||||
wipes the store. `Save` writes access before refresh, so a cross-process
|
||||
reader that sees a partial write adopts only after the full set lands.
|
||||
Logout holds `refreshMu` so an in-flight refresh can't resurrect the
|
||||
session.
|
||||
- Config: browser is the default only without username+password (ruling 14).
|
||||
The third-party notice is shown. The old session is signed out only after
|
||||
the new sign-in succeeds (ruling 17). The 2FA prompt loops through
|
||||
`password_2fa`. otp codes use the server clock.
|
||||
- Disconnect/Shutdown match rulings 15 and 17. A call racing Disconnect gets
|
||||
`errNotSignedIn` (`session.go:226-229`), so the Task 14 deferral is closed.
|
||||
- Error-mapping contract: DirNotFound vs ObjectNotFound by op; Mkdir exists
|
||||
is nil; Purge refuses the root by path and by ID; Rmdir asks the server
|
||||
uncached (ruling 16); Remove of a gone file is nil; SHA1-only `Hashes`.
|
||||
- `x-pm-appversion` matches the fake's grammar for releases, betas (build
|
||||
metadata) and dev builds (alpha).
|
||||
- `unlockMu` is the only package-level mutable state. protondrive.go holds
|
||||
no crypto or routes.
|
||||
@@ -0,0 +1,42 @@
|
||||
# Final whole-branch review — shared instructions
|
||||
|
||||
Repository: /home/user/rclone, branch protondrive-tier1, HEAD 29ad26944. Base of the
|
||||
work: 06700aaf2 (`git diff 06700aaf2..HEAD -- <paths>` shows the change; most backend
|
||||
files are new, so read them whole). This is a rewrite of rclone's Proton Drive backend
|
||||
(`backend/protondrive`) as a native Go client following Proton's official SDK, with a
|
||||
fake Proton server in `backend/protondrive/protontest` (fork-only).
|
||||
|
||||
Authorities, in order:
|
||||
1. `docs/specs/001-protondrive-tier1.md` — approved spec (binding).
|
||||
2. `docs/plans/001-protondrive-tier1-HANDOFF.md` section 7 — rulings that supersede
|
||||
the spec/plan where they disagree; section 6 — what has been proven live.
|
||||
3. `docs/DECISIONS.md` (newest first), `docs/TODO.md` (known, deliberately deferred).
|
||||
4. `AGENTS.md` — rclone code, comment and commit rules (comment style is enforced:
|
||||
godoc on every exported identifier, no narrating comments, "why" inline comments).
|
||||
|
||||
Every per-task review deferred its Minor findings to this review. They are in
|
||||
`.superpowers/sdd/001-protondrive-tier1/progress.md` (lines with "minor", "Minor",
|
||||
"Deferred minors") and in the `task-*-review.md` / `task-*-rereview*.md` files in the
|
||||
same directory. Re-check the ones in your area: still present? still minor?
|
||||
|
||||
Your job: find real defects across the whole of your area — correctness, security,
|
||||
data safety, concurrency, resource leaks, error handling, rclone contract violations,
|
||||
spec/ruling non-conformance, upstream-readiness (rclone maintainers' style, minimal
|
||||
cleverness, no package-level mutable state except `unlockMu`, seam rule: no crypto or
|
||||
API routes in protondrive.go, no fs.Fs/fs.Object below it). Do not re-litigate a
|
||||
recorded ruling unless you find it causes a defect the ruling didn't foresee; say so
|
||||
explicitly then. Do not report what docs/TODO.md already records unless it is worse
|
||||
than recorded.
|
||||
|
||||
Grade findings Critical (data loss, security break, crash on normal input) / Important
|
||||
(wrong behaviour a user will hit, contract violation, spec non-conformance, maintainers
|
||||
would reject) / Minor. Each with file:line, a concrete failure scenario, and a suggested
|
||||
fix. Verify before reporting: read the code path end to end; where cheap, prove it with
|
||||
a test in a scratch worktree (`git worktree add --detach
|
||||
/tmp/claude-1000/-home-user-rclone/8cc17642-4f33-4045-ad2c-177bf869b82a/scratchpad/fr-<you> HEAD`,
|
||||
remove with `git worktree remove <path>`, never --force).
|
||||
|
||||
SAFETY: every `go test` MUST set RCLONE_CONFIG=/notfound. Never read
|
||||
~/.config/rclone/rclone.conf or ~/proton_*password; never contact Proton hosts; never
|
||||
pass -remote or -capture-remote. Read-only on /home/user/rclone: do not modify, stage or
|
||||
commit there. Do not dispatch subagents.
|
||||
@@ -0,0 +1,232 @@
|
||||
# Final review: cryptography, node tree, caching (trust model)
|
||||
|
||||
Reviewer area: `backend/protondrive/crypto.go`, `nodes.go`, `cache.go`, the parts of
|
||||
`api/types.go` they use, and the parts of `download.go`, `upload.go` and `account.go`
|
||||
they feed into (contentKey, createRevision, unlockKey, publicKeys). HEAD 29ad26944.
|
||||
SDK reference: ~/.cache/protondrive-sdk-ref @ 05c9115 (read only).
|
||||
|
||||
All Critical and Important findings were reproduced against the fake server in a
|
||||
scratch worktree (now removed). The probes used small scratch-only helpers in
|
||||
`protontest` that model what a hostile server can do with only public keys:
|
||||
encrypt to a node's public key, and drop or replace signatures.
|
||||
|
||||
Counts: Critical 2, Important 3, Minor 8.
|
||||
|
||||
---
|
||||
|
||||
## Critical
|
||||
|
||||
### C1. An unsigned content key packet is accepted. Updates are then encrypted with a key the server chose, and ranged reads return content the server made up
|
||||
|
||||
- `crypto.go:614-620` (`decryptContentKey` checks the signature only
|
||||
`if contentKeyPacketSig != ""`), `download.go:191-227` (`contentKey`),
|
||||
`upload.go:312` (`createRevision` encrypts the new revision's blocks with
|
||||
`c.contentKey(n)`).
|
||||
- Anyone who has the node's public key can make a content key packet: generate a
|
||||
session key and encrypt it to that key. The server stores the armored node key, so it
|
||||
has the public key. If the server sends `ContentKeyPacketSignature: ""` (or null),
|
||||
`verifyErr` stays nil and nothing is logged, even without `ignore_signature_errors`.
|
||||
- **Confidentiality break (reproduced).** A file with a verified node key: the server
|
||||
swaps in its own content key packet and blanks the signature. `Object.Update` (every
|
||||
`sync` of a changed file) goes ahead without an error or a log line and encrypts
|
||||
the user's new content with the server's session key. The probe then decrypted the
|
||||
stored block of the new revision with the server's key and got
|
||||
`"the user's new secret content"` back.
|
||||
- **Integrity break (reproduced).** The server also re-encrypts block 1 under its key
|
||||
(the server lists the hash, and the size matches the recorded block size). A
|
||||
ranged read (`RangeOption{Start:1}`, as mounts and multi-thread copies make)
|
||||
returned `"ORGED BY SERVER"` with no error. A full read failed on the manifest.
|
||||
But a server that also renames the revision's `SignatureEmail` to a foreign
|
||||
address and serves keys for it (the known protocol limit) can sign a manifest
|
||||
over its own block hashes, so full reads can be forged too. docs/TODO.md says a
|
||||
ranged read can be fed "validly encrypted blocks of the same file swapped or from
|
||||
an older revision". This is worse: the content can be anything the server likes.
|
||||
- The SDK also skips verification when the signature is absent
|
||||
(`driveCrypto.decryptAndVerifySessionKey`, `cryptoService.ts:240-249`). But rulings
|
||||
18/19 and spec section 7 put the server in the threat model and are stricter than
|
||||
the SDK on the node key. A key the server can mint is the same hole one level down.
|
||||
- Fix: in `decryptContentKey`, return `errSignatureMissing` as `verifyErr` when the
|
||||
signature is empty, so `contentKey` refuses without the flag. If legacy unsigned
|
||||
content keys exist, allow the exemption only by a creation date, the way names and
|
||||
hash keys do, and say that the date is server-supplied. Add a hostile case to
|
||||
`hostileContents` and a write case ("Update of a file whose content key is
|
||||
unsigned is refused").
|
||||
|
||||
### C2. Node keys from the server are unlocked with whatever Argon2 parameters they carry, up to a 2 TiB allocation
|
||||
|
||||
- `crypto.go:431` (`decryptNodeKeys` → `account.go:204` `unlockKey` →
|
||||
`crypto.Key.Unlock`). go-crypto accepts an Argon2 S2K with memory exponent up to
|
||||
31 (`s2k.go:431`), which is 2^31 KiB, and runs the KDF before it can tell that the
|
||||
passphrase is wrong.
|
||||
- The armored `NodeKey` is server data. For nodes in a folder the user has shared
|
||||
for editing, or in a public upload link folder, it is written by other people's
|
||||
clients. A valid encrypted passphrase is easy to supply: copy any node's, or
|
||||
encrypt any string to the parent's public key. The passphrase signature is checked
|
||||
only after the unlock and only produces `verifyErr`. So listing the folder runs
|
||||
the KDF.
|
||||
- Reproduced (scratch internal test): a node key locked with Argon2 memory 2^19 KiB
|
||||
and `unlockKey` called with a wrong passphrase took 0.47 s and allocated 512 MiB
|
||||
before it returned "error in unlocking key". At exponent 24 to 31 that is 16 GiB
|
||||
to 2 TiB: an OOM kill or `fatal error: runtime: out of memory`, which nothing can
|
||||
recover. Every listing that includes the node crashes the process (rclone mount,
|
||||
sync). `passes` (up to 255) multiplies the CPU cost on top.
|
||||
- Not in docs/TODO.md. The decompression-bomb entries cover message sizes, not key
|
||||
unlocking. The same function unlocks address keys in `account.go`, which is outside
|
||||
this area but has the same cause.
|
||||
- Fix: before `Unlock`, walk the private key packets (`key.GetEntity().PrivateKey`
|
||||
and subkeys, `PrivateKey.S2KParams`/`s2k.Params.Mode()` or by parsing the packet).
|
||||
Refuse a node key whose S2K is not iterated-and-salted. `crypto.go:374-375`
|
||||
already notes that Argon2-locked node keys break the official web app, so no
|
||||
genuine node key uses Argon2. Make it an unreadable node. Optionally bound the
|
||||
RSA modulus size too.
|
||||
|
||||
---
|
||||
|
||||
## Important
|
||||
|
||||
### I1. A hash key whose signature fails or is missing is accepted and used for writes, so the server can test guesses at the names written
|
||||
|
||||
- `nodes.go:550-556` only logs `hashKeyVerifyErr`. `targetFolder` (`nodes.go:964-988`)
|
||||
checks `hashKey != nil` and the key's `unverified`, not the hash key's signature.
|
||||
`createFolder`, upload, rename, move and copy then compute
|
||||
`nameHash(name, parent.hashKey)`.
|
||||
- Reproduced: the server replaced folder `dir`'s hash key with an unsigned one of
|
||||
its own, encrypted to the folder's public key. rclone logged one ERROR, uploaded
|
||||
`dir/tax-return-2026.pdf`, and the stored `Hash` equalled
|
||||
HMAC(server key, "tax-return-2026.pdf"). The server can now test names against a
|
||||
dictionary, or confirm guesses, for everything rclone writes into that folder.
|
||||
Name hashes exist so that it can't.
|
||||
- The SDK folds hash-key and folder-xattr author failures into the node's
|
||||
`keyAuthor` (`cryptoService.ts:265-281`, `finalKeyAuthor`). Under rulings 18/21,
|
||||
rclone's equivalent is `keys.unverified`, but rclone folds in only the passphrase
|
||||
signature. This is a defect the rulings didn't foresee, not a re-litigation of
|
||||
them.
|
||||
- The `hashKeySignedSince` exemption (`nodes.go:551`) is decided by the
|
||||
server-supplied `CreateTime`. A server that back-dates the folder silences even the
|
||||
ERROR line.
|
||||
- Fix: in `targetFolder`, refuse a folder whose hash key signature failed unless
|
||||
`ignore_signature_errors` is set, using the same message shape as an unverified
|
||||
key. The smallest way is to keep `hashKeyVerifyErr` on the node, or fold it into
|
||||
`keys.unverified` for folders. Document that the date exemption depends on a
|
||||
server value.
|
||||
|
||||
### I2. Renaming or moving a node whose name signature failed reuses a session key the server can know, so the server reads the new name
|
||||
|
||||
- `nodes.go:1149-1157` (`rename`) and `nodes.go:1218-1224` (`encryptForNewParent`)
|
||||
reuse `nameSessionKey(n.link.Link.Name, ...)`. The name signature failure is only
|
||||
logged (`nodes.go:505-516`, docs/TODO.md "a name whose signature fails is listed").
|
||||
- Reproduced: the server replaced file `real.txt`'s encrypted name with
|
||||
`"shown.txt"` under a session key it generated, unsigned. rclone listed
|
||||
`shown.txt` with one ERROR. `operations.Move` to `secret-new-name.txt` succeeded,
|
||||
and the server decrypted the new stored name with its own session key:
|
||||
`"secret-new-name.txt"`. Back-dating `CreateTime` before 2021 removes the ERROR
|
||||
line too (`nameSignedSince`).
|
||||
- docs/TODO.md records only that names can be swapped and shown, not that they leak
|
||||
on rename or move. This is a confidentiality loss, not just mislabelling.
|
||||
- Fix: reuse the old name's session key only when the name's signature verified.
|
||||
Otherwise use a new session key (`encryptName(newName, nil, ...)`), or refuse
|
||||
without the flag. Keep `nameVerifyErr` on the node to decide.
|
||||
|
||||
### I3. Extended attributes whose signature is missing or invalid are used without `ignore_signature_errors`
|
||||
|
||||
- `nodes.go:557-580`: the xattr `verifyErr` is only logged and `n.X` is kept. Spec
|
||||
6.4 says `ignore_signature_errors` "downgrades content and authorship signature
|
||||
failures to a logged ERROR", so without it they should fail. The SDK treats a
|
||||
revision's xattr signature as `contentAuthor`.
|
||||
- Anyone with the node's public key can write xattrs. Reproduced: unsigned xattrs
|
||||
claiming `ModificationTime` 2099-01-01. rclone logged one ERROR,
|
||||
`Object.ModTime` returned 2099, and the file read fine. Consequences: the server
|
||||
can set modtime, size and SHA1 at will. For example, `rclone sync --update` to
|
||||
the remote then skips every changed file the server has marked as newer, so a
|
||||
backup silently stops updating. Forged SHA1 or size gives false `check` results
|
||||
and failed copies. Integrity of the content itself holds: block sizes are checked
|
||||
on read.
|
||||
- Not in docs/TODO.md (only names are recorded).
|
||||
- Fix: when the xattr signature fails and `ignore_signature_errors` is not set,
|
||||
drop the attributes (`n.X = nil`, as for undecryptable ones). The file then shows
|
||||
the encrypted size and the server's modtime, and the log says why.
|
||||
|
||||
---
|
||||
|
||||
## Minor
|
||||
|
||||
### M1. A node fetched while it is being changed can overwrite the newer cached node and stay cached
|
||||
`nodes.go:724` `cache.put(n)` after `fetchLinks`. `cache.go:52-63` has no
|
||||
per-node generation, unlike the per-folder change count of ruling 27. Goroutine A
|
||||
starts `fetchNode(X)` (cache miss) and receives revision R1. Goroutine B commits
|
||||
R2 and calls `cache.add(X@R2)`. A then calls `put(X@R1)`, which wins. With no
|
||||
expiry, reads through the cache serve R1's (still stored, obsolete) content until
|
||||
a re-list, and the next Update fails on a stale `CurrentRevisionID`. The window is
|
||||
narrow: concurrent ops on one object in one process, e.g. a mount. Fix: give
|
||||
`put` a token from before the fetch (a global or per-ID change count, bumped by
|
||||
`invalidate` and `add`), as `startFetch`/`endFetch` do.
|
||||
|
||||
### M2. Unbounded per-client maps
|
||||
`client.folderKeys` (every folder ever seen, private keys and passphrases, kept
|
||||
whether or not caching is on; Task 8 minor, still present), the node cache (no
|
||||
size bound or expiry), `client.reported` (one entry per node ID that logged a
|
||||
problem), and `account.foreign` (one key ring per distinct foreign email a server
|
||||
names). A long mount or a hostile server grows them without limit. `close()`
|
||||
still doesn't clear the private keys of file nodes held by the cache (Task 8).
|
||||
|
||||
### M3. `listChildren` loops forever when the server keeps answering `More` with the same `AnchorID`
|
||||
`nodes.go:837-857`. No progress check: a buggy or hostile server keeps rclone
|
||||
requesting until ctx ends, with `seen` and `all` growing if the IDs change. Fix:
|
||||
stop with an error when a page adds no new ID or repeats the anchor.
|
||||
|
||||
### M4. Root-of-trust and signature date exemptions use server-supplied times
|
||||
`nameSignedSince`, `hashKeySignedSince` (`nodes.go:514`, `:551`) and
|
||||
`contentKeyFallbackUntil` (`download.go:204`) read `Link.CreateTime`, which the
|
||||
server controls. Only the first two waive a check. The SDK does the same. Worth a
|
||||
comment and a docs/TODO.md line, as it feeds I1 and I2.
|
||||
|
||||
### M5. `decryptBlock` is used only by tests
|
||||
`crypto.go:684-696`. Production uses `blockDecrypter`. `reencryptPassphrase`
|
||||
(`crypto.go:448-450`) is a one-line alias of `encryptPassphrase`. Maintainers will
|
||||
ask for both to go: move `decryptBlock` to a test file and call
|
||||
`encryptPassphrase` directly.
|
||||
|
||||
### M6. `truncateName` can cut a UTF-8 sequence
|
||||
`nodes.go:599-605` slices bytes at 64, so log lines can carry invalid UTF-8. Cut
|
||||
on a rune boundary.
|
||||
|
||||
### M7. Earlier deferred minors re-checked
|
||||
- Still present: the node passphrase is not checked as valid UTF-8 (Task 5,
|
||||
`crypto.go:429`). `publicKeys` waiters get the fetching caller's ctx error
|
||||
(Task 4, `account.go:288-303`). `setChildrenLocked` overwrites a cached node
|
||||
without the changed-parent rule `put` applies, so after an external move the
|
||||
old folder's cached listing can still return it (Task 9, `cache.go:100-107`;
|
||||
only matters with milestone M5 events or a second `Fs`). An xattr `"Size": null` reads
|
||||
as 0 and skips the block-size fallback (Task 17, `crypto.go:848`; recorded in
|
||||
the handoff). Move's anonymous detection uses `SignatureEmail == ""` (Task 9,
|
||||
`nodes.go:1231`; only reachable with the flag after `checkVerified`).
|
||||
- Closed: `cache.put` leaving a moved node in its old parent's listing (Task 8),
|
||||
and a negative xattr Size with block sizes (Task 17; the fallback now sums the
|
||||
block sizes). The root name not being validated (Task 8) has no effect: it is
|
||||
never used as a path.
|
||||
|
||||
### M8. `publicKeyRing` is rebuilt on every own-address lookup
|
||||
`account.go:278-279`. Each `decryptNode` of a node signed by the user calls
|
||||
`ToPublic` on every address key (up to three lookups per node). Cache the public
|
||||
ring on the address. Performance only.
|
||||
|
||||
---
|
||||
|
||||
## Checked and found sound
|
||||
- Transitive distrust: `decryptNode` inherits `parent.unverified` before its own
|
||||
check. `createFolder` copies the parent's. Uploads into an unverified folder read
|
||||
back through `decryptNode`. `checkVerified` gates rename, move and copy. Root of
|
||||
trust (`bootstrap`): share creator and root signer through `acct.owns`
|
||||
(ASCII-only lowering, `emailKey`), and own addresses are never looked up remotely.
|
||||
- `decryptMessage` decrypts with the session key alone, so holders of decKR can't
|
||||
pass as signers. The intended-recipient check is sound. Decompression is bounded
|
||||
for metadata (`maxSmallMessageSize`, `maxXAttrsSize`) and for blocks
|
||||
(`LimitReader(maxBlockSize+1)` in `fetchOnce`). Integrity checks (block hash,
|
||||
sizes, block count, re-listed hash) do not depend on the flag.
|
||||
- Name validation on read and write: empty, `.`, `..`, `/`, NUL, UTF-8, and 255
|
||||
UTF-16 units (`utf16.RuneLen` is never -1 on valid UTF-8).
|
||||
- Parent chains: cycle detection over `below`, the depth cap of 2048, and a
|
||||
parentless non-root refused ("not in the drive").
|
||||
- Listing cache (ruling 27): `startFetch`/`endFetch` with change counts. `add`,
|
||||
`invalidate` (including an unknown parent, which bumps every listing in
|
||||
progress), `move` (both parents) and `trash` bump correctly.
|
||||
@@ -0,0 +1,212 @@
|
||||
# Final whole-branch review: tests, fake, harness, CI, docs, hygiene
|
||||
|
||||
Reviewer area: everything that is not production backend code. HEAD 29ad26944,
|
||||
base 06700aaf2. Verified in a scratch worktree (removed afterwards).
|
||||
|
||||
## Checks run
|
||||
|
||||
- `RCLONE_CONFIG=/notfound go test -race -count=1 ./backend/protondrive/...` on
|
||||
16 cores: PASS. `backend/protondrive` 333.6 s, `protontest` 38.3 s,
|
||||
`protonfake` 1.4 s.
|
||||
- The same package pinned to 2 CPUs (`taskset -c 0-1`, `-timeout 60m`) to
|
||||
approximate the slow Gitea runner: PASS in 420.7 s (70% of Go's default 10 min test timeout), no failures, timing assertions included.
|
||||
- `golangci-lint run ./backend/protondrive/...`: 0 issues. `go vet`: clean.
|
||||
- `GOPROXY=off go mod tidy -diff`: no diff. go.mod/go.sum: Proton-API-Bridge,
|
||||
go-proton-api, gluon, resty, goquery/cascadia, go-message/vcard, juniper,
|
||||
logrus, relvacode/iso8601 all gone; go-srp and gopenpgp/v3 promoted to direct.
|
||||
Nothing unexpected.
|
||||
- `TestIntegrationFake -v`: 97 subtests pass, 29 skip (matches the handoff).
|
||||
- Test-account usernames (recovered from the pre-rewrite object 901d7fd5b,
|
||||
which is still in the local object store): 0 hits in HEAD's tracked files,
|
||||
text or binary (`git grep -a`). No tokens, UIDs or 32-hex IDs in committed
|
||||
docs/testdata. Largest tracked files outside upstream's MANUAL.* are the three
|
||||
golden CLI blocks (4 MiB, 4 MiB, 1 MiB), as intended.
|
||||
- shellcheck on `init.d/TestProtonDriveFake`: SC2034 only (known minor).
|
||||
- Live-reach audit: every `NewFs`/`configmap.Simple` in the tests carries the
|
||||
fake's `endpoint`, except unit tests of the config store that never connect.
|
||||
`TestIntegration` is gated only by the remote being configured (rclone's norm;
|
||||
hence the standing `RCLONE_CONFIG=/notfound` rule) and skips during a golden
|
||||
capture; `TestGoldenCapture` needs all three `-capture*` flags;
|
||||
`TestIntegrationFake` skips under `-remote`. No other test can reach Proton.
|
||||
|
||||
## Critical
|
||||
|
||||
None.
|
||||
|
||||
## Important
|
||||
|
||||
### I-1. CI: `go test -race` has no `-timeout`; the package will hit Go's default 10 min on the slow runner
|
||||
|
||||
`.gitea/workflows/ci.yml:43` runs `go test -race ./backend/protondrive/...`
|
||||
with Go's default per-binary `-timeout 10m`. The `backend/protondrive` test
|
||||
binary alone takes 333 s on 16 cores under `-race`; pinned to 2 of this host's fast cores it took 420.7 s, 70% of the 10 min budget, so a runner whose cores are ~1.5x slower (shared VM, older CPU, the build cache cold on top) crosses it. The
|
||||
job's `timeout-minutes: 45` never comes into play: the test binary panics with
|
||||
"test timed out after 10m0s" first, and the job fails with a goroutine dump
|
||||
rather than a result. The owner described the runner as "slow and not very
|
||||
capable".
|
||||
|
||||
Fix: `go test -race -timeout 40m ./backend/protondrive/...` (inside the 45 min
|
||||
job budget), or drop `-race` on CI and keep a non-race run (much faster) with a
|
||||
nightly race job.
|
||||
|
||||
### I-2. CI: `go-version-file: go.mod` installs Go 1.26.0, so `govulncheck` will very likely fail on standard-library advisories
|
||||
|
||||
`ci.yml:22-27` takes the Go version from `go.mod`'s `go 1.26.0` directive (no
|
||||
`toolchain` line). `actions/setup-go` treats that as an exact version and
|
||||
installs 1.26.0, a release nine months older than the 1.27.4 every local
|
||||
verification used. `ci.yml:51` then runs `govulncheck@latest` against it;
|
||||
govulncheck reports reachable standard-library vulnerabilities fixed in later
|
||||
patch releases (net/http, crypto/tls, crypto/x509 are reached by this backend)
|
||||
and exits non-zero. So the first push most likely goes red on a finding that is
|
||||
about the CI toolchain, not the code; it also means CI tests on a compiler no
|
||||
one has run the suite with. Could not be proven offline (needs vuln.go.dev).
|
||||
|
||||
Fix: `go-version: '~1.27.3'` (or `stable`), matching upstream's
|
||||
`.github/workflows/build.yml:290` lint job; pin govulncheck to a version
|
||||
instead of `@latest` so a new govulncheck release can't break the build.
|
||||
|
||||
### I-3. The committed workspace copy of the ledger stops at the 2026-11-11 pause; the handoff tells a new agent to restore from it
|
||||
|
||||
`docs/plans/001-protondrive-tier1-HANDOFF.md:96-108` says a copy of every
|
||||
workspace file "is committed at `docs/plans/001-protondrive-tier1-workspace/`"
|
||||
and to resume with `cp docs/plans/001-protondrive-tier1-workspace/*.md
|
||||
.superpowers/sdd/...`; line 25 calls `progress.md` "the full execution ledger
|
||||
(every ruling, every deferred minor, every live probe)". The committed
|
||||
`progress.md` is 175 lines and ends "PAUSED 2026-11-11 ... Tasks 18–21 not
|
||||
started"; the live ledger (gitignored `.superpowers/`) is 244 lines. Missing
|
||||
from git: all Task 18–21 ledger entries (rulings on the lag fix and on
|
||||
draft deletion, the live test_all results, the browser sign-in proof, the
|
||||
config-create token incident), and 21 files: task-18/19/20/21/21f/21g
|
||||
briefs, reports, reviews and re-reviews. On another machine, or after the
|
||||
`.superpowers/` directory is cleaned, a new agent loses the deferred minors of
|
||||
Tasks 18–21 and the reasoning behind rulings the handoff only summarises.
|
||||
|
||||
Fix: re-copy `.superpowers/sdd/001-protondrive-tier1/*.md` into the workspace
|
||||
directory (and do it again after this final review) and commit.
|
||||
|
||||
### I-4. Handoff section 3 still describes the pre-Task-18 state
|
||||
|
||||
A new agent reading top-down is told things that are false now:
|
||||
`HANDOFF.md:3` "Written 2026-11-11 when development was paused"; `:50` "Last
|
||||
code commit: `9c90998f7`" (it is be0053607 / f8e398fce); `:76` verification
|
||||
"at `9c90998f7`"; `:82-91` "Not done, and easy to assume otherwise": user docs
|
||||
"still describe the old backend (Task 20)", wider suites "have never been run
|
||||
against the new backend (Task 18)", "No fork CI exists (Task 18)" — all three
|
||||
are done; `:130` "Remaining tasks" lists 18–21 as remaining; section 8
|
||||
(`:463-466`) says only Tasks 14, 16 and 17 have review files. The table at
|
||||
`:55-57` and section 6 are current, so the document contradicts itself.
|
||||
|
||||
Fix: rewrite section 3's header, commit references and "Not done" list (the
|
||||
real not-done list is: whole-branch review fix wave, 2FA/CAPTCHA/desktop
|
||||
clients live, root of trust on a Proton-made drive, the five cross-Fs tests),
|
||||
and mark the "Remaining tasks"/"Notes for Tasks 18 to 21" subsections as
|
||||
historical or delete them.
|
||||
|
||||
### I-5. The upstream recipe is self-contradictory: the upstream branch would not compile its tests
|
||||
|
||||
Spec 13.4 (`docs/specs/001-protondrive-tier1.md:729-732`) says the upstream
|
||||
branch contains `backend/protondrive/` and the `fstest/test_all/config.yaml`
|
||||
change; spec 14.4, the handoff (`:2`, `:505-511`) and ARCHITECTURE say
|
||||
`backend/protondrive/protontest` never goes upstream. 13 of the 17 test files
|
||||
import `protontest` (all but `crypto_internal_test.go`, `cache_internal_test.go`,
|
||||
`export_test.go`, `protondrive_test.go`), so leaving it out makes
|
||||
`go test ./backend/protondrive/` fail to build upstream, and the
|
||||
`TestProtonDriveFake:` entry in `config.yaml:616-630` refers to an init.d
|
||||
script and port that are also fork-only. Upstream is deferred, so this isn't
|
||||
a defect today, but whoever cuts the branch per the written recipe produces a
|
||||
broken PR, and the handoff (section 9) lists no remedy. Ties to open question
|
||||
14.6.
|
||||
|
||||
Fix: record in the handoff section 9 and spec 13.4 that either protontest goes
|
||||
upstream with the tests (and question 14.6 must be answered first), or the
|
||||
upstream branch drops every protontest-dependent test file plus the
|
||||
`TestProtonDriveFake` config.yaml entry, PORTS.md row and init.d script, and
|
||||
keeps only `protondrive_test.go` and the two standalone internal tests.
|
||||
|
||||
## Minor
|
||||
|
||||
1. **Timing assertions in parallel tests are tight for a slow `-race`
|
||||
runner.** `nodes_internal_test.go:503` and `:523` assert a plain
|
||||
`about()` call finishes in under 500 ms inside `t.Parallel()` tests
|
||||
that run alongside ~120 other parallel tests doing multi-MiB crypto under
|
||||
the race detector; `:487-488` (`TestRetryAfterHonoured`) asserts a 2 s Retry-After call under 5 s.
|
||||
They passed on the 2-CPU run, so this is a margin concern, not an observed flake. Fix: 2-5 s bounds, which still catch a held pacer (the
|
||||
cases guard against a 999999999 s or 60 s hold).
|
||||
2. **Goroutine-count leak checks remain** (`download_internal_test.go:105-118`,
|
||||
`nodes_internal_test.go:582-596`, known). They run in sequential tests, so
|
||||
false failures need a stray goroutine started by something else; the
|
||||
`<= baseline` form can also pass vacuously when goroutines of an earlier
|
||||
test exit during the window. Recorded flake of `TestUploadShortSource` in the
|
||||
handoff. Consider `go.uber.org/goleak`-style filtering is not available
|
||||
without a dependency; keep, but don't expand the pattern.
|
||||
3. **`TestListingRacingChange` can hang to the package timeout**
|
||||
(`nodes_internal_test.go:1222-1228`, known, still present): `<-held` has no
|
||||
timeout and `release` is closed only after `require.NoError(test.change)`;
|
||||
a failure leaves the fake's handler blocked and `httptest.Server.Close`
|
||||
waiting. Fix: `t.Cleanup(func(){ close(release) })` guarded by sync.Once,
|
||||
and a `select` with `time.After` on `held`.
|
||||
4. **`protonfake` `TestRun` leaks the `run` goroutine on a failed require**
|
||||
(`protontest/cmd/protonfake/main_test.go:17-34`, known, still present).
|
||||
Fix: `t.Cleanup` closing `stop`.
|
||||
5. **`TestClientBootstrapCreateVolumeRace` waits a fixed 2 s** (`nodes_internal_test.go:128-147`,
|
||||
known, still present).
|
||||
6. **`TestUploadBlock502Once` doesn't check the fault fired**
|
||||
(`upload_internal_test.go:230-237`): a mistyped pattern would let it pass
|
||||
as a plain upload. Add `assert.Equal(t, 2, srv.Requests()[storage route])`
|
||||
or similar.
|
||||
7. **`TestAcquireTokenCancel` sleeps 20 ms to "let the wait start"**
|
||||
(`upload_internal_test.go:479`): on a slow runner the goroutine may not
|
||||
have started, so the cancelled-while-waiting path is not exercised (the
|
||||
test still passes via the early ctx check). Harmless, not a flake.
|
||||
8. **No test pins the 408 exclusion of `refusedCommit`** (21f re-review N3,
|
||||
still open).
|
||||
9. **Task 20 re-review nits still present**: `download_internal_test.go:621`
|
||||
names the unrecorded case "assumed"; `:178` `min(v.blocks, 1)*3` is always 3.
|
||||
10. **init.d `stop` has no SIGKILL escalation, SC2034 unused `i`**
|
||||
(`fstest/testserver/init.d/TestProtonDriveFake:51-62`, Task 18 minors,
|
||||
still present).
|
||||
11. **CI hygiene**: `on: push` plus `pull_request` with no `concurrency` group
|
||||
runs twice per PR push on a slow runner; `govulncheck@latest` is
|
||||
unpinned (see I-2).
|
||||
12. **The fake's read lag is never combined with the contract suite.**
|
||||
`Server.SetReadLag` (`protontest/drive.go:1749-1757`) models the live
|
||||
read-after-write lag that broke `TestPurge`/vfs on the real server, but only
|
||||
targeted upload tests turn it on; `TestIntegrationFake` and `test_all`
|
||||
against the fake run with no lag, so the next lag-intolerant path
|
||||
(`activeNode`, recorded in TODO) can't show up offline. Consider a second
|
||||
`fstests.Run` (or a `protonfake -read-lag` flag for `test_all`) with a lag
|
||||
of 1-2 reads.
|
||||
13. **Conformance checklist contradicts ruling 11**:
|
||||
`docs/specs/001-conformance-checklist.md:27` (upload row) says "a draft is
|
||||
deleted only on positive proof it never committed"; since be0053607 the
|
||||
rule is "only when the server explicitly refused the commit, on a single
|
||||
unresent attempt; no read-back decides deletion".
|
||||
14. **User docs: the second "unknown" error isn't documented.**
|
||||
`docs/content/protondrive.md:289-293` names only "upload outcome unknown";
|
||||
`errUploadUnconfirmed` ("upload committed, but the server doesn't show it
|
||||
yet", `upload.go:43`) gets the same no-retry treatment and is what a user
|
||||
will see under server lag.
|
||||
15. **User docs claim behaviour proven only against the fake**:
|
||||
CAPTCHA (`protondrive.md:35-37`), 2FA prompting (`:131-147`) and "`rclone
|
||||
about` reports the whole account" (`:548`) are unverified live, while
|
||||
`docs/TODO.md` says the docs "avoid claiming" 2FA and CAPTCHA. Either soften
|
||||
or fix the TODO wording.
|
||||
16. **Lost TODO from the live run**: the ledger's 2026-11-12 incident note
|
||||
("`rclone config create` printed the new remote's client_* tokens and
|
||||
salted key pass to stdout ... check whether client_* options are marked
|
||||
Sensitive") is not in `docs/TODO.md`. The options are `Sensitive`
|
||||
(`protondrive.go:113-142`), but `config create`/`config show` print
|
||||
unredacted (`fs/config/ui.go` `ShowRemote`) — generic rclone behaviour,
|
||||
yet worth a docs line ("the output of `rclone config create` contains the
|
||||
session; don't paste it") and a TODO entry, since the salted key password
|
||||
survives a logout.
|
||||
17. **`docs/PERMISSIONS.md` / `.codex/` out of step**: `golangci-lint run` was
|
||||
added to the Allow tier and to `.claude/` and `.gemini/`, but not to
|
||||
`.codex/rules/project.rules`, against AGENTS.md's "change them together".
|
||||
18. **Old username-bearing commit still in the local object store**
|
||||
(`901d7fd5b`, reachable from the reflog). Not tracked and not on the
|
||||
remote after the force-push; it disappears with reflog expiry/gc. Mention
|
||||
it in the handoff so a `git push --mirror` or a backup bundle of this
|
||||
checkout isn't made before then.
|
||||
19. **Handoff ruling 13** still reads as current ("stays in `go.mod` until
|
||||
Task 19"); mark it done (Task 19 review minor, still present).
|
||||
@@ -0,0 +1,160 @@
|
||||
# Final review: data transfer (upload.go, download.go, Object Open/Update/Put/PutStream)
|
||||
|
||||
Reviewer area: `backend/protondrive/upload.go`, `backend/protondrive/download.go`,
|
||||
`protondrive.go` Object.Open/Update, Fs.Put/PutStream, `api/types.go` as far as these use it.
|
||||
HEAD 29ad26944. `go test -race -run 'TestUpload|TestDownload|TestAcquire|TestIntended'`
|
||||
passes in a scratch worktree (40 s).
|
||||
|
||||
Counts: Critical 0, Important 2, Minor 9.
|
||||
|
||||
## Data-safety walk (no defect found)
|
||||
|
||||
I traced every path that can delete something during an upload:
|
||||
|
||||
- `upload` -> `deleteDraft` runs only when `uploadDraft` reports `committed=false`. That
|
||||
happens before the commit (verification, blocks, `check`, manifest, xattrs) or when
|
||||
`refusedCommit` is true on `attempts == 1` (upload.go:403). `refusedCommit` needs an
|
||||
`*api.Error` with a Proton code and a 4xx status other than 408, so a 200 response with an
|
||||
error envelope, an HTML 4xx and a 5xx all leave the draft. Any pacer resend (5xx, 408, 429,
|
||||
network) makes `attempts > 1`.
|
||||
- The `session.call` 401 -> refresh -> resend happens inside one pacer attempt. The first
|
||||
send got a 401, so the server never processed it. If the refresh then fails with a coded
|
||||
4xx, `refusedCommit` matches the refresh's error and the draft is deleted. That is safe,
|
||||
because the commit was never processed. Go's transport does not replay a PUT unless
|
||||
nothing was written.
|
||||
- When ctx is cancelled, `fserrors.ContextError` in `shouldRetry` turns even a successful
|
||||
commit into a ctx error. The detached single read (upload.go:411-414) then still finds the
|
||||
committed revision.
|
||||
- `resolveConflict` deletes a draft (with `delete_multiple`) in only two cases: the details
|
||||
show a pure draft node (`ConflictRevisionID == ""` and a draft revision) left by our UID,
|
||||
or `replace_existing_draft` is set. It never deletes on a retry. A file conflict becomes
|
||||
a new revision, and a folder, draft or trashed node is refused.
|
||||
- The read back after a commit never decides deletion. Lag gets about 7 s of reads that
|
||||
stop when ctx is done. A commit that is not ours ends as Unconfirmed, Superseded or
|
||||
OutcomeUnknown, and in each case only the cache is forgotten.
|
||||
|
||||
The upload pipeline (readBlocks -> requestURLs -> workers) frees every block on every
|
||||
path: the worker frees after upload, `requestURLs` frees on a URL error or cancel, and
|
||||
`readBlocks` frees on cancel. Both channels are unbuffered, so no block is stranded. Slots
|
||||
and tokens are always given back, and an abandoned `acquireToken` waiter returns its token.
|
||||
The size and SHA1 checks, the early long-source failure, the empty file and the
|
||||
exact-multiple-of-4 MiB case are all correct.
|
||||
|
||||
On the download side: the full read lists every block and checks the manifest before any
|
||||
fetch. Per-block hash and recorded-size checks are bounded by `maxBlocks` and
|
||||
`maxFileBlocks` before `r.sizes[b.Index-1]` is indexed, so that index cannot go out of
|
||||
range. A range with sizes not recorded starts from block 1 with `skip=offset`. A
|
||||
whole-file range goes to `openWhole` after `FixRangeOption`. `Close` drains every slot.
|
||||
|
||||
## Important
|
||||
|
||||
### I-1 `draft_client_uid` is not stable for remotes from environment variables or connection strings, so a leftover draft blocks its name
|
||||
|
||||
protondrive.go:652-657, upload.go:292-297.
|
||||
|
||||
`NewFs` makes a random UID and calls `m.Set`. `fs.setConfigFile.Set` (fs/configmap.go:90-94)
|
||||
saves nothing unless the remote is a section of the config file. A remote set up with
|
||||
`RCLONE_CONFIG_X_*` variables (typical in Docker and CI) or a connection string therefore
|
||||
gets a new UID on every run, and on every `NewFs` in the same process. By design, drafts
|
||||
are now left after every ambiguous commit failure, and also when the process dies
|
||||
mid-upload. On the next run the UID differs, so `resolveConflict` sees a foreign draft and
|
||||
fails with "another client is uploading the file (set replace_existing_draft...)". That
|
||||
repeats on every run for that name until the user sets the option.
|
||||
|
||||
This breaks spec 6.3 step 1 ("a stable per-remote draft client ID") for a common way of
|
||||
running rclone.
|
||||
|
||||
Fix: when the value cannot be saved, derive it deterministically instead of randomly. For
|
||||
example, use a hash of the account's user ID (known after sign-in) plus the remote name, or
|
||||
make the random UID only when `fs.ConfigFileHasSection(name)` and derive it otherwise. At
|
||||
the least, document it and log a NOTICE when the UID isn't persisted.
|
||||
|
||||
### I-2 Multi-thread copies of files that record a size but not block sizes download data quadratically
|
||||
|
||||
download.go:142-151, protondrive.go:1150-1173.
|
||||
|
||||
`docs/TODO.md` records that a ranged read of such a file "downloads from the first block
|
||||
every time" and is slow on a mount. The case below is worse than that entry says.
|
||||
|
||||
`Size()` is known (xattr `Size`), so rclone's multi-thread copy turns on by itself above
|
||||
`--multi-thread-cutoff` (256 MiB) when copying to local. Each 64 MiB chunk is a separate
|
||||
`Open` with a `RangeOption`, and each one downloads and decrypts from block 1 up to the end
|
||||
of its chunk. It also lists every block page to the end of the file, because `last = -1`
|
||||
when sizes aren't recorded. Total transfer is about S²/(2·64 MiB): about 8.5 GiB for a
|
||||
1 GiB file and about 800 GiB for 10 GiB.
|
||||
|
||||
The code supports these files on purpose ("old clients didn't all write 4 MiB blocks"). How
|
||||
many real files lack `BlockSizes` is unknown (every golden capture records them), which is
|
||||
the only reason this isn't higher.
|
||||
|
||||
Fix, cheapest first:
|
||||
1. A small per-client cache of revisionID -> learnt block sizes, filled as a reader decrypts
|
||||
blocks, so later chunks start at the nearest known block. This is the fix the TODO plans,
|
||||
and it should be raised in priority.
|
||||
2. Until then, a ranged read with sizes not recorded and offset > 0 could at least stop the
|
||||
listing once the range is covered (it can't know where, so this is weak).
|
||||
3. Or document that multi-thread copies of such files should use
|
||||
`--multi-thread-streams 0`.
|
||||
|
||||
## Minor
|
||||
|
||||
M-1 **Two processes sharing one config can delete each other's in-progress draft**
|
||||
(upload.go:292-303). The UID is per remote, not per process. If a mount and a cron
|
||||
`rclone move`, or two overlapping runs, upload the same new name, B deletes A's live draft
|
||||
and A's upload fails. If A commits within the RTT between B's 2500 response and B's
|
||||
`delete_multiple`, B sends `delete_multiple` against an active file. What the server does
|
||||
then is the open TODO question. With `move`, A has already deleted its source. The TODO
|
||||
records only the `--retries` path. Add this case there. A per-process suffix on ClientUID,
|
||||
with "own" meaning the remote-wide prefix plus not-this-process, doesn't help. The real
|
||||
mitigation is the TODO's server check, or re-reading the link and confirming it is still a
|
||||
draft just before deleting (this narrows the window but does not close it).
|
||||
|
||||
M-2 **`pm-storage-token` is still sent on redirects to other hosts** (download.go:264-270,
|
||||
upload.go:868; Task 10 minor). Go's client drops only Authorization, Cookie and
|
||||
WWW-Authenticate when a redirect leaves the host. Set `CheckRedirect` on the storage calls,
|
||||
or refuse redirects for them.
|
||||
|
||||
M-3 **The `upload` godoc has a broken reflow** (upload.go:104-109: "...is
|
||||
errUploadSuperseded,\n// and one the server..."). That is the 21f re-review's N2 and it is
|
||||
still present. Rewrap the paragraph.
|
||||
|
||||
M-4 **`errUploadMismatch` is a `NoRetryError`** (protondrive.go:359). A source that changes
|
||||
while it is being read (a growing log file, or a local file still being written) fails for
|
||||
the whole run, including the `--retries` passes, where the usual rclone outcome is a retry
|
||||
at the high level. `NoLowLevelRetryError` would avoid a pointless immediate re-read and still
|
||||
let a later pass succeed.
|
||||
|
||||
M-5 **A range read of a file whose block sizes aren't recorded keeps fetching after the
|
||||
range ends** (download.go:145-151, 338-346). `last` stays -1 when sizes aren't recorded, so
|
||||
up to `downloadAhead` blocks (16 MiB) past the end of the range are downloaded and
|
||||
decrypted before `Close`. This adds to I-2.
|
||||
|
||||
M-6 **An own leftover draft revision on an active file is never deleted** (upload.go:279-290;
|
||||
Task 11 minor, still present). The conflict path uploads a new revision and leaves our old
|
||||
`ConflictDraftRevisionID` taking quota. A `DELETE` of that draft revision when
|
||||
`ConflictDraftClientUID` is ours would be safe: it is a draft revision, not the active one.
|
||||
|
||||
M-7 **Updating an unverified file logs "reading"** (upload.go:312 -> download.go:196; Task
|
||||
16 minor, still present).
|
||||
|
||||
M-8 **The `errUploadSuperseded` message is misleading** when the node reads back as draft
|
||||
or trashed (upload.go:437-438; Task 11 minor, still present). The message says
|
||||
"revision ... is active".
|
||||
|
||||
M-9 **The `createDraft` retry can report its own draft as a foreign one** (upload.go:237-259,
|
||||
294-295). Sequence: own draft deleted, re-POST gets a 5xx, the pacer resends, and the lost
|
||||
first response leaves our new draft, so the resend sees a 2500 with our UID. Because
|
||||
`retried` is set, the client reports "a draft of the file came back after it was deleted".
|
||||
Nothing is lost and the next run replaces the draft. When `ConflictDraftClientUID` is ours
|
||||
on the retried pass, the error could say so instead.
|
||||
|
||||
Still present and already in docs/TODO.md (not re-graded): `Close` waits behind the
|
||||
context-free `downloadTokens.Get`; `countContent` reads the whole file again for every
|
||||
suffix-range open of a file with no recorded size; `--retries` can reach the own-draft
|
||||
`delete_multiple`; unbounded block-list metadata bytes; ranged reads don't check the
|
||||
manifest; no test pins the 408 exclusion in `refusedCommit` (N3; the 408 case in
|
||||
nodes_internal_test.go:428 covers only `shouldRetry`).
|
||||
|
||||
Closed since earlier reviews: `bytes=0-` and whole-file ranges now go through `openWhole`
|
||||
(ruling 26). The read back now stops when ctx is done. Cleanup no longer deletes after a
|
||||
read back, so Task 11's "restore deletes our superseded history entry" can no longer happen.
|
||||
@@ -0,0 +1,168 @@
|
||||
# Flake report: TestClientBootstrapCreateVolumeRace
|
||||
|
||||
Commit: `e23de0b58 protondrive: make the volume creation race test deterministic`
|
||||
(branch `protondrive-tier1`, not pushed).
|
||||
|
||||
## Summary
|
||||
|
||||
- **Root cause.** The test assumed both clients' first `GET drive/v2/shares/my-files` would be handled before either client had created the volume. Nothing guaranteed that. `loadAccount` holds the process-wide `unlockMu` while it unlocks keys, so the second client's first GET always goes out at least one key unlock after the first client's. In a whole-package run, other parallel tests queue on the same mutex, so the gap can be much longer. The old `Delay: 500ms` fault delayed *both* GETs by the same amount, so it did nothing to keep them together. When the first client's create finished before the second client's GET was handled, the second client found the volume and never POSTed. That made `assert.Equal(t, 2, srv.Requests()["POST /drive/volumes"])` fail with `expected: 2, actual: 1`.
|
||||
- **The client is correct.** The losing interleaving ("the volume already exists by the time I look") is the client's ordinary existing-drive path. Only the test's assertion depended on timing.
|
||||
- **Second bug, found by the deterministic test.** Once both create-volume POSTs were forced to run concurrently, the race detector found a genuine data race in the **fake** (`protontest.handleCreateVolume` / `checkVolumeRequest`). go-crypto caches signature-check results inside the key the first time the key is used. The new user's address key had never been used, so two concurrent validations wrote to it at once. It is now fixed: each request is checked with its own copy of the key.
|
||||
|
||||
## 1. Reproduction
|
||||
|
||||
- In isolation it never failed: `-race -count=50` passed, and so did `-count=20 -cpu 1,2`. CPU contention alone does not reproduce it.
|
||||
- It reproduced with **6 concurrent whole-package `-race -v` runs** of a prebuilt test binary: 3 of 6 runs failed (an earlier batch of 6 had 0 failures). All three failures were identical:
|
||||
|
||||
```
|
||||
=== NAME TestClientBootstrapCreateVolumeRace
|
||||
nodes_internal_test.go:139:
|
||||
Error Trace: /home/user/rclone/backend/protondrive/nodes_internal_test.go:139
|
||||
Error: Not equal:
|
||||
expected: 2
|
||||
actual : 1
|
||||
Test: TestClientBootstrapCreateVolumeRace
|
||||
--- FAIL: TestClientBootstrapCreateVolumeRace (8.92s)
|
||||
```
|
||||
|
||||
Line 139 was the `POST /drive/volumes` count assertion. None of those runs logged `WARNING: DATA RACE`.
|
||||
|
||||
## 2. Root cause, with evidence
|
||||
|
||||
I added a temporary copy of the test (since deleted) whose HTTP transport logged the send and completion time of every drive request for each client.
|
||||
|
||||
**In isolation** the margin is tiny. c1's first GET goes out about 130ms after c0's: that is c0's key unlock, serialized by `unlockMu`. c0's create finishes at 792ms, and c1's GET is handled at about 757ms, so only about 35ms separates the passing ordering from the failing one:
|
||||
|
||||
```
|
||||
c0 GET /drive/v2/shares/my-files sent=129ms done=630ms status=422
|
||||
c1 GET /drive/v2/shares/my-files sent=256ms done=757ms status=422
|
||||
c0 POST /drive/volumes sent=685ms done=792ms status=200
|
||||
c0 GET /drive/v2/shares/my-files sent=793ms done=794ms status=200
|
||||
c1 POST /drive/volumes sent=814ms done=814ms status=422 (2500)
|
||||
c1 GET /drive/v2/shares/my-files sent=825ms done=825ms status=200
|
||||
```
|
||||
|
||||
**Under whole-package load** the gap between the first GETs grew to between 240ms and 1.17s (runs that still passed):
|
||||
|
||||
```
|
||||
c1 GET sent=418ms ... c0 GET sent=1.267s (gap 850ms)
|
||||
c0 GET sent=239ms ... c1 GET sent=1.144s (gap 905ms)
|
||||
c1 GET sent=466ms ... c0 GET sent=1.638s (gap 1.17s)
|
||||
```
|
||||
|
||||
When the gap is longer than the winner's "keys + POST + validation" time (about 160ms idle, several hundred ms under load), the second GET is handled after the volume exists. The second client then never POSTs, which is the failure above.
|
||||
|
||||
**Why the gap exists.** `loadAccount` (`account.go`) takes `unlockMu`, a package-level mutex: "unlocks for different sessions run one at a time". So the second client's GET is always at least one unlock behind. In the full package, every parallel test's `newClient` queues on the same mutex, which explains why it only reproduced in whole-package runs.
|
||||
|
||||
**Candidates from the brief:**
|
||||
|
||||
| Candidate | Verdict |
|
||||
|---|---|
|
||||
| Loser re-reads my-files before the winner's volume is visible | Ruled out. The fake answers 2500 only under `s.mu` with `u.drive != nil`, and the trace shows the loser's re-GET always returned 200. |
|
||||
| Timing assumption in the test (Delay vs real ordering) | **Root cause.** The Delay shifted both GETs equally and could not keep them together. |
|
||||
| Fake's create handler validating outside the lock, interleaving badly | Not the flake's cause; the client handles both 2500 paths identically. But it hid a real **data race in the fake**, exposed once the POSTs were forced to run together (section 3b). |
|
||||
| Pacer / Retry-After | Ruled out. No 429 or 5xx appears in the traces; 4xx is not retried. |
|
||||
| A data race reported in the original flake | None in the failing runs. A race appeared only after the POSTs were made concurrent, and it is in the fake. |
|
||||
|
||||
**The production path is correct for every two-client interleaving:**
|
||||
1. Both see 2501 and both POST. One gets 200; the other gets 2500, which `createVolume` treats as success, then re-reads my-files and finds the volume, because a 2500 means it exists.
|
||||
2. The second client looks after the first has created the volume. It gets 200 straight away: the normal existing-drive path, with no POST.
|
||||
3. The winner's POST commits but the response is lost or 5xx. The pacer retries, gets 2500, treats it as success, then re-reads. Correct.
|
||||
|
||||
The only residual assumption is that a real server's 2500 implies the volume is immediately readable through my-files (see Concerns).
|
||||
|
||||
## 3. Fix
|
||||
|
||||
### a. Make the interleaving deterministic (test and fake)
|
||||
- `protontest.Fault` gains a `Gather bool` option: "hold each faulted request until all Times have arrived". `Inject` panics if `Gather` is set without `Times`. A held request still ends when the client gives up, so a stuck test cannot hang.
|
||||
- `TestClientBootstrapCreateVolumeRace` now:
|
||||
- gathers both `GET my-files` requests, so both clients see 2501 whatever their timing;
|
||||
- gathers both `POST /drive/volumes` requests, so both creations are validated at once and the loser hits the fake's late "already exists" check;
|
||||
- starts the second client as late as the server allows: after the first client finishes, or after 2s if the server is holding the first client back for it. This models a long `unlockMu` queue;
|
||||
- runs under a context with a one-minute timeout.
|
||||
- A new `TestFaultInjection/Gather` subtest checks that the first request is not answered until the second arrives, and that requests after `Times` are not held.
|
||||
|
||||
### b. Data race in the fake (found by 3a)
|
||||
With the POSTs gathered, 3 of about 180 focused runs failed with `race detected during execution of test`. Every race stack was entirely in the fake:
|
||||
|
||||
```
|
||||
WARNING: DATA RACE
|
||||
Write at 0x00c0000f00f0 by goroutine 42:
|
||||
github.com/ProtonMail/go-crypto/openpgp/v2.(*Identity).LatestValidSelfCertification() user.go:212
|
||||
...(*Entity).DecryptionKeys() / ReadMessage() / gopenpgp Decrypt()
|
||||
protontest.decryptArmored() protontest/crypto.go:239
|
||||
protontest.unlockDriveKey() protontest/crypto.go:280
|
||||
protontest.checkVolumeRequest() protontest/drive.go:1677
|
||||
protontest.(*Server).handleCreateVolume() protontest/drive.go:1656
|
||||
Previous read at 0x00c0000f00f0 by goroutine 32:
|
||||
(same stack, the other create-volume request)
|
||||
```
|
||||
|
||||
go-crypto's `LatestValidSelfCertification` caches `sig.Valid` inside the key on first use. A user created without a drive has never used its address key, so two concurrent `checkVolumeRequest` calls both did that first write. Other handlers share keys that setup has already used, so they only ever read. The fix is in `handleCreateVolume`: validate with `addrKey.key.Copy()`, the key's own copy for that request. After this, 200 focused runs and 12 whole-package runs reported 0 races.
|
||||
|
||||
## 4. Before / after (the forced interleaving)
|
||||
|
||||
**Before**: the final test shape with the old `Delay: 500ms` GET fault in place of the Gather faults, at `-race -count=10`:
|
||||
|
||||
```
|
||||
--- FAIL: TestClientBootstrapCreateVolumeRace (2.30s)
|
||||
nodes_internal_test.go:153:
|
||||
Error: Not equal:
|
||||
expected: 2
|
||||
actual : 1
|
||||
...
|
||||
fails: 10/10
|
||||
```
|
||||
|
||||
**After** (Gather faults, before the key-copy fix): 3 of about 180 runs failed with the fake data race in 3b. That run also hit go test's 10-minute default timeout, because each run costs about 2.3s by design.
|
||||
|
||||
**After** (committed tree): `RCLONE_CONFIG=/notfound go test -race -timeout 40m -count=200 -run 'TestClientBootstrapCreateVolumeRace$' ./backend/protondrive/`
|
||||
|
||||
```
|
||||
ok github.com/rclone/rclone/backend/protondrive 610.471s
|
||||
FAIL lines: 0 races: 0
|
||||
```
|
||||
|
||||
`TestFaultInjection/Gather`: 30 of 30 passed at `-race -count=30`.
|
||||
|
||||
## 5. Verification
|
||||
|
||||
- Focused test: `-race -count=200` passed, with 0 races (above).
|
||||
- Whole package, sequential, on commit e23de0b58: `RCLONE_CONFIG=/notfound go test -race -count=1 -v ./backend/protondrive/...` six times in a row:
|
||||
|
||||
```
|
||||
run 1 exit=0 protondrive 95.1s ok, protontest 32.6s ok fails=0 races=0
|
||||
run 2 exit=0 protondrive 98.3s ok, protontest 33.3s ok fails=0 races=0
|
||||
run 3 exit=0 protondrive 91.7s ok, protontest 32.4s ok fails=0 races=0
|
||||
run 4 exit=0 protondrive 90.4s ok, protontest 32.5s ok fails=0 races=0
|
||||
run 5 exit=0 protondrive 92.9s ok, protontest 31.9s ok fails=0 races=0
|
||||
run 6 exit=0 protondrive 91.8s ok, protontest 32.4s ok fails=0 races=0
|
||||
```
|
||||
|
||||
- Whole package under contention, the same setup that reproduced the bug 3 of 6 times: 6 concurrent `-race -v` runs of the committed binary all passed, with 0 races. The volume test took 12 to 14s per run.
|
||||
- `go build ./...`, `go vet ./backend/protondrive/...` and `golangci-lint run ./backend/protondrive/...` (0 issues) are all clean.
|
||||
- A `protontest` run at `-race -count=20` "FAIL"ed only on go test's 10-minute default timeout. One package run takes 33s, so 20 runs need about 660s. This is not a hang: individual runs and the Gather subtest at count=30 all pass.
|
||||
|
||||
## Other intermittent failure seen (not fixed, out of scope)
|
||||
|
||||
`TestUploadShortSource` failed once, in one of the six concurrent whole-package runs of the first loaded batch (the one that also reproduced the volume flake):
|
||||
|
||||
```
|
||||
=== RUN TestUploadShortSource
|
||||
upload_internal_test.go:196:
|
||||
Error Trace: /home/user/rclone/backend/protondrive/download_internal_test.go:114
|
||||
/home/user/rclone/backend/protondrive/upload_internal_test.go:196
|
||||
/home/user/rclone/backend/protondrive/upload_internal_test.go:201
|
||||
Error: Condition never satisfied
|
||||
Test: TestUploadShortSource
|
||||
Messages: goroutines left running
|
||||
--- FAIL: TestUploadShortSource (4.14s)
|
||||
```
|
||||
|
||||
`requireNoLeaks` waits up to 2s for the **process-wide** `runtime.NumGoroutine()` to return to its baseline. The test is not `t.Parallel()`, so it runs in the sequential phase. On a heavily loaded machine (six race-enabled package runs at once) 2s may not be enough for server and transport goroutines to wind down. A real leak is not ruled out. It did not recur in the 12 later whole-package runs (6 sequential and 6 concurrent).
|
||||
|
||||
## Concerns
|
||||
|
||||
- The test now costs about 2.3s, which is the deliberate 2s wait. It runs in parallel with the rest of the package, so wall-clock impact is small.
|
||||
- The client assumes a real server's 2500 on `POST /drive/volumes` means the volume can be read at once through my-files. If Proton's API were eventually consistent there, the losing client would fail with a clear `get drive: ... 2501` error rather than corrupt anything. I did not add a retry for this unverified case.
|
||||
- go-crypto writes into a key on first use. In the client, the first use of each folder key happens on one goroutine in `decryptNode`, which decrypts the hash key before any concurrent `decryptChildren`. So the listing path is not exposed, and no client-side race appeared in any run. Any future code that uses a *freshly unlocked* key from several goroutines at once would hit the same race as the fake did.
|
||||
Loaded 100 of 174 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user