protondrive: check a file's signed block list before handing out any of it

A full read verified the manifest signature only at the end, so blocks
a server swapped, dropped or took from an older revision reached the
reader - and rclone cat or a mount - before the error. A full read now
lists every block and verifies the manifest before releasing the first
byte, and a block listed again with a different hash is an integrity
error. Ranged reads are unchanged.

The fake's "rollback" corruption is renamed "block-rollback": serving a
whole earlier revision under the current revision ID can't be detected,
which the spec and TODO now say.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Lucas WintherandClaude Opus 5.5 committed 2026-11-11 18:51:15 +01:00
1 parent 88654eb04d
commit 9598e22d90
9 files changed
+135 -53

No files matched your search

+32 -10
View File
@@ -71,11 +71,12 @@ const (
// open returns a reader of limit bytes of the content of file n from
// offset, or to the end if limit < 0.
//
// A read from offset 0 to the end checks the manifest signature last:
// if it fails, the final Read returns errIntegrity, unless
// IgnoreSignatureErrors is set, when it is logged. A node key or
// content key signature which fails is errIntegrity from open, on the
// same terms.
// A read from offset 0 to the end lists every block and checks the
// manifest signature before releasing any byte: if it fails, the first
// Read returns errIntegrity, unless IgnoreSignatureErrors is set, when
// it is logged. A ranged read checks each block against its listed
// hash but can't check the manifest. A node key or content key
// signature which fails is errIntegrity from open, on the same terms.
// The reader must be closed.
func (c *client) open(ctx context.Context, n *node, offset, limit int64) (io.ReadCloser, error) {
if n.IsFolder || n.link == nil || n.link.File == nil || n.RevisionID == "" {
@@ -309,8 +310,9 @@ type blockReader struct {
}
// list lists the blocks first to last (to the end if last < 0) and
// fetches them, then sends the end of the download. If verify is set it
// checks the manifest, the list having started at the first block.
// fetches them, then sends the end of the download. If verify is set,
// the list having started at the first block, it lists every block and
// checks the manifest before fetching any.
//
// A list of more than maxBlocks blocks is an error.
func (r *blockReader) list(first, last, maxBlocks int, verify bool) {
@@ -323,10 +325,17 @@ func (r *blockReader) list(first, last, maxBlocks int, verify bool) {
}
}
// listedBlock is a block of the list with its decoded hash.
type listedBlock struct {
block api.Block
hash []byte
}
// listBlocks does the work of list, returning the verdict on the file.
func (r *blockReader) listBlocks(first, last, maxBlocks int, verify bool) error {
var (
hashes [][]byte
listed []listedBlock // blocks waiting for the manifest check, if verify
manifest string
signer string
index = first
@@ -370,9 +379,11 @@ func (r *blockReader) listBlocks(first, last, maxBlocks int, verify bool) error
return fmt.Errorf("block %d: %w", index, err)
}
if verify {
// Nothing is released before the manifest vouches for
// every block's hash and place.
hashes = append(hashes, hash)
}
if !r.fetch(b, hash) {
listed = append(listed, listedBlock{block: b, hash: hash})
} else if !r.fetch(b, hash) {
return r.ctx.Err()
}
if index == last {
@@ -387,7 +398,15 @@ func (r *blockReader) listBlocks(first, last, maxBlocks int, verify bool) error
if !verify {
return nil
}
return r.verifyManifest(hashes, manifest, signer)
if err := r.verifyManifest(hashes, manifest, signer); err != nil {
return err
}
for _, lb := range listed {
if !r.fetch(lb.block, lb.hash) {
return r.ctx.Err()
}
}
return nil
}
// decodeBlockHash returns the base64 SHA-256 hash.
@@ -466,6 +485,9 @@ func (r *blockReader) fetchBlock(b api.Block, hash []byte) blockResult {
if b, err = r.c.blockURL(r.ctx, r.n, index); err != nil {
return blockResult{index: index, err: fmt.Errorf("block %d: %w", index, err)}
}
if relisted, err := decodeBlockHash(b.Hash); err != nil || !bytes.Equal(relisted, hash) {
return blockResult{index: index, err: fmt.Errorf("block %d: %w: listed again with another hash", index, errIntegrity)}
}
}
}
+24 -1
View File
@@ -305,7 +305,7 @@ func TestDownloadBadManifest(t *testing.T) {
data, err := readRange(t, c, n, 0, -1)
require.ErrorIs(t, err, errIntegrity)
assert.Contains(t, err.Error(), "ignore_signature_errors")
assert.True(t, bytes.Equal(content, data), "the bytes come before the verdict")
assert.Empty(t, data, "no byte comes before the verdict")
// A ranged read can't check the manifest.
data, err = readRange(t, c, n, 0, 100)
@@ -350,6 +350,29 @@ func TestDownloadStorage404Refetch(t *testing.T) {
assert.Equal(t, 2, srv.Requests()[storagePattern])
}
func TestDownloadRefetchOtherHash(t *testing.T) {
t.Parallel()
c, srv, u := newTestClient(t)
n := seedFile(t, c, u, "f", testContent(1<<20))
// The block's URL expires, and the server lists it again with
// another block, which the checked manifest doesn't cover.
srv.Inject(protontest.Fault{Pattern: "/storage/blocks/{id}", Status: http.StatusNotFound, Times: 1, Delay: 300 * time.Millisecond})
done := make(chan struct{})
var data []byte
var err error
go func() {
defer close(done)
data, err = readRange(t, c, n, 0, -1)
}()
require.Eventually(t, func() bool { return srv.Requests()[storagePattern] == 1 }, 5*time.Second, 5*time.Millisecond)
u.Corrupt(n.ID, "foreign-block")
<-done
require.ErrorIs(t, err, errIntegrity)
assert.Contains(t, err.Error(), "listed again with another hash")
assert.Empty(t, data)
}
func TestDownloadRetries5xx(t *testing.T) {
t.Parallel()
c, srv, u := newTestClient(t)
+6 -4
View File
@@ -791,9 +791,11 @@ func (u *User) insertLink(parentID string, l *link) string {
// - "swap-blocks": the first two blocks trade places, each still
// matching its own hash
// - "truncate-blocks": the last block is dropped from the revision
// - "rollback": the active revision serves the blocks of the revision
// before it, which SeedRevision makes, under its own ID, manifest
// signature and extended attributes
// - "block-rollback": the active revision serves the blocks of the
// revision before it, which SeedRevision makes, under its own ID,
// manifest signature and extended attributes. (A server serving the
// earlier revision whole under the active revision's ID can't be
// detected, so isn't modelled.)
// - "foreign-block": the first block is the same plaintext encrypted
// with another content key, with its hash and block signature to
// match
@@ -844,7 +846,7 @@ func (u *User) Corrupt(id string, what string) {
needBlocks(r, 1)
r.blocks = r.blocks[:len(r.blocks)-1]
})
case "rollback":
case "block-rollback":
var previous *revision
for _, r := range l.revisions {
if r.state == revisionStateObsolete {
+2 -2
View File
@@ -197,12 +197,12 @@ func TestSeedFormats(t *testing.T) {
}
func TestCorrupt(t *testing.T) {
for _, what := range []string{"name", "name-signature", "passphrase-signature", "block-hash", "manifest", "swap-blocks", "truncate-blocks", "rollback", "foreign-block"} {
for _, what := range []string{"name", "name-signature", "passphrase-signature", "block-hash", "manifest", "swap-blocks", "truncate-blocks", "block-rollback", "foreign-block"} {
t.Run(what, func(t *testing.T) {
s := New(t)
u := s.AddUser("user", "pass")
victim := u.SeedFile(u.RootID(), "victim", patterned(blockSize+10), time.Now())
if what == "rollback" {
if what == "block-rollback" {
u.SeedRevision(victim, patterned(blockSize+20), time.Now())
}
bystander := u.SeedFile(u.RootID(), "bystander", []byte("fine"), time.Now())
+14 -16
View File
@@ -81,9 +81,6 @@ type hostileContent struct {
// signatureOnly is set if only a signature check catches the
// tampering, which ignore_signature_errors turns off.
signatureOnly bool
// released is how many bytes the reader may hand out before its
// error: those of the blocks read before the check which fails.
released int
}
// seedCorrupted returns a prepare which seeds the file and applies the
@@ -98,12 +95,17 @@ func seedCorrupted(what string) func(u *protontest.User, content []byte) string
// seedRolledBack returns a prepare which seeds the file with old
// content, gives it a new revision with the content, then serves the
// old revision's blocks under the new revision.
// old revision's blocks under the new revision's manifest.
//
// A server serving the old revision whole - blocks, manifest and
// extended attributes - under the new revision's ID can't be detected:
// nothing signed binds them to the revision. The SDK can't detect it
// either.
func seedRolledBack(old []byte) func(u *protontest.User, content []byte) string {
return func(u *protontest.User, content []byte) string {
id := u.SeedFile(u.RootID(), "victim.bin", old, time.Now())
u.SeedRevision(id, content, time.Now())
u.Corrupt(id, "rollback")
u.Corrupt(id, "block-rollback")
return id
}
}
@@ -118,24 +120,21 @@ var hostileContents = []hostileContent{{
content: patternedContent(2*blockSize+10, 2),
prepare: seedCorrupted("swap-blocks"),
signatureOnly: true,
released: 2*blockSize + 10,
}, {
name: "swapped blocks of different sizes",
content: patternedContent(blockSize+10, 3),
prepare: seedCorrupted("swap-blocks"),
}, {
name: "truncated block list",
content: patternedContent(blockSize+10, 4),
prepare: seedCorrupted("truncate-blocks"),
released: blockSize,
name: "truncated block list",
content: patternedContent(blockSize+10, 4),
prepare: seedCorrupted("truncate-blocks"),
}, {
name: "older revision content under the current revision ID",
name: "older revision's blocks under the current revision",
content: patternedContent(blockSize+10, 5),
prepare: seedRolledBack(patternedContent(blockSize+10, 6)),
signatureOnly: true,
released: blockSize + 10,
}, {
name: "older revision content of another size under the current revision ID",
name: "older revision's blocks of another size under the current revision",
content: patternedContent(20, 7),
prepare: seedRolledBack(patternedContent(10, 8)),
}, {
@@ -190,8 +189,7 @@ func TestHostileServer(t *testing.T) {
require.NoError(t, err, "the file is listed")
got, err := readAll(ctx, o)
require.Error(t, err, "read without error")
assert.LessOrEqual(t, len(got), tc.released, "bytes released before the error")
assert.False(t, bytes.Equal(got, tc.content), "the content was returned")
assert.Empty(t, got, "bytes released before the error")
requireCopyFails(ctx, t, f, path)
// Turning signature checks off leaves the integrity
@@ -202,7 +200,7 @@ func TestHostileServer(t *testing.T) {
got, err = readAll(ctx, o)
if !tc.signatureOnly {
require.Error(t, err, "read without error with signature checks off")
assert.LessOrEqual(t, len(got), tc.released, "bytes released before the error")
assert.Empty(t, got, "bytes released before the error")
}
if bytes.Equal(got, tc.content) {
// Only a signature was tampered with, and the content
+21
View File
@@ -5,6 +5,27 @@ it with a new one that links back. One entry per decision a future maintainer
would otherwise have to reverse-engineer.
No index: `grep -n '^## ' docs/DECISIONS.md` lists every entry.
## 2026-11-11 - Proton Drive verifies the manifest before releasing a full read
**Status:** accepted
**Context:** A full read checked the manifest signature at EOF, as the
SDK does, so blocks swapped, truncated or from an older revision, each
matching its listed hash and recorded size, reached the reader before
the error: `rclone cat` and streaming mounts showed wrong bytes (spec
9.6 asks for "an integrity error, not data").
**Decision:** A full read lists every block page first, verifies the
manifest over the listed hashes, then fetches; each block is checked
against its listed hash, and a block listed again with another hash
is an integrity error. Pages are fetched one after another. Ranged
reads keep checking blocks against their listed hashes only.
**Alternatives:** Keeping the SDK's EOF check; verifying ranged reads
too by listing the whole file on each ranged open (costly for mounts
and multi-thread copies without a hash cache).
**Consequences:** No extra requests, but a file of more than 20 blocks
waits for its whole list before the first byte. Block metadata, not
data, is held meanwhile. A whole earlier revision served under the
current ID stays undetectable (protocol limit).
## 2026-11-11 - Proton Drive distrusts what a forged folder key vouches for
**Status:** accepted
+15 -10
View File
@@ -8,16 +8,21 @@ Small, specific, independently actionable. Remove only when actually done.
## Next
- [ ] protondrive: a full read checks the manifest signature only at EOF
(spec 6.4, as the SDK does), so a hostile server serving blocks
swapped, or an older revision's blocks, of the recorded sizes gets
that wrong plaintext handed to the reader before the final `Read`
fails (`TestHostileServer` "swapped blocks", "older revision
content"). `operations.Copy` discards it, but `rclone cat` and a
streaming mount don't. Consider listing every block page and checking
the manifest before releasing the first byte, at least when the block
list fits in one page; with `ignore_signature_errors` only the SHA1
check of a copy catches these.
- [ ] protondrive: a full read lists every block page before releasing
the first byte, one request per 20 blocks in sequence, so a very large
file waits (about 12 requests for 1 GiB, about 1280 for 100 GiB) before
data flows. Fetching pages in parallel under the 15-request metadata
limit would cut that.
- [ ] protondrive: a ranged read (offset > 0 or a limit, as multi-thread
copies and mounts make) checks each block against the hash the server
lists for it, its decryption and its recorded size, but not the
manifest, so a server can serve validly encrypted blocks of the same
file swapped or from an older revision of the same size. The planned
fix is a bounded per-revision cache of manifest-verified block hashes.
- [ ] protondrive: a server can serve an earlier revision whole (blocks,
manifest and extended attributes, all validly signed) under the current
revision ID undetected, as nothing signed binds them to the revision;
the SDK can't detect it either. Raise with Proton.
- [ ] protondrive: `--dump headers` logs Proton's `pm-storage-token` and
`x-pm-uid` (fshttp redacts only `Authorization`), and `--dump bodies`
logs the access, refresh and upload tokens in JSON bodies (seen in
+2 -2
View File
@@ -573,8 +573,8 @@ library what must stay readable.
`func (u *User) Corrupt(id string, what string)` with `what` one of
`"name"`, `"name-signature"`, `"passphrase-signature"`, `"block-hash"`,
`"manifest"`, `"swap-blocks"`, `"truncate-blocks"`, and (Task 16)
`"rollback"` (the active revision serves the previous revision's
blocks) and `"foreign-block"` (the first block encrypted with another
`"block-rollback"` (the active revision serves the previous
revision's blocks) and `"foreign-block"` (the first block encrypted with another
content key).
- Secrets for log hygiene tests (Task 16):
`func (s *Server) IssuedSecrets() []string` (every access and refresh
+19 -8
View File
@@ -293,11 +293,19 @@ AES-GCM blocks; names, passphrases, hash keys and attributes without AEAD).
- Ranged reads map an offset to a block using the recorded block sizes, so
`NoMultiThreading` is dropped and multi-thread downloads work. At most 10
blocks are in flight for download across the `Fs` (SDK value).
- A full, sequential read verifies the manifest signature at EOF. Failure is
an error. An advanced option `ignore_signature_errors` downgrades content
and authorship signature failures to a logged `ERROR` for users with
legacy or shared data (#8003, #7344). **(call)** Ranged reads cannot verify
the manifest; this is documented, as it is in the SDK.
- A full, sequential read lists every block and verifies the manifest
signature over the listed hashes before releasing the first byte; each
block is then checked against its listed hash as it arrives, and a block
listed again (for a new URL) with another hash is an integrity error.
Failure is an error. An advanced option `ignore_signature_errors`
downgrades content and authorship signature failures to a logged `ERROR`
for users with legacy or shared data (#8003, #7344). **(call)** Ranged
reads check each block against its listed hash but cannot verify the
manifest; this is documented, as it is in the SDK.
- A server serving an earlier revision whole (blocks, manifest and
extended attributes, all validly signed) under the current revision ID is
undetectable: nothing signed binds them to the revision. The SDK cannot
detect it either; it is a protocol limit to raise with Proton.
- Old-format files (pre-AEAD) remain readable.
- `Close` releases everything; no reader outlives its `Object.Open` caller.
@@ -535,9 +543,12 @@ CI, long runs are manual.
- **Hostile server.** The fake returns: a block with a wrong hash; blocks
swapped or truncated; a block list shorter than the manifest; an older
revision's content under the current revision ID; a name decrypting to
`../x`, `a/b` or containing NUL; a 10 MiB "name"; an attacker-signed
passphrase. Each must produce an integrity error, not data.
revision's blocks under the current revision; a block encrypted with
another key; a name decrypting to `../x`, `a/b` or containing NUL; a 10
MiB "name"; an attacker-signed passphrase; a folder with a server-made
key and anonymous files beneath it. Each must produce an integrity error,
not data: a full read releases no byte before the error. (A whole earlier
revision served under the current ID is undetectable; see 6.4.)
- **Secret hygiene.** A test runs login, upload and download at `-vv`
(without `--dump`), then searches the captured log for the known password,
key password, tokens, session keys and upload tokens. A second test checks