protondrive: check a file's signed block list before handing out any of it
A full read verified the manifest signature only at the end, so blocks a server swapped, dropped or took from an older revision reached the reader - and rclone cat or a mount - before the error. A full read now lists every block and verifies the manifest before releasing the first byte, and a block listed again with a different hash is an integrity error. Ranged reads are unchanged. The fake's "rollback" corruption is renamed "block-rollback": serving a whole earlier revision under the current revision ID can't be detected, which the spec and TODO now say. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
88654eb04d
commit
9598e22d90
9 files changed
+135
-53
No files matched your search
@@ -71,11 +71,12 @@ const (
|
||||
// open returns a reader of limit bytes of the content of file n from
|
||||
// offset, or to the end if limit < 0.
|
||||
//
|
||||
// A read from offset 0 to the end checks the manifest signature last:
|
||||
// if it fails, the final Read returns errIntegrity, unless
|
||||
// IgnoreSignatureErrors is set, when it is logged. A node key or
|
||||
// content key signature which fails is errIntegrity from open, on the
|
||||
// same terms.
|
||||
// A read from offset 0 to the end lists every block and checks the
|
||||
// manifest signature before releasing any byte: if it fails, the first
|
||||
// Read returns errIntegrity, unless IgnoreSignatureErrors is set, when
|
||||
// it is logged. A ranged read checks each block against its listed
|
||||
// hash but can't check the manifest. A node key or content key
|
||||
// signature which fails is errIntegrity from open, on the same terms.
|
||||
// The reader must be closed.
|
||||
func (c *client) open(ctx context.Context, n *node, offset, limit int64) (io.ReadCloser, error) {
|
||||
if n.IsFolder || n.link == nil || n.link.File == nil || n.RevisionID == "" {
|
||||
@@ -309,8 +310,9 @@ type blockReader struct {
|
||||
}
|
||||
|
||||
// list lists the blocks first to last (to the end if last < 0) and
|
||||
// fetches them, then sends the end of the download. If verify is set it
|
||||
// checks the manifest, the list having started at the first block.
|
||||
// fetches them, then sends the end of the download. If verify is set,
|
||||
// the list having started at the first block, it lists every block and
|
||||
// checks the manifest before fetching any.
|
||||
//
|
||||
// A list of more than maxBlocks blocks is an error.
|
||||
func (r *blockReader) list(first, last, maxBlocks int, verify bool) {
|
||||
@@ -323,10 +325,17 @@ func (r *blockReader) list(first, last, maxBlocks int, verify bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// listedBlock is a block of the list with its decoded hash.
|
||||
type listedBlock struct {
|
||||
block api.Block
|
||||
hash []byte
|
||||
}
|
||||
|
||||
// listBlocks does the work of list, returning the verdict on the file.
|
||||
func (r *blockReader) listBlocks(first, last, maxBlocks int, verify bool) error {
|
||||
var (
|
||||
hashes [][]byte
|
||||
listed []listedBlock // blocks waiting for the manifest check, if verify
|
||||
manifest string
|
||||
signer string
|
||||
index = first
|
||||
@@ -370,9 +379,11 @@ func (r *blockReader) listBlocks(first, last, maxBlocks int, verify bool) error
|
||||
return fmt.Errorf("block %d: %w", index, err)
|
||||
}
|
||||
if verify {
|
||||
// Nothing is released before the manifest vouches for
|
||||
// every block's hash and place.
|
||||
hashes = append(hashes, hash)
|
||||
}
|
||||
if !r.fetch(b, hash) {
|
||||
listed = append(listed, listedBlock{block: b, hash: hash})
|
||||
} else if !r.fetch(b, hash) {
|
||||
return r.ctx.Err()
|
||||
}
|
||||
if index == last {
|
||||
@@ -387,7 +398,15 @@ func (r *blockReader) listBlocks(first, last, maxBlocks int, verify bool) error
|
||||
if !verify {
|
||||
return nil
|
||||
}
|
||||
return r.verifyManifest(hashes, manifest, signer)
|
||||
if err := r.verifyManifest(hashes, manifest, signer); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, lb := range listed {
|
||||
if !r.fetch(lb.block, lb.hash) {
|
||||
return r.ctx.Err()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// decodeBlockHash returns the base64 SHA-256 hash.
|
||||
@@ -466,6 +485,9 @@ func (r *blockReader) fetchBlock(b api.Block, hash []byte) blockResult {
|
||||
if b, err = r.c.blockURL(r.ctx, r.n, index); err != nil {
|
||||
return blockResult{index: index, err: fmt.Errorf("block %d: %w", index, err)}
|
||||
}
|
||||
if relisted, err := decodeBlockHash(b.Hash); err != nil || !bytes.Equal(relisted, hash) {
|
||||
return blockResult{index: index, err: fmt.Errorf("block %d: %w: listed again with another hash", index, errIntegrity)}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -305,7 +305,7 @@ func TestDownloadBadManifest(t *testing.T) {
|
||||
data, err := readRange(t, c, n, 0, -1)
|
||||
require.ErrorIs(t, err, errIntegrity)
|
||||
assert.Contains(t, err.Error(), "ignore_signature_errors")
|
||||
assert.True(t, bytes.Equal(content, data), "the bytes come before the verdict")
|
||||
assert.Empty(t, data, "no byte comes before the verdict")
|
||||
|
||||
// A ranged read can't check the manifest.
|
||||
data, err = readRange(t, c, n, 0, 100)
|
||||
@@ -350,6 +350,29 @@ func TestDownloadStorage404Refetch(t *testing.T) {
|
||||
assert.Equal(t, 2, srv.Requests()[storagePattern])
|
||||
}
|
||||
|
||||
func TestDownloadRefetchOtherHash(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newTestClient(t)
|
||||
n := seedFile(t, c, u, "f", testContent(1<<20))
|
||||
|
||||
// The block's URL expires, and the server lists it again with
|
||||
// another block, which the checked manifest doesn't cover.
|
||||
srv.Inject(protontest.Fault{Pattern: "/storage/blocks/{id}", Status: http.StatusNotFound, Times: 1, Delay: 300 * time.Millisecond})
|
||||
done := make(chan struct{})
|
||||
var data []byte
|
||||
var err error
|
||||
go func() {
|
||||
defer close(done)
|
||||
data, err = readRange(t, c, n, 0, -1)
|
||||
}()
|
||||
require.Eventually(t, func() bool { return srv.Requests()[storagePattern] == 1 }, 5*time.Second, 5*time.Millisecond)
|
||||
u.Corrupt(n.ID, "foreign-block")
|
||||
<-done
|
||||
require.ErrorIs(t, err, errIntegrity)
|
||||
assert.Contains(t, err.Error(), "listed again with another hash")
|
||||
assert.Empty(t, data)
|
||||
}
|
||||
|
||||
func TestDownloadRetries5xx(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newTestClient(t)
|
||||
|
||||
@@ -791,9 +791,11 @@ func (u *User) insertLink(parentID string, l *link) string {
|
||||
// - "swap-blocks": the first two blocks trade places, each still
|
||||
// matching its own hash
|
||||
// - "truncate-blocks": the last block is dropped from the revision
|
||||
// - "rollback": the active revision serves the blocks of the revision
|
||||
// before it, which SeedRevision makes, under its own ID, manifest
|
||||
// signature and extended attributes
|
||||
// - "block-rollback": the active revision serves the blocks of the
|
||||
// revision before it, which SeedRevision makes, under its own ID,
|
||||
// manifest signature and extended attributes. (A server serving the
|
||||
// earlier revision whole under the active revision's ID can't be
|
||||
// detected, so isn't modelled.)
|
||||
// - "foreign-block": the first block is the same plaintext encrypted
|
||||
// with another content key, with its hash and block signature to
|
||||
// match
|
||||
@@ -844,7 +846,7 @@ func (u *User) Corrupt(id string, what string) {
|
||||
needBlocks(r, 1)
|
||||
r.blocks = r.blocks[:len(r.blocks)-1]
|
||||
})
|
||||
case "rollback":
|
||||
case "block-rollback":
|
||||
var previous *revision
|
||||
for _, r := range l.revisions {
|
||||
if r.state == revisionStateObsolete {
|
||||
|
||||
@@ -197,12 +197,12 @@ func TestSeedFormats(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCorrupt(t *testing.T) {
|
||||
for _, what := range []string{"name", "name-signature", "passphrase-signature", "block-hash", "manifest", "swap-blocks", "truncate-blocks", "rollback", "foreign-block"} {
|
||||
for _, what := range []string{"name", "name-signature", "passphrase-signature", "block-hash", "manifest", "swap-blocks", "truncate-blocks", "block-rollback", "foreign-block"} {
|
||||
t.Run(what, func(t *testing.T) {
|
||||
s := New(t)
|
||||
u := s.AddUser("user", "pass")
|
||||
victim := u.SeedFile(u.RootID(), "victim", patterned(blockSize+10), time.Now())
|
||||
if what == "rollback" {
|
||||
if what == "block-rollback" {
|
||||
u.SeedRevision(victim, patterned(blockSize+20), time.Now())
|
||||
}
|
||||
bystander := u.SeedFile(u.RootID(), "bystander", []byte("fine"), time.Now())
|
||||
|
||||
@@ -81,9 +81,6 @@ type hostileContent struct {
|
||||
// signatureOnly is set if only a signature check catches the
|
||||
// tampering, which ignore_signature_errors turns off.
|
||||
signatureOnly bool
|
||||
// released is how many bytes the reader may hand out before its
|
||||
// error: those of the blocks read before the check which fails.
|
||||
released int
|
||||
}
|
||||
|
||||
// seedCorrupted returns a prepare which seeds the file and applies the
|
||||
@@ -98,12 +95,17 @@ func seedCorrupted(what string) func(u *protontest.User, content []byte) string
|
||||
|
||||
// seedRolledBack returns a prepare which seeds the file with old
|
||||
// content, gives it a new revision with the content, then serves the
|
||||
// old revision's blocks under the new revision.
|
||||
// old revision's blocks under the new revision's manifest.
|
||||
//
|
||||
// A server serving the old revision whole - blocks, manifest and
|
||||
// extended attributes - under the new revision's ID can't be detected:
|
||||
// nothing signed binds them to the revision. The SDK can't detect it
|
||||
// either.
|
||||
func seedRolledBack(old []byte) func(u *protontest.User, content []byte) string {
|
||||
return func(u *protontest.User, content []byte) string {
|
||||
id := u.SeedFile(u.RootID(), "victim.bin", old, time.Now())
|
||||
u.SeedRevision(id, content, time.Now())
|
||||
u.Corrupt(id, "rollback")
|
||||
u.Corrupt(id, "block-rollback")
|
||||
return id
|
||||
}
|
||||
}
|
||||
@@ -118,24 +120,21 @@ var hostileContents = []hostileContent{{
|
||||
content: patternedContent(2*blockSize+10, 2),
|
||||
prepare: seedCorrupted("swap-blocks"),
|
||||
signatureOnly: true,
|
||||
released: 2*blockSize + 10,
|
||||
}, {
|
||||
name: "swapped blocks of different sizes",
|
||||
content: patternedContent(blockSize+10, 3),
|
||||
prepare: seedCorrupted("swap-blocks"),
|
||||
}, {
|
||||
name: "truncated block list",
|
||||
content: patternedContent(blockSize+10, 4),
|
||||
prepare: seedCorrupted("truncate-blocks"),
|
||||
released: blockSize,
|
||||
name: "truncated block list",
|
||||
content: patternedContent(blockSize+10, 4),
|
||||
prepare: seedCorrupted("truncate-blocks"),
|
||||
}, {
|
||||
name: "older revision content under the current revision ID",
|
||||
name: "older revision's blocks under the current revision",
|
||||
content: patternedContent(blockSize+10, 5),
|
||||
prepare: seedRolledBack(patternedContent(blockSize+10, 6)),
|
||||
signatureOnly: true,
|
||||
released: blockSize + 10,
|
||||
}, {
|
||||
name: "older revision content of another size under the current revision ID",
|
||||
name: "older revision's blocks of another size under the current revision",
|
||||
content: patternedContent(20, 7),
|
||||
prepare: seedRolledBack(patternedContent(10, 8)),
|
||||
}, {
|
||||
@@ -190,8 +189,7 @@ func TestHostileServer(t *testing.T) {
|
||||
require.NoError(t, err, "the file is listed")
|
||||
got, err := readAll(ctx, o)
|
||||
require.Error(t, err, "read without error")
|
||||
assert.LessOrEqual(t, len(got), tc.released, "bytes released before the error")
|
||||
assert.False(t, bytes.Equal(got, tc.content), "the content was returned")
|
||||
assert.Empty(t, got, "bytes released before the error")
|
||||
requireCopyFails(ctx, t, f, path)
|
||||
|
||||
// Turning signature checks off leaves the integrity
|
||||
@@ -202,7 +200,7 @@ func TestHostileServer(t *testing.T) {
|
||||
got, err = readAll(ctx, o)
|
||||
if !tc.signatureOnly {
|
||||
require.Error(t, err, "read without error with signature checks off")
|
||||
assert.LessOrEqual(t, len(got), tc.released, "bytes released before the error")
|
||||
assert.Empty(t, got, "bytes released before the error")
|
||||
}
|
||||
if bytes.Equal(got, tc.content) {
|
||||
// Only a signature was tampered with, and the content
|
||||
|
||||
@@ -5,6 +5,27 @@ it with a new one that links back. One entry per decision a future maintainer
|
||||
would otherwise have to reverse-engineer.
|
||||
No index: `grep -n '^## ' docs/DECISIONS.md` lists every entry.
|
||||
|
||||
## 2026-11-11 - Proton Drive verifies the manifest before releasing a full read
|
||||
|
||||
**Status:** accepted
|
||||
**Context:** A full read checked the manifest signature at EOF, as the
|
||||
SDK does, so blocks swapped, truncated or from an older revision, each
|
||||
matching its listed hash and recorded size, reached the reader before
|
||||
the error: `rclone cat` and streaming mounts showed wrong bytes (spec
|
||||
9.6 asks for "an integrity error, not data").
|
||||
**Decision:** A full read lists every block page first, verifies the
|
||||
manifest over the listed hashes, then fetches; each block is checked
|
||||
against its listed hash, and a block listed again with another hash
|
||||
is an integrity error. Pages are fetched one after another. Ranged
|
||||
reads keep checking blocks against their listed hashes only.
|
||||
**Alternatives:** Keeping the SDK's EOF check; verifying ranged reads
|
||||
too by listing the whole file on each ranged open (costly for mounts
|
||||
and multi-thread copies without a hash cache).
|
||||
**Consequences:** No extra requests, but a file of more than 20 blocks
|
||||
waits for its whole list before the first byte. Block metadata, not
|
||||
data, is held meanwhile. A whole earlier revision served under the
|
||||
current ID stays undetectable (protocol limit).
|
||||
|
||||
## 2026-11-11 - Proton Drive distrusts what a forged folder key vouches for
|
||||
|
||||
**Status:** accepted
|
||||
|
||||
+15
-10
@@ -8,16 +8,21 @@ Small, specific, independently actionable. Remove only when actually done.
|
||||
|
||||
## Next
|
||||
|
||||
- [ ] protondrive: a full read checks the manifest signature only at EOF
|
||||
(spec 6.4, as the SDK does), so a hostile server serving blocks
|
||||
swapped, or an older revision's blocks, of the recorded sizes gets
|
||||
that wrong plaintext handed to the reader before the final `Read`
|
||||
fails (`TestHostileServer` "swapped blocks", "older revision
|
||||
content"). `operations.Copy` discards it, but `rclone cat` and a
|
||||
streaming mount don't. Consider listing every block page and checking
|
||||
the manifest before releasing the first byte, at least when the block
|
||||
list fits in one page; with `ignore_signature_errors` only the SHA1
|
||||
check of a copy catches these.
|
||||
- [ ] protondrive: a full read lists every block page before releasing
|
||||
the first byte, one request per 20 blocks in sequence, so a very large
|
||||
file waits (about 12 requests for 1 GiB, about 1280 for 100 GiB) before
|
||||
data flows. Fetching pages in parallel under the 15-request metadata
|
||||
limit would cut that.
|
||||
- [ ] protondrive: a ranged read (offset > 0 or a limit, as multi-thread
|
||||
copies and mounts make) checks each block against the hash the server
|
||||
lists for it, its decryption and its recorded size, but not the
|
||||
manifest, so a server can serve validly encrypted blocks of the same
|
||||
file swapped or from an older revision of the same size. The planned
|
||||
fix is a bounded per-revision cache of manifest-verified block hashes.
|
||||
- [ ] protondrive: a server can serve an earlier revision whole (blocks,
|
||||
manifest and extended attributes, all validly signed) under the current
|
||||
revision ID undetected, as nothing signed binds them to the revision;
|
||||
the SDK can't detect it either. Raise with Proton.
|
||||
- [ ] protondrive: `--dump headers` logs Proton's `pm-storage-token` and
|
||||
`x-pm-uid` (fshttp redacts only `Authorization`), and `--dump bodies`
|
||||
logs the access, refresh and upload tokens in JSON bodies (seen in
|
||||
|
||||
@@ -573,8 +573,8 @@ library what must stay readable.
|
||||
`func (u *User) Corrupt(id string, what string)` with `what` one of
|
||||
`"name"`, `"name-signature"`, `"passphrase-signature"`, `"block-hash"`,
|
||||
`"manifest"`, `"swap-blocks"`, `"truncate-blocks"`, and (Task 16)
|
||||
`"rollback"` (the active revision serves the previous revision's
|
||||
blocks) and `"foreign-block"` (the first block encrypted with another
|
||||
`"block-rollback"` (the active revision serves the previous
|
||||
revision's blocks) and `"foreign-block"` (the first block encrypted with another
|
||||
content key).
|
||||
- Secrets for log hygiene tests (Task 16):
|
||||
`func (s *Server) IssuedSecrets() []string` (every access and refresh
|
||||
|
||||
@@ -293,11 +293,19 @@ AES-GCM blocks; names, passphrases, hash keys and attributes without AEAD).
|
||||
- Ranged reads map an offset to a block using the recorded block sizes, so
|
||||
`NoMultiThreading` is dropped and multi-thread downloads work. At most 10
|
||||
blocks are in flight for download across the `Fs` (SDK value).
|
||||
- A full, sequential read verifies the manifest signature at EOF. Failure is
|
||||
an error. An advanced option `ignore_signature_errors` downgrades content
|
||||
and authorship signature failures to a logged `ERROR` for users with
|
||||
legacy or shared data (#8003, #7344). **(call)** Ranged reads cannot verify
|
||||
the manifest; this is documented, as it is in the SDK.
|
||||
- A full, sequential read lists every block and verifies the manifest
|
||||
signature over the listed hashes before releasing the first byte; each
|
||||
block is then checked against its listed hash as it arrives, and a block
|
||||
listed again (for a new URL) with another hash is an integrity error.
|
||||
Failure is an error. An advanced option `ignore_signature_errors`
|
||||
downgrades content and authorship signature failures to a logged `ERROR`
|
||||
for users with legacy or shared data (#8003, #7344). **(call)** Ranged
|
||||
reads check each block against its listed hash but cannot verify the
|
||||
manifest; this is documented, as it is in the SDK.
|
||||
- A server serving an earlier revision whole (blocks, manifest and
|
||||
extended attributes, all validly signed) under the current revision ID is
|
||||
undetectable: nothing signed binds them to the revision. The SDK cannot
|
||||
detect it either; it is a protocol limit to raise with Proton.
|
||||
- Old-format files (pre-AEAD) remain readable.
|
||||
- `Close` releases everything; no reader outlives its `Object.Open` caller.
|
||||
|
||||
@@ -535,9 +543,12 @@ CI, long runs are manual.
|
||||
|
||||
- **Hostile server.** The fake returns: a block with a wrong hash; blocks
|
||||
swapped or truncated; a block list shorter than the manifest; an older
|
||||
revision's content under the current revision ID; a name decrypting to
|
||||
`../x`, `a/b` or containing NUL; a 10 MiB "name"; an attacker-signed
|
||||
passphrase. Each must produce an integrity error, not data.
|
||||
revision's blocks under the current revision; a block encrypted with
|
||||
another key; a name decrypting to `../x`, `a/b` or containing NUL; a 10
|
||||
MiB "name"; an attacker-signed passphrase; a folder with a server-made
|
||||
key and anonymous files beneath it. Each must produce an integrity error,
|
||||
not data: a full read releases no byte before the error. (A whole earlier
|
||||
revision served under the current ID is undetectable; see 6.4.)
|
||||
- **Secret hygiene.** A test runs login, upload and download at `-vv`
|
||||
(without `--dump`), then searches the captured log for the known password,
|
||||
key password, tokens, session keys and upload tokens. A second test checks
|
||||
|
||||
Reference in new issue
Block a user