protondrive: add request budget tests and fuzz targets
Request budgets, measured against the fake server and asserted at the measured counts, for listing (cold and warm, with and without enable_caching), a no-op sync by modification time and by checksum, NewObject three folders deep, uploads and making folders. A TestMain hook fails the package if any test, through any fake server, requested a removed share-scoped route; the fake keeps a process-wide tally of them for it. Fuzz targets, seeded from testdata/oldlib and from what the fake writes, for name validation, link decoding, extended attributes, API error bodies with Retry-After and session fork payloads. Each checks invariants of what is accepted. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
90b2a48e71
commit
12ce57ed31
9 files changed
+759
-9
No files matched your search
@@ -0,0 +1,344 @@
|
||||
package protondrive_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/protondrive/protontest"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
fssync "github.com/rclone/rclone/fs/sync"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Routes counted by the request budgets.
|
||||
const (
|
||||
linksRoute = "POST /drive/v2/volumes/{volumeID}/links"
|
||||
revisionDraftRoute = "POST /drive/v2/volumes/{volumeID}/files/{linkID}/revisions"
|
||||
verifierRoute = "GET /drive/v2/volumes/{volumeID}/links/{linkID}/revisions/{revisionID}/verification"
|
||||
blockURLsRoute = "POST /drive/blocks"
|
||||
blockPutRoute = "POST /storage/upload/{token}"
|
||||
commitRoute = "PUT /drive/v2/volumes/{volumeID}/files/{linkID}/revisions/{revisionID}"
|
||||
createFolderRoute = "POST /drive/v2/volumes/{volumeID}/folders"
|
||||
)
|
||||
|
||||
// TestMain runs the tests, then fails the run if any of them, through
|
||||
// any fake server, requested a removed share-scoped route.
|
||||
func TestMain(m *testing.M) {
|
||||
code := m.Run()
|
||||
if legacy := protontest.LegacyRequests(); len(legacy) > 0 {
|
||||
fmt.Fprintf(os.Stderr, "FAIL: TestBudgetNoLegacyRoutes: removed share-scoped routes were requested: %v\n", legacy)
|
||||
code = 1
|
||||
}
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
// totalRequests returns the sum of the counts in requests.
|
||||
func totalRequests(requests map[string]int) int {
|
||||
n := 0
|
||||
for _, count := range requests {
|
||||
n += count
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// ceilDiv returns n/d rounded up.
|
||||
func ceilDiv(n, d int) int {
|
||||
return (n + d - 1) / d
|
||||
}
|
||||
|
||||
// childPages returns the number of child ID pages the fake returns for
|
||||
// a folder of n children with page size pageSize: at least one.
|
||||
func childPages(n, pageSize int) int {
|
||||
return max(1, ceilDiv(n, pageSize))
|
||||
}
|
||||
|
||||
// seedFiles seeds n small files into folder parentID.
|
||||
func seedFiles(u *protontest.User, parentID string, n int) {
|
||||
for i := range n {
|
||||
u.SeedFile(parentID, fmt.Sprintf("file%03d.txt", i), []byte(fmt.Sprintf("file %d", i)), time.Now())
|
||||
}
|
||||
}
|
||||
|
||||
// budgetSizes are the folder sizes the listing budgets are checked at:
|
||||
// around the metadata batch of 100 and the child page of 150.
|
||||
var budgetSizes = []int{0, 1, 100, 101, 250}
|
||||
|
||||
// budgetPageSize is the child ID page size the listing budgets use.
|
||||
const budgetPageSize = 150
|
||||
|
||||
// seedBudgetFolders seeds a folder "nN" of N files for each of sizes.
|
||||
func seedBudgetFolders(srv *protontest.Server, u *protontest.User, sizes []int) {
|
||||
srv.SetChildrenPageSize(budgetPageSize)
|
||||
for _, n := range sizes {
|
||||
seedFiles(u, u.SeedFolder(u.RootID(), fmt.Sprintf("n%d", n)), n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBudgetList checks listing a folder of N files costs one request
|
||||
// per page of child IDs and one per batch of 100 nodes, within spec
|
||||
// 9.4's P + ceil(N/100) + 1. The Fs is rooted at the folder, so finding
|
||||
// it is part of NewFs.
|
||||
//
|
||||
// Measured, per route (children pages + links batches):
|
||||
// N=0: 1+0, N=1: 1+1, N=100: 1+1, N=101: 1+2, N=250: 2+3 - exactly
|
||||
// P + ceil(N/100), so the bound is that, without the spec's +1.
|
||||
func TestBudgetList(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
remote, srv, u := setupFakeRemote(t)
|
||||
seedBudgetFolders(srv, u, budgetSizes)
|
||||
for _, n := range budgetSizes {
|
||||
t.Run(fmt.Sprint(n), func(t *testing.T) {
|
||||
f := newFakeFs(t, remote, fmt.Sprintf("n%d", n))
|
||||
srv.ResetRequests()
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, n)
|
||||
|
||||
got := srv.Requests()
|
||||
pages, batches := childPages(n, budgetPageSize), ceilDiv(n, 100)
|
||||
assert.LessOrEqual(t, got[childrenRoute], pages, "%v", got)
|
||||
assert.LessOrEqual(t, got[linksRoute], batches, "%v", got)
|
||||
assert.LessOrEqual(t, totalRequests(got), pages+batches, "%v", got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBudgetWarmList checks a second listing costs nothing with
|
||||
// enable_caching at its default, and with it off costs the same as the
|
||||
// first, no more.
|
||||
//
|
||||
// It is checked at the sizes of TestBudgetList which differ in kind -
|
||||
// empty, one batch, several pages and batches - to save time.
|
||||
//
|
||||
// Measured: warm 0 at every N; with caching off the second listing
|
||||
// repeats the cold breakdown of TestBudgetList exactly.
|
||||
func TestBudgetWarmList(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
sizes := []int{0, 1, 250}
|
||||
remote, srv, u := setupFakeRemote(t)
|
||||
seedBudgetFolders(srv, u, sizes)
|
||||
for _, caching := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("caching=%v", caching), func(t *testing.T) {
|
||||
if !caching {
|
||||
t.Setenv(fakeEnvPrefix+"ENABLE_CACHING", "false")
|
||||
}
|
||||
for _, n := range sizes {
|
||||
f := newFakeFs(t, remote, fmt.Sprintf("n%d", n))
|
||||
srv.ResetRequests()
|
||||
_, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
cold := srv.Requests()
|
||||
|
||||
srv.ResetRequests()
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, n)
|
||||
warm := srv.Requests()
|
||||
if caching {
|
||||
assert.Empty(t, warm, "N=%d", n)
|
||||
} else {
|
||||
assert.Equal(t, cold, warm, "N=%d", n)
|
||||
assert.LessOrEqual(t, totalRequests(warm), childPages(n, budgetPageSize)+ceilDiv(n, 100), "N=%d: %v", n, warm)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBudgetNoopSync checks a cold sync of a tree already in sync, by
|
||||
// size and modification time and by --checksum, lists each folder once
|
||||
// and transfers nothing: SHA-1s come from the metadata.
|
||||
//
|
||||
// For D = 6 folders (the root and 5 below it) and N = 200 files (40 per
|
||||
// folder), the plan's bound is 2*D + ceil(N/100) + 5 = 19. Measured, per
|
||||
// route, either way: 6 children pages + 6 links batches = 12, one page
|
||||
// and one batch per folder, as no folder has more than 100 children.
|
||||
// The bound is that: 2*D.
|
||||
func TestBudgetNoopSync(t *testing.T) {
|
||||
const (
|
||||
folders = 5
|
||||
filesPerFolder = 40
|
||||
budget = 2 * (folders + 1)
|
||||
)
|
||||
remote, srv, u := setupFakeRemote(t)
|
||||
local := t.TempDir()
|
||||
modTime := time.Now().Add(-time.Hour).Truncate(time.Second)
|
||||
for d := range folders {
|
||||
dir := fmt.Sprintf("dir%d", d)
|
||||
id := u.SeedFolder(u.RootID(), dir)
|
||||
require.NoError(t, os.Mkdir(filepath.Join(local, dir), 0o777))
|
||||
for i := range filesPerFolder {
|
||||
name := fmt.Sprintf("file%02d.txt", i)
|
||||
content := []byte(fmt.Sprintf("content of %s/%s", dir, name))
|
||||
u.SeedFile(id, name, content, modTime)
|
||||
p := filepath.Join(local, dir, name)
|
||||
require.NoError(t, os.WriteFile(p, content, 0o666))
|
||||
require.NoError(t, os.Chtimes(p, modTime, modTime))
|
||||
}
|
||||
}
|
||||
|
||||
for _, checksum := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("checksum=%v", checksum), func(t *testing.T) {
|
||||
ctx, ci := fs.AddConfig(context.Background())
|
||||
ci.CheckSum = checksum
|
||||
src, err := fs.NewFs(ctx, local)
|
||||
require.NoError(t, err)
|
||||
f := newFakeFs(t, remote, "")
|
||||
srv.ResetRequests()
|
||||
require.NoError(t, fssync.Sync(ctx, f, src, false))
|
||||
|
||||
got := srv.Requests()
|
||||
assert.LessOrEqual(t, totalRequests(got), budget, "%v", got)
|
||||
assert.Zero(t, blockTransfers(got), "%v", got)
|
||||
assert.Zero(t, got[draftRoute]+got[revisionDraftRoute], "drafts created: %v", got)
|
||||
for route := range got {
|
||||
assert.NotContains(t, route, "/revisions/", "file content read: %v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBudgetNewObject checks finding a file three folders deep, cold,
|
||||
// costs one listing, a children page and a links batch, per folder on
|
||||
// its path.
|
||||
//
|
||||
// The plan's bound is 2 per level + 1 = 7 for three levels. Measured
|
||||
// for a/b/c/file.txt, per route: 4 children pages + 4 links batches =
|
||||
// 8, as the root is listed too - it is a fourth folder on the path,
|
||||
// and Proton has no route to look a child up by name. The bound is 2
|
||||
// per folder listed: 2*4.
|
||||
func TestBudgetNewObject(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
remote, srv, u := setupFakeRemote(t)
|
||||
parent := u.RootID()
|
||||
for _, name := range []string{"a", "b", "c"} {
|
||||
seedFiles(u, parent, 3)
|
||||
parent = u.SeedFolder(parent, name)
|
||||
}
|
||||
u.SeedFile(parent, "file.txt", []byte("deep"), time.Now())
|
||||
|
||||
f := newFakeFs(t, remote, "")
|
||||
srv.ResetRequests()
|
||||
o, err := f.NewObject(ctx, "a/b/c/file.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(4), o.Size())
|
||||
|
||||
got := srv.Requests()
|
||||
const listed = 4 // the root, a, b and c
|
||||
assert.LessOrEqual(t, got[childrenRoute], listed, "%v", got)
|
||||
assert.LessOrEqual(t, got[linksRoute], listed, "%v", got)
|
||||
assert.LessOrEqual(t, totalRequests(got), 2*listed, "%v", got)
|
||||
}
|
||||
|
||||
// TestBudgetUpload checks the requests of uploading new files to a
|
||||
// folder: one listing of the folder to check the name is free, the
|
||||
// first time only, then per file a draft, its verification code, at
|
||||
// most one block URL request per block, one PUT per block, the commit
|
||||
// and one read of the committed node.
|
||||
//
|
||||
// Measured, per route, into a folder of 3 files:
|
||||
//
|
||||
// small.txt, cold: children 1 + links 1 (the listing), draft 1,
|
||||
// verifier 1, block URLs 1, block PUTs 1, commit 1, links 1 = 8
|
||||
// big.bin, 3 blocks: draft 1, verifier 1, block URLs 1-3 (as blocks
|
||||
// are ready), block PUTs 3, commit 1, links 1 = 8-10
|
||||
// again.txt: as small.txt without the listing = 6
|
||||
func TestBudgetUpload(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
remote, srv, u := setupFakeRemote(t)
|
||||
seedFiles(u, u.SeedFolder(u.RootID(), "dir"), 3)
|
||||
f := newFakeFs(t, remote, "dir")
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
size int
|
||||
blocks int
|
||||
listed bool // the folder is listed to check the name is free
|
||||
}{
|
||||
{"small.txt", 5, 1, true},
|
||||
{"big.bin", 2*blockSize + 5, 3, false},
|
||||
{"again.txt", 5, 1, false},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
srv.ResetRequests()
|
||||
content := patternedContent(test.size, 1)
|
||||
_, err := operations.Copy(ctx, f, nil, test.name, object.NewMemoryObject(test.name, time.Now(), content))
|
||||
require.NoError(t, err)
|
||||
|
||||
got := srv.Requests()
|
||||
listing := 0
|
||||
if test.listed {
|
||||
listing = 2 // a children page and a links batch
|
||||
assert.Equal(t, 1, got[childrenRoute], "%v", got)
|
||||
} else {
|
||||
assert.Zero(t, got[childrenRoute], "folder listed again: %v", got)
|
||||
}
|
||||
assert.Equal(t, 1, got[draftRoute], "%v", got)
|
||||
assert.Equal(t, 1, got[verifierRoute], "%v", got)
|
||||
assert.GreaterOrEqual(t, got[blockURLsRoute], 1, "%v", got)
|
||||
assert.LessOrEqual(t, got[blockURLsRoute], test.blocks, "%v", got)
|
||||
assert.Equal(t, test.blocks, got[blockPutRoute], "%v", got)
|
||||
assert.Equal(t, 1, got[commitRoute], "%v", got)
|
||||
assert.Equal(t, listing/2+1, got[linksRoute], "%v", got)
|
||||
assert.LessOrEqual(t, totalRequests(got), listing+4+2*test.blocks, "%v", got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBudgetMkdir checks making folders side by side lists their
|
||||
// parent once, to check the first name is free, and then costs one
|
||||
// request per folder.
|
||||
//
|
||||
// Measured, per route, into a folder of 3 files: x: children 1 +
|
||||
// links 1 + create 1 = 3; y and z: create 1 each.
|
||||
func TestBudgetMkdir(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
remote, srv, u := setupFakeRemote(t)
|
||||
seedFiles(u, u.SeedFolder(u.RootID(), "dir"), 3)
|
||||
f := newFakeFs(t, remote, "dir")
|
||||
srv.ResetRequests()
|
||||
for _, name := range []string{"x", "y", "z"} {
|
||||
require.NoError(t, f.Mkdir(ctx, name))
|
||||
}
|
||||
got := srv.Requests()
|
||||
assert.Equal(t, map[string]int{childrenRoute: 1, linksRoute: 1, createFolderRoute: 3}, got)
|
||||
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, entries, 6, "the cached listing holds the new folders")
|
||||
}
|
||||
|
||||
// TestBudgetNoLegacyRoutes checks a run of everyday operations requests
|
||||
// no removed share-scoped route. TestMain checks the same of every
|
||||
// test in the package, through every fake server.
|
||||
func TestBudgetNoLegacyRoutes(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
remote, srv, u := setupFakeRemote(t)
|
||||
seedFiles(u, u.SeedFolder(u.RootID(), "dir"), 3)
|
||||
f := newFakeFs(t, remote, "")
|
||||
o, err := operations.Copy(ctx, f, nil, "dir/new.txt", object.NewMemoryObject("dir/new.txt", time.Now(), []byte("new")))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "new", readObject(t, o))
|
||||
_, err = operations.Move(ctx, f, nil, "moved/new.txt", o)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, operations.Purge(ctx, f, "dir"))
|
||||
_, err = f.List(ctx, "moved")
|
||||
require.NoError(t, err)
|
||||
_, err = f.Features().About(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
for route := range srv.Requests() {
|
||||
if strings.Contains(route, "/drive/shares/") {
|
||||
assert.Equal(t, "GET /drive/shares/{shareID}", route)
|
||||
}
|
||||
}
|
||||
assert.Empty(t, protontest.LegacyRequests())
|
||||
}
|
||||
@@ -44,7 +44,7 @@ func TestComputeKeyPassword(t *testing.T) {
|
||||
}
|
||||
|
||||
// sealForkPayload encrypts plain as Proton's sign-in page does.
|
||||
func sealForkPayload(t *testing.T, key []byte, plain, aad string) string {
|
||||
func sealForkPayload(t testing.TB, key []byte, plain, aad string) string {
|
||||
t.Helper()
|
||||
block, err := aes.NewCipher(key)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -679,7 +679,7 @@ func blockFileName(i int) string {
|
||||
}
|
||||
|
||||
// loadFrozenFile returns the file in dir.
|
||||
func loadFrozenFile(t *testing.T, dir string) frozenFile {
|
||||
func loadFrozenFile(t testing.TB, dir string) frozenFile {
|
||||
t.Helper()
|
||||
var f frozenFile
|
||||
for name, v := range frozenFileParts(&f) {
|
||||
|
||||
@@ -0,0 +1,346 @@
|
||||
package protondrive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode/utf16"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/ProtonMail/gopenpgp/v3/crypto"
|
||||
"github.com/rclone/rclone/backend/protondrive/api"
|
||||
"github.com/rclone/rclone/backend/protondrive/protontest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Fuzz targets for everything which parses bytes the server controls.
|
||||
// Each is seeded from the old library's fixtures in testdata/oldlib and
|
||||
// from payloads the fake server makes, and checks invariants of what
|
||||
// is accepted, not only that nothing panics.
|
||||
|
||||
// oldlibFixture is the part of testdata/oldlib/file.json and
|
||||
// folder.json the fuzz targets use.
|
||||
type oldlibFixture struct {
|
||||
NodeKey string
|
||||
NodePassphrasePlain string
|
||||
NamePlain string
|
||||
ChildName string
|
||||
XAttr string
|
||||
}
|
||||
|
||||
// loadOldlibFixture returns testdata/oldlib/name.
|
||||
func loadOldlibFixture(t testing.TB, name string) oldlibFixture {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join("testdata/oldlib", name))
|
||||
require.NoError(t, err)
|
||||
var fx oldlibFixture
|
||||
require.NoError(t, json.Unmarshal(data, &fx))
|
||||
return fx
|
||||
}
|
||||
|
||||
// fuzzLinks returns a client for a user of a fake server holding the
|
||||
// keys of the old library's v6 file fixture, the root folder's keys,
|
||||
// and the JSON of a link of each kind the client reads: the fixture
|
||||
// itself, and nodes the fake makes as SDK clients would.
|
||||
func fuzzLinks(f *testing.F) (*client, *nodeKeys, [][]byte) {
|
||||
frozen := loadFrozenFile(f, filepath.Join(frozenFileDir, "v6"))
|
||||
srv := protontest.New(f)
|
||||
u := srv.AddUserWithKeys(frozen.Email, "pass", frozen.Keys)
|
||||
c := newTestClientFor(f, srv, u, clientOptions{})
|
||||
root := u.RootID()
|
||||
ids := []string{
|
||||
u.SeedStoredFile(root, frozen.Stored),
|
||||
u.SeedFolder(root, "folder"),
|
||||
u.SeedFile(root, "file.txt", []byte("content"), time.Now()),
|
||||
u.SeedFile(root, "empty", nil, time.Now()),
|
||||
u.SeedLegacyFile(root, "legacy", []byte("legacy")),
|
||||
u.SeedAnonymousFile(root, "anonymous", []byte("anonymous"), time.Now()),
|
||||
u.SeedAnonymousFolder(root, "anonymous folder"),
|
||||
u.SeedFileClaimingBlockSizes(root, "claims", []byte("claims"), []int64{1, 2, 3}),
|
||||
u.SeedDoc(root, "doc"),
|
||||
}
|
||||
ctx := context.Background()
|
||||
links, err := c.fetchLinks(ctx, ids)
|
||||
require.NoError(f, err)
|
||||
require.Len(f, links, len(ids))
|
||||
var seeds [][]byte
|
||||
for _, d := range links {
|
||||
data, err := json.Marshal(d)
|
||||
require.NoError(f, err)
|
||||
seeds = append(seeds, data)
|
||||
}
|
||||
keys, err := c.folderKeysOf(ctx, root)
|
||||
require.NoError(f, err)
|
||||
return c, keys, seeds
|
||||
}
|
||||
|
||||
func FuzzValidateName(f *testing.F) {
|
||||
for _, name := range []string{
|
||||
loadOldlibFixture(f, "file.json").NamePlain,
|
||||
loadOldlibFixture(f, "folder.json").NamePlain,
|
||||
loadOldlibFixture(f, "folder.json").ChildName,
|
||||
"file.txt", ".", "..", "...", "", "a/b", "/", "a\x00b", " ", "é", "é",
|
||||
strings.Repeat("a", maxNameLength), strings.Repeat("a", maxNameLength+1),
|
||||
strings.Repeat("\U0001F600", maxNameLength/2), strings.Repeat("\U0001F600", maxNameLength/2+1),
|
||||
"\xff\xfe",
|
||||
} {
|
||||
f.Add(name)
|
||||
}
|
||||
f.Fuzz(func(t *testing.T, name string) {
|
||||
if validateName(name) != nil {
|
||||
return
|
||||
}
|
||||
assert.NotEmpty(t, name)
|
||||
assert.NotEqual(t, ".", name)
|
||||
assert.NotEqual(t, "..", name)
|
||||
assert.NotContains(t, name, "/")
|
||||
assert.NotContains(t, name, "\x00")
|
||||
assert.LessOrEqual(t, len(utf16.Encode([]rune(name))), maxNameLength)
|
||||
})
|
||||
}
|
||||
|
||||
func FuzzDecodeLink(f *testing.F) {
|
||||
c, rootKeys, seeds := fuzzLinks(f)
|
||||
for _, seed := range seeds {
|
||||
f.Add(seed)
|
||||
// Hostile values in links which otherwise decrypt.
|
||||
var d api.LinkDetails
|
||||
require.NoError(f, json.Unmarshal(seed, &d))
|
||||
for _, change := range []func(d *api.LinkDetails){
|
||||
func(d *api.LinkDetails) { d.Link.LinkID = "" },
|
||||
func(d *api.LinkDetails) { d.Link.Type = 3 - d.Link.Type },
|
||||
func(d *api.LinkDetails) {
|
||||
if d.File != nil && d.File.ActiveRevision != nil {
|
||||
d.File.ActiveRevision.EncryptedSize = -5
|
||||
}
|
||||
},
|
||||
} {
|
||||
changed := d
|
||||
changed.File = cloneFile(d.File)
|
||||
change(&changed)
|
||||
data, err := json.Marshal(changed)
|
||||
require.NoError(f, err)
|
||||
f.Add(data)
|
||||
}
|
||||
}
|
||||
// Shapes the client can't decrypt.
|
||||
for _, format := range []string{"legacy", "v6"} {
|
||||
data, err := os.ReadFile(filepath.Join(frozenFileDir, format, "link.json"))
|
||||
require.NoError(f, err)
|
||||
f.Add(data)
|
||||
}
|
||||
f.Add([]byte(`{}`))
|
||||
f.Add([]byte(`{"Link":{"LinkID":"x","Type":1,"ParentLinkID":"p"},"Folder":{}}`))
|
||||
ctx := context.Background()
|
||||
f.Fuzz(func(t *testing.T, data []byte) {
|
||||
var d api.LinkDetails
|
||||
if json.Unmarshal(data, &d) != nil {
|
||||
return
|
||||
}
|
||||
n, unreadable, err := c.decryptNode(ctx, &d, rootKeys)
|
||||
if err != nil || unreadable != nil {
|
||||
assert.Nil(t, n)
|
||||
return
|
||||
}
|
||||
require.NotNil(t, n)
|
||||
assert.NotEmpty(t, n.ID)
|
||||
assert.Equal(t, d.Link.LinkID, n.ID)
|
||||
assert.True(t, utf8.ValidString(n.Name), "%q", n.Name)
|
||||
if d.Link.ParentLinkID != "" {
|
||||
assert.NoError(t, validateName(n.Name))
|
||||
}
|
||||
assert.True(t, n.IsFolder == (d.Link.Type == api.LinkTypeFolder))
|
||||
for _, original := range []bool{true, false} {
|
||||
assert.GreaterOrEqual(t, n.size(original), int64(0), "original=%v", original)
|
||||
}
|
||||
if n.X != nil {
|
||||
checkXAttrs(t, n.X)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// cloneFile returns a copy of file, sharing nothing changeable with it.
|
||||
func cloneFile(file *api.File) *api.File {
|
||||
if file == nil {
|
||||
return nil
|
||||
}
|
||||
out := *file
|
||||
if file.ActiveRevision != nil {
|
||||
rev := *file.ActiveRevision
|
||||
out.ActiveRevision = &rev
|
||||
}
|
||||
return &out
|
||||
}
|
||||
|
||||
// checkXAttrs checks the invariants of parsed extended attributes.
|
||||
//
|
||||
// That the block sizes add up to the size isn't one: as in the SDK
|
||||
// neither is checked against the other, and a read checks each block
|
||||
// against its recorded size, so a file whose attributes disagree fails
|
||||
// to read rather than reading wrong.
|
||||
func checkXAttrs(t *testing.T, x *xattrs) {
|
||||
t.Helper()
|
||||
assert.GreaterOrEqual(t, x.Size, int64(0))
|
||||
if x.BlockSizes != nil {
|
||||
assert.NotEmpty(t, x.BlockSizes)
|
||||
}
|
||||
for i, size := range x.BlockSizes {
|
||||
assert.Positive(t, size, "block %d", i+1)
|
||||
}
|
||||
if !x.ModTime.IsZero() {
|
||||
assert.Equal(t, time.UTC, x.ModTime.Location())
|
||||
// RFC 3339 years, moved by a zone offset of under a day.
|
||||
assert.True(t, x.ModTime.Year() >= -1 && x.ModTime.Year() <= 10000, "%v", x.ModTime)
|
||||
}
|
||||
assert.NotContains(t, x.Extra, "Common")
|
||||
}
|
||||
|
||||
func FuzzDecryptXAttrs(f *testing.F) {
|
||||
fx := loadOldlibFixture(f, "file.json")
|
||||
key, err := unlockKey(fx.NodeKey, []byte(fx.NodePassphrasePlain))
|
||||
require.NoError(f, err)
|
||||
nodeKR, err := crypto.NewKeyRing(key)
|
||||
require.NoError(f, err)
|
||||
now := time.Now()
|
||||
|
||||
// The old library's attributes, encrypted and in plain text.
|
||||
res, _, _, err := decryptMessage(fx.XAttr, nodeKR, nil, now, maxXAttrsSize)
|
||||
require.NoError(f, err)
|
||||
f.Add(fx.XAttr, res.Bytes())
|
||||
// Attributes as the fake writes them for SDK clients.
|
||||
c, _, seeds := fuzzLinks(f)
|
||||
for _, seed := range seeds {
|
||||
var d api.LinkDetails
|
||||
require.NoError(f, json.Unmarshal(seed, &d))
|
||||
if d.File == nil || d.File.ActiveRevision == nil || d.File.ActiveRevision.XAttr == "" {
|
||||
continue
|
||||
}
|
||||
n, err := c.getNode(context.Background(), d.Link.LinkID)
|
||||
require.NoError(f, err)
|
||||
res, _, _, err := decryptMessage(d.File.ActiveRevision.XAttr, n.keys.keyRing, nil, now, maxXAttrsSize)
|
||||
require.NoError(f, err)
|
||||
f.Add("", res.Bytes())
|
||||
}
|
||||
for _, plain := range []string{
|
||||
`{"Common":{"ModificationTime":"2021-09-16T07:40:54+0000","Size":-1,"BlockSizes":[4194304,-1],"Digests":{"SHA1":"x"}}}`,
|
||||
`{"Common":{"Size":1152921504606846976,"BlockSizes":[1152921504606846976,1]}}`,
|
||||
`{"Common":{"ModificationTime":"99999-01-01T00:00:00Z"}}`,
|
||||
`{"Common":{"ModificationTime":"0000-01-01T00:00:00+23:59"}}`,
|
||||
`{"Common":{"BlockSizes":[]},"Media":{"Width":1}}`,
|
||||
`{"Common":{"BlockSizes":[` + strings.Repeat("4194304,", 10000) + `1]}}`,
|
||||
strings.Repeat(`{"Common":`, 20000) + strings.Repeat("}", 20000),
|
||||
`{"Common":` + strings.Repeat("[", 100000) + strings.Repeat("]", 100000) + `}`,
|
||||
} {
|
||||
f.Add("", []byte(plain))
|
||||
}
|
||||
|
||||
f.Fuzz(func(t *testing.T, enc string, plain []byte) {
|
||||
if x, _, err := decryptXAttrs(enc, nodeKR, nodeKR, now); err == nil {
|
||||
checkXAttrs(t, x)
|
||||
}
|
||||
if len(plain) > maxXAttrsSize/2 {
|
||||
return
|
||||
}
|
||||
enc, err := encryptArmored(plain, nil, nodeKR, nodeKR, true, now)
|
||||
require.NoError(t, err)
|
||||
x, verifyErr, err := decryptXAttrs(enc, nodeKR, nodeKR, now)
|
||||
if !utf8.Valid(plain) {
|
||||
assert.Error(t, err)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, verifyErr)
|
||||
checkXAttrs(t, x)
|
||||
})
|
||||
}
|
||||
|
||||
func FuzzDecodeAPIError(f *testing.F) {
|
||||
for _, seed := range []struct {
|
||||
status int
|
||||
body string
|
||||
retryAfter string
|
||||
}{
|
||||
{422, `{"Code":2500,"Error":"A file or folder with that name already exists","Details":{"ConflictLinkID":"abc","ConflictRevisionID":"def"}}`, ""},
|
||||
{422, `{"Code":2501,"Error":"File or folder not found"}`, ""},
|
||||
{429, `{"Code":85131,"Error":"Too many recent API requests"}`, "17"},
|
||||
{429, ``, "Wed, 21 Oct 2015 07:28:00 GMT"},
|
||||
{429, ``, "99999999999999999999999"},
|
||||
{401, `{"Code":401,"Error":"Invalid access token"}`, ""},
|
||||
{400, `{"Code":2064,"Error":"protontest: invalid x-pm-appversion \"x\""}`, ""},
|
||||
{500, `<html>Internal Server Error</html>`, ""},
|
||||
{200, `{"Code":"1000"}`, ""},
|
||||
{0, `{"Error":null,"Details":[1,2,3]}`, "-5"},
|
||||
} {
|
||||
f.Add(seed.status, []byte(seed.body), seed.retryAfter)
|
||||
}
|
||||
f.Fuzz(func(t *testing.T, status int, body []byte, retryAfterValue string) {
|
||||
resp := &http.Response{
|
||||
StatusCode: status,
|
||||
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
|
||||
Header: http.Header{"Retry-After": {retryAfterValue}},
|
||||
Body: io.NopCloser(bytes.NewReader(body)),
|
||||
}
|
||||
err := errorHandler(resp)
|
||||
var apiErr *api.Error
|
||||
require.ErrorAs(t, err, &apiErr)
|
||||
assert.Equal(t, status, apiErr.Status)
|
||||
assert.NotEmpty(t, apiErr.Message)
|
||||
assert.NotEmpty(t, err.Error())
|
||||
assert.Equal(t, apiErr.Code, apiErrorCode(err))
|
||||
|
||||
wait := retryAfter(resp)
|
||||
assert.Positive(t, wait)
|
||||
assert.LessOrEqual(t, wait, maxRetryAfter)
|
||||
})
|
||||
}
|
||||
|
||||
func FuzzDecryptForkPayload(f *testing.F) {
|
||||
key := bytes.Repeat([]byte{7}, 32)
|
||||
for _, seed := range []struct{ plain, aad string }{
|
||||
{`{"type":"default","keyPassword":"the-key-password"}`, forkPayloadAAD},
|
||||
{`{"keyPassword":""}`, forkPayloadAAD},
|
||||
{`{"keyPassword":null}`, forkPayloadAAD},
|
||||
{`{"type":"default"}`, forkPayloadAAD},
|
||||
{`not json`, forkPayloadAAD},
|
||||
{`{"keyPassword":"wrong additional data"}`, "other"},
|
||||
} {
|
||||
f.Add(sealForkPayload(f, key, seed.plain, seed.aad))
|
||||
}
|
||||
f.Add("")
|
||||
f.Add("not base64!")
|
||||
f.Add(base64.StdEncoding.EncodeToString(make([]byte, 12)))
|
||||
f.Fuzz(func(t *testing.T, payload string) {
|
||||
keyPassword, err := decryptForkPayload(payload, key)
|
||||
if err != nil {
|
||||
assert.Empty(t, keyPassword)
|
||||
return
|
||||
}
|
||||
// Success only for a payload sealed with key: open it here
|
||||
// independently.
|
||||
blob, err := base64.StdEncoding.DecodeString(payload)
|
||||
require.NoError(t, err)
|
||||
block, err := aes.NewCipher(key)
|
||||
require.NoError(t, err)
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
require.NoError(t, err)
|
||||
require.GreaterOrEqual(t, len(blob), gcm.NonceSize())
|
||||
plain, err := gcm.Open(nil, blob[:gcm.NonceSize()], blob[gcm.NonceSize():], []byte(forkPayloadAAD))
|
||||
require.NoError(t, err, "accepted a payload which doesn't authenticate")
|
||||
var p struct{ KeyPassword *string }
|
||||
require.NoError(t, json.Unmarshal(plain, &p))
|
||||
require.NotNil(t, p.KeyPassword)
|
||||
assert.Equal(t, *p.KeyPassword, keyPassword)
|
||||
})
|
||||
}
|
||||
@@ -33,6 +33,27 @@ var (
|
||||
legacyRouteRe = regexp.MustCompile(`^/drive/shares/[^/]+/(links|folders|files|events)(/|$)`)
|
||||
)
|
||||
|
||||
// legacyRequests counts the requests for legacy routes, per "METHOD
|
||||
// path", made to every server in the process, so a test binary can
|
||||
// check none was made after all its tests have run.
|
||||
var legacyRequests = struct {
|
||||
mu sync.Mutex
|
||||
counts map[string]int
|
||||
}{counts: make(map[string]int)}
|
||||
|
||||
// LegacyRequests returns a copy of the count, per "METHOD path", of
|
||||
// the requests for removed share-scoped routes made to every server in
|
||||
// the process since it started. ResetRequests doesn't change it.
|
||||
func LegacyRequests() map[string]int {
|
||||
legacyRequests.mu.Lock()
|
||||
defer legacyRequests.mu.Unlock()
|
||||
out := make(map[string]int, len(legacyRequests.counts))
|
||||
for k, v := range legacyRequests.counts {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Fault describes a scripted failure for requests to one route.
|
||||
type Fault struct {
|
||||
Method string // HTTP method to match; empty matches any
|
||||
@@ -262,6 +283,9 @@ func (s *Server) serveHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
switch {
|
||||
case legacyRouteRe.MatchString(r.URL.Path):
|
||||
legacyRequests.mu.Lock()
|
||||
legacyRequests.counts[r.Method+" "+r.URL.Path]++
|
||||
legacyRequests.mu.Unlock()
|
||||
writeError(w, http.StatusGone, 0, "protontest: legacy route "+r.URL.Path+" was removed")
|
||||
return
|
||||
case r.Header.Get("x-pm-drive-sdk-version") != "":
|
||||
|
||||
@@ -65,6 +65,24 @@ func TestLegacyRoutesGone(t *testing.T) {
|
||||
assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
|
||||
}
|
||||
|
||||
func TestLegacyRequestsProcessWide(t *testing.T) {
|
||||
before := LegacyRequests()
|
||||
s1, s2 := New(t), New(t)
|
||||
do(t, s1, "GET", "/drive/shares/abc/links/def", map[string]string{"x-pm-appversion": testAppVersion})
|
||||
do(t, s2, "GET", "/drive/shares/abc/links/def", map[string]string{"x-pm-appversion": testAppVersion})
|
||||
do(t, s2, "POST", "/drive/shares/abc/files", nil)
|
||||
do(t, s2, "GET", "/drive/shares/abc", nil)
|
||||
s1.ResetRequests()
|
||||
s2.ResetRequests()
|
||||
|
||||
after := LegacyRequests()
|
||||
assert.Equal(t, before["GET /drive/shares/abc/links/def"]+2, after["GET /drive/shares/abc/links/def"])
|
||||
assert.Equal(t, before["POST /drive/shares/abc/files"]+1, after["POST /drive/shares/abc/files"], "counted whatever its headers")
|
||||
assert.NotContains(t, after, "GET /drive/shares/abc")
|
||||
after["changed"] = 1
|
||||
assert.NotContains(t, LegacyRequests(), "changed", "a copy is returned")
|
||||
}
|
||||
|
||||
func TestUnknownRoute(t *testing.T) {
|
||||
s := New(t)
|
||||
resp, body := do(t, s, "GET", "/core/v4/nothing", map[string]string{"x-pm-appversion": testAppVersion})
|
||||
|
||||
@@ -10,13 +10,13 @@ import (
|
||||
|
||||
// newTestClient returns a client with caching enabled for a new user
|
||||
// of a new fake server, signed in without a login round trip.
|
||||
func newTestClient(t *testing.T) (*client, *protontest.Server, *protontest.User) {
|
||||
func newTestClient(t testing.TB) (*client, *protontest.Server, *protontest.User) {
|
||||
t.Helper()
|
||||
return newTestClientOpt(t, clientOptions{EnableCaching: true})
|
||||
}
|
||||
|
||||
// newTestClientOpt is newTestClient with the options opt.
|
||||
func newTestClientOpt(t *testing.T, opt clientOptions) (*client, *protontest.Server, *protontest.User) {
|
||||
func newTestClientOpt(t testing.TB, opt clientOptions) (*client, *protontest.Server, *protontest.User) {
|
||||
t.Helper()
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUser("user", "pass")
|
||||
@@ -24,7 +24,7 @@ func newTestClientOpt(t *testing.T, opt clientOptions) (*client, *protontest.Ser
|
||||
}
|
||||
|
||||
// newTestClientFor returns a client for a new session of u.
|
||||
func newTestClientFor(t *testing.T, srv *protontest.Server, u *protontest.User, opt clientOptions) *client {
|
||||
func newTestClientFor(t testing.TB, srv *protontest.Server, u *protontest.User, opt clientOptions) *client {
|
||||
t.Helper()
|
||||
c, err := newClient(context.Background(), newAccountSession(srv, u), opt)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -133,3 +133,17 @@ Small, specific, independently actionable. Remove only when actually done.
|
||||
`unlockAddressKey`) is decrypted without a decompression limit, unlike
|
||||
node metadata (`decryptMessage`); a hostile server could send a
|
||||
compression bomb there too. Seen in Task 17.
|
||||
- [ ] protondrive: API error bodies are read whole and their `Error`
|
||||
message is logged as the server sent it, of any length and with any
|
||||
control characters (session.go `errorHandler`). Bound and sanitise it
|
||||
if a hostile server is in scope for logs. Seen in Task 17.
|
||||
- [ ] protondrive: a new revision (`createRevision`), a move, rename or
|
||||
trash still forget the parent folder's cached listing, so `Put` over
|
||||
existing files in one folder lists it again each time (`sync` is not
|
||||
affected: it calls `Update`). New files and folders are added to the
|
||||
listing instead since Task 17.
|
||||
- [ ] protondrive: extended attributes with a SHA-1 but no `Size` make a
|
||||
file report size 0; the sum of `BlockSizes`, or the encrypted size,
|
||||
would be a better fallback. Seen in Task 17.
|
||||
- [ ] protondrive: spec 9.5 asks for a fuzz target for event decoding;
|
||||
add it with events (fuzz_internal_test.go).
|
||||
@@ -139,6 +139,10 @@ a test in the task shown.
|
||||
- `func (s *Server) Requests() map[string]int` - count per
|
||||
`"METHOD route-pattern"`, e.g. `"GET /drive/v2/volumes/{volumeID}/folders/{linkID}/children"`.
|
||||
- `func (s *Server) ResetRequests()`
|
||||
- `func LegacyRequests() map[string]int` - count per `"METHOD path"` of
|
||||
requests for removed share-scoped routes to every server in the
|
||||
process (added in Task 17 for the package's `TestMain` check);
|
||||
`ResetRequests` doesn't change it.
|
||||
- `type Fault struct { Method, Pattern string; Skip, Times int; Status, Code int; RetryAfter int; Drop bool; Delay time.Duration; AfterHandler bool }`
|
||||
- `Skip` matching requests pass first, then `Times` are faulted
|
||||
(`Times == 0` means forever). `Drop` closes the connection.
|
||||
@@ -1192,7 +1196,7 @@ Decisions:
|
||||
- Create: `backend/protondrive/budget_test.go`
|
||||
- Create: `backend/protondrive/fuzz_internal_test.go`
|
||||
|
||||
- [ ] **Step 1: Write budget tests**, using `Requests()` deltas after
|
||||
- [x] **Step 1: Write budget tests**, using `Requests()` deltas after
|
||||
`NewFs` (exclude bootstrap by `ResetRequests()`):
|
||||
- `TestBudgetList`: folder of N in {0, 1, 100, 101, 250} files with page
|
||||
size 150 costs at most `P + ceil(N/100) + 1` requests (P = child
|
||||
@@ -1206,14 +1210,14 @@ Decisions:
|
||||
- `TestBudgetNoLegacyRoutes`: after the whole `fstests` run, assert in a
|
||||
`TestMain` hook that no counted route matches the deprecated patterns.
|
||||
Record the measured values in a comment next to each bound.
|
||||
- [ ] **Step 2: Write fuzz targets** seeded from `testdata/oldlib`:
|
||||
- [x] **Step 2: Write fuzz targets** seeded from `testdata/oldlib`:
|
||||
`FuzzDecryptXAttrs`, `FuzzDecodeAPIError`, `FuzzDecryptForkPayload`,
|
||||
`FuzzDecodeLink` (link JSON -> `node`), `FuzzValidateName`. Each must not
|
||||
panic and must not return success with an invalid name.
|
||||
- [ ] **Step 3: Run** `go test ./backend/protondrive/ -run 'TestBudget'` and
|
||||
- [x] **Step 3: Run** `go test ./backend/protondrive/ -run 'TestBudget'` and
|
||||
`go test ./backend/protondrive/ -fuzz FuzzDecryptXAttrs -fuzztime 30s`
|
||||
(repeat per target) - fix findings.
|
||||
- [ ] **Step 4: Commit** `protondrive: add request budget tests and fuzz targets`.
|
||||
- [x] **Step 4: Commit** `protondrive: add request budget tests and fuzz targets`.
|
||||
|
||||
### Task 18: Wider hermetic suites and fork CI
|
||||
|
||||
|
||||
Reference in new issue
Block a user