protondrive: make the volume creation race test deterministic

TestClientBootstrapCreateVolumeRace failed now and then in whole
package race runs with "expected 2 POST /drive/volumes, got 1".

loadAccount holds the process-wide unlockMu while unlocking keys, so
the second client's first GET drive/v2/shares/my-files is sent one
key unlock, or under load many queued unlocks, after the first's. The
500ms Delay fault on both GETs shifted them equally and so did not
keep them together: when the first client's create had finished
before the second client's GET was handled, the second client found
the volume and never POSTed. The client handles that ordering
correctly; only the test's assertion depended on timing.

Add a Gather option to protontest faults, which holds each faulted
request until all Times have arrived, and use it for both GETs and
both POSTs. The test now starts the second client as late as the
server lets it, which failed 10 of 10 runs with the Delay fault.

Gathering the POSTs runs both create volume checks in the fake at
once, which exposed a data race there: go-crypto records signature
checks in a key on its first use, and the new user's address key had
never been used. Check each request with its own copy of the key.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Lucas WintherandClaude Opus 5.5 committed 2026-11-11 16:37:14 +01:00
1 parent e65c785c15
commit e23de0b586
4 files changed
+86 -17

No files matched your search

+22 -7
View File
@@ -120,17 +120,32 @@ func TestClientBootstrapCreateVolumeRace(t *testing.T) {
srv := protontest.New(t)
u := srv.AddUserWithoutDrive("new", "pass")
// Both clients find no volume before either creates one, so one of
// them loses the creation with code 2500.
srv.Inject(protontest.Fault{Method: "GET", Pattern: "/drive/v2/shares/my-files", Times: 2, Delay: 500 * time.Millisecond})
// them loses the creation with code 2500, and both creations are
// checked at once.
srv.Inject(protontest.Fault{Method: "GET", Pattern: "/drive/v2/shares/my-files", Times: 2, Gather: true})
srv.Inject(protontest.Fault{Method: "POST", Pattern: "/drive/volumes", Times: 2, Gather: true})
// The second client starts once the first has finished, or after 2s
// if the server holds the first back for it, as if its key unlock
// were queued behind others on unlockMu, so its first look at the
// drive comes as late as the server lets it.
ctx, cancel := context.WithTimeout(t.Context(), time.Minute)
defer cancel()
firstDone := make(chan struct{})
clients := make([]*client, 2)
errs := make([]error, 2)
var wg sync.WaitGroup
for i := range clients {
wg.Go(func() {
clients[i], errs[i] = newClient(context.Background(), newAccountSession(srv, u), clientOptions{})
})
}
wg.Go(func() {
defer close(firstDone)
clients[0], errs[0] = newClient(ctx, newAccountSession(srv, u), clientOptions{})
})
wg.Go(func() {
select {
case <-firstDone:
case <-time.After(2 * time.Second):
}
clients[1], errs[1] = newClient(ctx, newAccountSession(srv, u), clientOptions{})
})
wg.Wait()
for i, c := range clients {
require.NoError(t, errs[i])
+9 -1
View File
@@ -1653,7 +1653,15 @@ func (s *Server) handleCreateVolume(w http.ResponseWriter, r *http.Request, u *U
}
addrKey = addr.keys[i]
}
if err := checkVolumeRequest(&req, addrKey.key, s.unlockKey); err != nil {
// go-crypto records signature checks in the key on first use, so
// concurrent checks each need their own copy of a key nothing has
// used yet.
checkKey, err := addrKey.key.Copy()
if err != nil {
writeError(w, http.StatusInternalServerError, 0, "protontest: copy address key: "+err.Error())
return
}
if err := checkVolumeRequest(&req, checkKey, s.unlockKey); err != nil {
writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidValue, "protontest: invalid volume key material: "+err.Error())
return
}
+26 -9
View File
@@ -37,24 +37,29 @@ type Fault struct {
Pattern string // route pattern as registered, e.g. "/tests/ping"
Skip int // matching requests let through before faulting
Times int // matching requests faulted after Skip; 0 means forever
Status int // HTTP status to return; 0 with Drop false only delays
Status int // HTTP status to return; 0 with Drop false only holds the request
Code int // Proton code in the error body
RetryAfter int // Retry-After header in seconds, if non-zero
Drop bool // close the connection instead of responding
Delay time.Duration // wait before handling the request
Gather bool // hold each faulted request until all Times have arrived
AfterHandler bool // run the real handler, then replace its response
}
// fault is an injected Fault with the count of requests it has matched.
type fault struct {
Fault
seen int
seen int
gathered chan struct{} // closed when the last of Times has arrived, if Gather
}
// fires records a matching request and reports whether it is faulted.
func (f *fault) fires() bool {
f.seen++
n := f.seen - f.Skip
if f.Gather && n == f.Times {
close(f.gathered)
}
return n > 0 && (f.Times == 0 || n <= f.Times)
}
@@ -141,11 +146,14 @@ func (s *Server) ResetRequests() {
}
// Inject adds a fault. Faults are checked in the order they were added
// and the first one that fires is applied.
// and the first one that fires is applied. A Gather fault needs Times.
func (s *Server) Inject(f Fault) {
if f.Gather && f.Times <= 0 {
panic("protontest: a Gather fault needs Times")
}
s.mu.Lock()
defer s.mu.Unlock()
s.faults = append(s.faults, &fault{Fault: f})
s.faults = append(s.faults, &fault{Fault: f, gathered: make(chan struct{})})
}
// SetClockSkew shifts the Date header of every response by d.
@@ -208,29 +216,38 @@ func (s *Server) serveHTTP(w http.ResponseWriter, r *http.Request) {
// activeFault records the request against every matching fault and
// returns the first one that fires, or nil.
func (s *Server) activeFault(method, pattern string) *Fault {
func (s *Server) activeFault(method, pattern string) *fault {
s.mu.Lock()
defer s.mu.Unlock()
var active *Fault
var active *fault
for _, f := range s.faults {
if f.Pattern != pattern || (f.Method != "" && f.Method != method) {
continue
}
if f.fires() && active == nil {
active = &f.Fault
active = f
}
}
return active
}
// serveFault applies f to a request for a registered route.
func (s *Server) serveFault(w http.ResponseWriter, r *http.Request, f *Fault) {
if f.Delay > 0 {
func (s *Server) serveFault(w http.ResponseWriter, r *http.Request, f *fault) {
if f.Delay > 0 || f.Gather {
// The server notices a client giving up, and cancels the request
// context, only once the request body has been read.
if body, err := io.ReadAll(r.Body); err == nil {
r.Body = io.NopCloser(bytes.NewReader(body))
}
}
if f.Gather {
select {
case <-f.gathered:
case <-r.Context().Done():
return
}
}
if f.Delay > 0 {
select {
case <-time.After(f.Delay):
case <-r.Context().Done():
@@ -177,6 +177,35 @@ func TestFaultInjection(t *testing.T) {
assert.GreaterOrEqual(t, time.Since(start), 100*time.Millisecond)
})
t.Run("Gather", func(t *testing.T) {
s := New(t)
s.Inject(Fault{Pattern: "/tests/ping", Times: 2, Gather: true})
req, err := http.NewRequest("GET", s.URL()+"/tests/ping", nil)
require.NoError(t, err)
req.Header.Set("x-pm-appversion", testAppVersion)
first := make(chan int)
go func() {
resp, err := s.Client().Do(req)
if err != nil {
first <- 0
return
}
_ = resp.Body.Close()
first <- resp.StatusCode
}()
select {
case <-first:
t.Fatal("first request answered before the second arrived")
case <-time.After(100 * time.Millisecond):
}
resp, _ := ping(t, s)
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, http.StatusOK, <-first)
// Requests after Times are not held.
resp, _ = ping(t, s)
assert.Equal(t, http.StatusOK, resp.StatusCode)
})
t.Run("AfterHandler", func(t *testing.T) {
s := New(t)
called := 0