protondrive: make the volume creation race test deterministic
TestClientBootstrapCreateVolumeRace failed now and then in whole package race runs with "expected 2 POST /drive/volumes, got 1". loadAccount holds the process-wide unlockMu while unlocking keys, so the second client's first GET drive/v2/shares/my-files is sent one key unlock, or under load many queued unlocks, after the first's. The 500ms Delay fault on both GETs shifted them equally and so did not keep them together: when the first client's create had finished before the second client's GET was handled, the second client found the volume and never POSTed. The client handles that ordering correctly; only the test's assertion depended on timing. Add a Gather option to protontest faults, which holds each faulted request until all Times have arrived, and use it for both GETs and both POSTs. The test now starts the second client as late as the server lets it, which failed 10 of 10 runs with the Delay fault. Gathering the POSTs runs both create volume checks in the fake at once, which exposed a data race there: go-crypto records signature checks in a key on its first use, and the new user's address key had never been used. Check each request with its own copy of the key. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e65c785c15
commit
e23de0b586
4 files changed
+86
-17
No files matched your search
@@ -120,17 +120,32 @@ func TestClientBootstrapCreateVolumeRace(t *testing.T) {
|
||||
srv := protontest.New(t)
|
||||
u := srv.AddUserWithoutDrive("new", "pass")
|
||||
// Both clients find no volume before either creates one, so one of
|
||||
// them loses the creation with code 2500.
|
||||
srv.Inject(protontest.Fault{Method: "GET", Pattern: "/drive/v2/shares/my-files", Times: 2, Delay: 500 * time.Millisecond})
|
||||
// them loses the creation with code 2500, and both creations are
|
||||
// checked at once.
|
||||
srv.Inject(protontest.Fault{Method: "GET", Pattern: "/drive/v2/shares/my-files", Times: 2, Gather: true})
|
||||
srv.Inject(protontest.Fault{Method: "POST", Pattern: "/drive/volumes", Times: 2, Gather: true})
|
||||
|
||||
// The second client starts once the first has finished, or after 2s
|
||||
// if the server holds the first back for it, as if its key unlock
|
||||
// were queued behind others on unlockMu, so its first look at the
|
||||
// drive comes as late as the server lets it.
|
||||
ctx, cancel := context.WithTimeout(t.Context(), time.Minute)
|
||||
defer cancel()
|
||||
firstDone := make(chan struct{})
|
||||
clients := make([]*client, 2)
|
||||
errs := make([]error, 2)
|
||||
var wg sync.WaitGroup
|
||||
for i := range clients {
|
||||
wg.Go(func() {
|
||||
clients[i], errs[i] = newClient(context.Background(), newAccountSession(srv, u), clientOptions{})
|
||||
})
|
||||
}
|
||||
wg.Go(func() {
|
||||
defer close(firstDone)
|
||||
clients[0], errs[0] = newClient(ctx, newAccountSession(srv, u), clientOptions{})
|
||||
})
|
||||
wg.Go(func() {
|
||||
select {
|
||||
case <-firstDone:
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
clients[1], errs[1] = newClient(ctx, newAccountSession(srv, u), clientOptions{})
|
||||
})
|
||||
wg.Wait()
|
||||
for i, c := range clients {
|
||||
require.NoError(t, errs[i])
|
||||
|
||||
@@ -1653,7 +1653,15 @@ func (s *Server) handleCreateVolume(w http.ResponseWriter, r *http.Request, u *U
|
||||
}
|
||||
addrKey = addr.keys[i]
|
||||
}
|
||||
if err := checkVolumeRequest(&req, addrKey.key, s.unlockKey); err != nil {
|
||||
// go-crypto records signature checks in the key on first use, so
|
||||
// concurrent checks each need their own copy of a key nothing has
|
||||
// used yet.
|
||||
checkKey, err := addrKey.key.Copy()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, 0, "protontest: copy address key: "+err.Error())
|
||||
return
|
||||
}
|
||||
if err := checkVolumeRequest(&req, checkKey, s.unlockKey); err != nil {
|
||||
writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidValue, "protontest: invalid volume key material: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
@@ -37,24 +37,29 @@ type Fault struct {
|
||||
Pattern string // route pattern as registered, e.g. "/tests/ping"
|
||||
Skip int // matching requests let through before faulting
|
||||
Times int // matching requests faulted after Skip; 0 means forever
|
||||
Status int // HTTP status to return; 0 with Drop false only delays
|
||||
Status int // HTTP status to return; 0 with Drop false only holds the request
|
||||
Code int // Proton code in the error body
|
||||
RetryAfter int // Retry-After header in seconds, if non-zero
|
||||
Drop bool // close the connection instead of responding
|
||||
Delay time.Duration // wait before handling the request
|
||||
Gather bool // hold each faulted request until all Times have arrived
|
||||
AfterHandler bool // run the real handler, then replace its response
|
||||
}
|
||||
|
||||
// fault is an injected Fault with the count of requests it has matched.
|
||||
type fault struct {
|
||||
Fault
|
||||
seen int
|
||||
seen int
|
||||
gathered chan struct{} // closed when the last of Times has arrived, if Gather
|
||||
}
|
||||
|
||||
// fires records a matching request and reports whether it is faulted.
|
||||
func (f *fault) fires() bool {
|
||||
f.seen++
|
||||
n := f.seen - f.Skip
|
||||
if f.Gather && n == f.Times {
|
||||
close(f.gathered)
|
||||
}
|
||||
return n > 0 && (f.Times == 0 || n <= f.Times)
|
||||
}
|
||||
|
||||
@@ -141,11 +146,14 @@ func (s *Server) ResetRequests() {
|
||||
}
|
||||
|
||||
// Inject adds a fault. Faults are checked in the order they were added
|
||||
// and the first one that fires is applied.
|
||||
// and the first one that fires is applied. A Gather fault needs Times.
|
||||
func (s *Server) Inject(f Fault) {
|
||||
if f.Gather && f.Times <= 0 {
|
||||
panic("protontest: a Gather fault needs Times")
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.faults = append(s.faults, &fault{Fault: f})
|
||||
s.faults = append(s.faults, &fault{Fault: f, gathered: make(chan struct{})})
|
||||
}
|
||||
|
||||
// SetClockSkew shifts the Date header of every response by d.
|
||||
@@ -208,29 +216,38 @@ func (s *Server) serveHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// activeFault records the request against every matching fault and
|
||||
// returns the first one that fires, or nil.
|
||||
func (s *Server) activeFault(method, pattern string) *Fault {
|
||||
func (s *Server) activeFault(method, pattern string) *fault {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var active *Fault
|
||||
var active *fault
|
||||
for _, f := range s.faults {
|
||||
if f.Pattern != pattern || (f.Method != "" && f.Method != method) {
|
||||
continue
|
||||
}
|
||||
if f.fires() && active == nil {
|
||||
active = &f.Fault
|
||||
active = f
|
||||
}
|
||||
}
|
||||
return active
|
||||
}
|
||||
|
||||
// serveFault applies f to a request for a registered route.
|
||||
func (s *Server) serveFault(w http.ResponseWriter, r *http.Request, f *Fault) {
|
||||
if f.Delay > 0 {
|
||||
func (s *Server) serveFault(w http.ResponseWriter, r *http.Request, f *fault) {
|
||||
if f.Delay > 0 || f.Gather {
|
||||
// The server notices a client giving up, and cancels the request
|
||||
// context, only once the request body has been read.
|
||||
if body, err := io.ReadAll(r.Body); err == nil {
|
||||
r.Body = io.NopCloser(bytes.NewReader(body))
|
||||
}
|
||||
}
|
||||
if f.Gather {
|
||||
select {
|
||||
case <-f.gathered:
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
if f.Delay > 0 {
|
||||
select {
|
||||
case <-time.After(f.Delay):
|
||||
case <-r.Context().Done():
|
||||
|
||||
@@ -177,6 +177,35 @@ func TestFaultInjection(t *testing.T) {
|
||||
assert.GreaterOrEqual(t, time.Since(start), 100*time.Millisecond)
|
||||
})
|
||||
|
||||
t.Run("Gather", func(t *testing.T) {
|
||||
s := New(t)
|
||||
s.Inject(Fault{Pattern: "/tests/ping", Times: 2, Gather: true})
|
||||
req, err := http.NewRequest("GET", s.URL()+"/tests/ping", nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("x-pm-appversion", testAppVersion)
|
||||
first := make(chan int)
|
||||
go func() {
|
||||
resp, err := s.Client().Do(req)
|
||||
if err != nil {
|
||||
first <- 0
|
||||
return
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
first <- resp.StatusCode
|
||||
}()
|
||||
select {
|
||||
case <-first:
|
||||
t.Fatal("first request answered before the second arrived")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
resp, _ := ping(t, s)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
assert.Equal(t, http.StatusOK, <-first)
|
||||
// Requests after Times are not held.
|
||||
resp, _ = ping(t, s)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
})
|
||||
|
||||
t.Run("AfterHandler", func(t *testing.T) {
|
||||
s := New(t)
|
||||
called := 0
|
||||
|
||||
Reference in new issue
Block a user