fix(readiness): degrade optional fleet providers without blocking fallback
This commit is contained in:
@@ -26,8 +26,9 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
type Probe struct {
|
||||
Name string
|
||||
Check func(context.Context) error
|
||||
Name string
|
||||
Check func(context.Context) error
|
||||
Optional bool
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
@@ -59,11 +60,13 @@ func (h ReadinessHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
err := probe.Check(ctx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
ready = false
|
||||
result.State, result.Reason, result.Detail = "DEGRADED", "DEPENDENCY_UNAVAILABLE", err.Error()
|
||||
if !probe.Optional {
|
||||
ready = false
|
||||
}
|
||||
}
|
||||
}
|
||||
if result.State != "READY" {
|
||||
if result.State != "READY" && !probe.Optional {
|
||||
ready = false
|
||||
}
|
||||
results = append(results, result)
|
||||
@@ -96,6 +99,15 @@ func HTTPProbe(client *http.Client, endpoint string) Probe {
|
||||
}}
|
||||
}
|
||||
|
||||
// OptionalHTTPProbe reports dependency degradation without making the whole
|
||||
// runtime unavailable. This is used for opportunistic fleet members when a
|
||||
// qualified local fallback remains available.
|
||||
func OptionalHTTPProbe(client *http.Client, endpoint string) Probe {
|
||||
probe := HTTPProbe(client, endpoint)
|
||||
probe.Optional = true
|
||||
return probe
|
||||
}
|
||||
|
||||
func TCPProbe(name, host, port string) Probe {
|
||||
return Probe{Name: name, Check: func(ctx context.Context) error {
|
||||
connection, err := (&net.Dialer{}).DialContext(ctx, "tcp", net.JoinHostPort(host, port))
|
||||
|
||||
@@ -26,3 +26,16 @@ func TestReadinessReturns200WhenAllProbesPass(t *testing.T) {
|
||||
t.Fatalf("readiness response = %d %s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadinessStaysAvailableWithOptionalDependencyDegraded(t *testing.T) {
|
||||
handler := ReadinessHandler{Runtime: "test", Probes: []Probe{
|
||||
{Name: "lifeboat", Check: func(context.Context) error { return nil }},
|
||||
{Name: "opportunistic-gpu", Optional: true, Check: func(context.Context) error { return errors.New("timeout") }},
|
||||
}}
|
||||
recorder := httptest.NewRecorder()
|
||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/readyz", nil))
|
||||
body := recorder.Body.String()
|
||||
if recorder.Code != http.StatusOK || !strings.Contains(body, `"status":"ready"`) || !strings.Contains(body, `"state":"DEGRADED"`) {
|
||||
t.Fatalf("readiness response = %d %s", recorder.Code, body)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user