-
Notifications
You must be signed in to change notification settings - Fork 1.4k
feat: add in-memory transport for chatd -> aibridge routing #25576
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
82c290d
8aa664b
846c27a
ee9ba43
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| package aibridge | ||
|
|
||
| import ( | ||
| "context" | ||
| "net/http" | ||
|
|
||
| "github.com/google/uuid" | ||
| ) | ||
|
|
||
| // Source identifies the call site that asked aibridge for a transport. It is | ||
| // attached to the request context so downstream handlers and logs can attribute | ||
| // traffic without changing behavior based on the value. | ||
| type Source string | ||
|
|
||
| // SourceAgents is chatd traffic originating from a Coder agent. | ||
| const SourceAgents Source = "agents" | ||
|
|
||
| type sourceCtxKey struct{} | ||
|
|
||
| // WithSource returns a copy of ctx carrying the given Source. Use this on the | ||
| // request context before invoking a downstream handler so [SourceFromContext] | ||
| // can recover it for logging. | ||
| func WithSource(ctx context.Context, src Source) context.Context { | ||
| return context.WithValue(ctx, sourceCtxKey{}, src) | ||
| } | ||
|
|
||
| // SourceFromContext returns the Source attached by [WithSource], or the empty | ||
| // string when no Source is set. | ||
| func SourceFromContext(ctx context.Context) Source { | ||
| src, _ := ctx.Value(sourceCtxKey{}).(Source) | ||
| return src | ||
| } | ||
|
|
||
| // TransportFactory returns an [http.RoundTripper] that dispatches an aibridge | ||
| // request in-process for a given ai_providers row. | ||
| // | ||
| // Implementations live in coderd/aibridged. coderd registers an in-process | ||
| // factory on coderd.API.AIBridgeTransportFactory at startup so callers route | ||
| // traffic through the daemon without going through the gated HTTP route. | ||
| // | ||
| // Source is informational: implementations must not gate on it. It is attached | ||
| // to the request context so handlers can include it in logs and metrics. | ||
| type TransportFactory interface { | ||
| TransportFor(providerID uuid.UUID, source Source) (http.RoundTripper, error) | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,101 @@ | ||
| package coderd_test | ||
|
|
||
| import ( | ||
| "io" | ||
| "net/http" | ||
| "net/http/httptest" | ||
| "testing" | ||
|
|
||
| "github.com/google/uuid" | ||
| "github.com/stretchr/testify/require" | ||
|
|
||
| "github.com/coder/coder/v2/coderd/aibridge" | ||
| "github.com/coder/coder/v2/coderd/coderdtest" | ||
| "github.com/coder/coder/v2/testutil" | ||
| ) | ||
|
|
||
| // stubTransportFactory wires a deterministic handler through the | ||
| // AIBridgeTransportFactory hook so the AGPL side of the in-memory pipe can be | ||
| // exercised without pulling coderd/aibridged in. | ||
| type stubTransportFactory struct { | ||
| handler http.Handler | ||
| calls chan callRecord | ||
| } | ||
|
|
||
| type callRecord struct { | ||
| providerID uuid.UUID | ||
| source aibridge.Source | ||
| } | ||
|
|
||
| func (f *stubTransportFactory) TransportFor(providerID uuid.UUID, source aibridge.Source) (http.RoundTripper, error) { | ||
| f.calls <- callRecord{providerID: providerID, source: source} | ||
| return &handlerRoundTripper{handler: f.handler}, nil | ||
| } | ||
|
|
||
| // handlerRoundTripper is a minimal http.RoundTripper for the AGPL test. It | ||
| // does not stream; coderd/aibridged.transport_test.go already covers | ||
| // streaming semantics. | ||
| type handlerRoundTripper struct{ handler http.Handler } | ||
|
|
||
| func (h *handlerRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { | ||
| rec := httptest.NewRecorder() | ||
| h.handler.ServeHTTP(rec, req) | ||
| resp := rec.Result() | ||
| resp.Request = req | ||
| return resp, nil | ||
| } | ||
|
|
||
| // Verify that a factory stored on coderd.API.AIBridgeTransportFactory is | ||
| // observable through the normal API lifecycle: cli/server.go registers it | ||
| // when the bridge daemon starts (see RegisterInMemoryAIBridgedHTTPHandler). | ||
| func TestAIBridgeTransportFactory_Registration(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| _, _, api := coderdtest.NewWithAPI(t, nil) | ||
|
|
||
| require.Nil(t, api.AIBridgeTransportFactory.Load(), | ||
| "AGPL coderd must not pre-populate the factory") | ||
|
|
||
| stub := &stubTransportFactory{ | ||
| handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||
| w.Header().Set("Content-Type", "application/json") | ||
| w.WriteHeader(http.StatusOK) | ||
| _, _ = w.Write([]byte(`{"bridged":true}`)) | ||
| }), | ||
| calls: make(chan callRecord, 4), | ||
| } | ||
|
|
||
| var asInterface aibridge.TransportFactory = stub | ||
| api.AIBridgeTransportFactory.Store(&asInterface) | ||
|
|
||
| loaded := api.AIBridgeTransportFactory.Load() | ||
| require.NotNil(t, loaded) | ||
|
|
||
| providerID := uuid.New() | ||
| rt, err := (*loaded).TransportFor(providerID, aibridge.SourceAgents) | ||
| require.NoError(t, err) | ||
| require.NotNil(t, rt) | ||
|
|
||
| select { | ||
| case got := <-stub.calls: | ||
| require.Equal(t, providerID, got.providerID) | ||
| require.Equal(t, aibridge.SourceAgents, got.source) | ||
| default: | ||
| t.Fatal("factory was not invoked") | ||
| } | ||
|
|
||
| ctx := testutil.Context(t, testutil.WaitShort) | ||
| req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://aibridge/v1/messages", nil) | ||
| require.NoError(t, err) | ||
|
|
||
| client := &http.Client{Transport: rt} | ||
| resp, err := client.Do(req) | ||
| require.NoError(t, err) | ||
| defer resp.Body.Close() | ||
|
|
||
| body, err := io.ReadAll(resp.Body) | ||
| require.NoError(t, err) | ||
| require.Equal(t, http.StatusOK, resp.StatusCode) | ||
| require.Equal(t, `{"bridged":true}`, string(body)) | ||
| require.Equal(t, "application/json", resp.Header.Get("Content-Type")) | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,172 @@ | ||
| package aibridged | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io" | ||
| "net/http" | ||
| "sync" | ||
|
|
||
| "github.com/google/uuid" | ||
| "golang.org/x/xerrors" | ||
|
|
||
| "github.com/coder/coder/v2/coderd/aibridge" | ||
| ) | ||
|
|
||
| // NewTransportFactory returns an [aibridge.TransportFactory] whose RoundTripper | ||
| // dispatches requests to handler in-process, streaming the response body | ||
| // through an [io.Pipe] so SSE/NDJSON/chunked responses propagate token-by-token | ||
| // just as they would over the wire. | ||
| // | ||
| // handler is typically the aibridged HTTP entrypoint registered via | ||
| // [API.RegisterInMemoryAIBridgedHTTPHandler]. | ||
| func NewTransportFactory(handler http.Handler) aibridge.TransportFactory { | ||
| return &transportFactory{handler: handler} | ||
| } | ||
|
|
||
| type transportFactory struct { | ||
| handler http.Handler | ||
| } | ||
|
|
||
| // TransportFor returns an in-process [http.RoundTripper] that dispatches | ||
| // requests through the aibridged handler. The source is attached to the | ||
| // request context for downstream logging; routing does not depend on it. | ||
| func (f *transportFactory) TransportFor(_ uuid.UUID, source aibridge.Source) (http.RoundTripper, error) { | ||
| if f.handler == nil { | ||
| return nil, xerrors.New("aibridged handler not registered") | ||
| } | ||
| return &inMemoryRoundTripper{handler: f.handler, source: source}, nil | ||
| } | ||
|
|
||
| // inMemoryRoundTripper implements [http.RoundTripper] by invoking handler | ||
| // in a goroutine and streaming its response back through an [io.Pipe]. | ||
| type inMemoryRoundTripper struct { | ||
| handler http.Handler | ||
| source aibridge.Source | ||
| } | ||
|
|
||
| func (t *inMemoryRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { | ||
| pr, pw := io.Pipe() | ||
| rw := &pipeResponseWriter{ | ||
| header: http.Header{}, | ||
| body: pw, | ||
| gotHeaders: make(chan struct{}), | ||
| status: http.StatusOK, | ||
| } | ||
|
|
||
| // Cloning preserves caller-supplied headers and context but lets the | ||
| // handler operate on its own request value without surprising the caller | ||
| // if it mutates Headers or stores the request. The Source is attached to | ||
| // the served context so downstream handlers can log the call site. | ||
| served := req.Clone(aibridge.WithSource(req.Context(), t.source)) | ||
|
|
||
| handlerDone := make(chan struct{}) | ||
| go func() { | ||
|
dannykopping marked this conversation as resolved.
|
||
| defer func() { | ||
| if r := recover(); r != nil { | ||
| // Mirror net/http.Server behavior: a panicking handler | ||
| // produces a 500 instead of crashing the process. | ||
| rw.WriteHeader(http.StatusInternalServerError) | ||
| _ = pw.CloseWithError(xerrors.Errorf("handler panicked: %v", r)) | ||
| } | ||
| // Make sure we always unblock RoundTrip even if the handler | ||
| // returns before writing headers (e.g. handler returns early | ||
| // without writing). | ||
| rw.ensureHeaders() | ||
| // If the request context was canceled, surface that as a | ||
| // body-read error so the caller sees a network-style failure | ||
| // rather than EOF. Otherwise close cleanly. | ||
| if cerr := served.Context().Err(); cerr != nil { | ||
| _ = pw.CloseWithError(cerr) | ||
| } else { | ||
| _ = pw.Close() | ||
| } | ||
| close(handlerDone) | ||
| }() | ||
| t.handler.ServeHTTP(rw, served) | ||
| }() | ||
|
|
||
| // Close the pipe eagerly when the caller cancels, so an unresponsive | ||
| // handler does not strand the consumer's body read. The handler's own | ||
| // context derives from req.Context(), so it observes the same | ||
| // cancellation independently. The goroutine also exits when the handler | ||
| // completes normally (handlerDone closes) to avoid leaking a parked | ||
| // goroutine per successful request. | ||
| go func() { | ||
|
dannykopping marked this conversation as resolved.
|
||
| select { | ||
| case <-served.Context().Done(): | ||
| _ = pw.CloseWithError(served.Context().Err()) | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. review: this is explicitly called out as being safe to call concurrently.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I suppose the defense-in-depth here is fine to leave as-is, right?
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yep! My comment was for future readers wondering "is this concurrently safe"? |
||
| case <-handlerDone: | ||
| // Handler finished; nothing to cancel. | ||
| } | ||
| }() | ||
|
|
||
| select { | ||
| case <-rw.gotHeaders: | ||
| case <-served.Context().Done(): | ||
| return nil, served.Context().Err() | ||
| } | ||
|
|
||
| return &http.Response{ | ||
| Status: fmt.Sprintf("%d %s", rw.status, http.StatusText(rw.status)), | ||
| StatusCode: rw.status, | ||
| Proto: "HTTP/1.1", | ||
| ProtoMajor: 1, | ||
| ProtoMinor: 1, | ||
| Header: rw.frozenHeader, | ||
| Body: pr, | ||
| Request: req, | ||
| ContentLength: -1, // streaming; unknown length | ||
| }, nil | ||
| } | ||
|
|
||
| // pipeResponseWriter is an [http.ResponseWriter] that streams the response | ||
| // body into an [io.PipeWriter]. The first call to WriteHeader (implicit or | ||
| // explicit) closes gotHeaders so the RoundTrip caller can return an | ||
| // *http.Response while the handler keeps writing. | ||
| type pipeResponseWriter struct { | ||
| header http.Header | ||
| frozenHeader http.Header | ||
| body *io.PipeWriter | ||
|
|
||
| once sync.Once | ||
| gotHeaders chan struct{} | ||
| status int | ||
| } | ||
|
|
||
| func (w *pipeResponseWriter) Header() http.Header { | ||
| return w.header | ||
| } | ||
|
|
||
| func (w *pipeResponseWriter) WriteHeader(status int) { | ||
| w.once.Do(func() { | ||
| w.status = status | ||
| w.frozenHeader = w.header.Clone() | ||
| close(w.gotHeaders) | ||
| }) | ||
| } | ||
|
|
||
| func (w *pipeResponseWriter) Write(p []byte) (int, error) { | ||
| // net/http semantics: an implicit 200 OK on first Write if the handler | ||
| // did not call WriteHeader explicitly. | ||
| w.WriteHeader(http.StatusOK) | ||
| return w.body.Write(p) | ||
| } | ||
|
|
||
| // Flush is a no-op: pipe writes are already synchronous with the reader, so | ||
| // each Write is observed as soon as the reader consumes it. We satisfy | ||
| // [http.Flusher] so handlers that type-assert it (the aibridge library does | ||
| // for SSE) do not fall back to buffered mode. | ||
| func (*pipeResponseWriter) Flush() {} | ||
|
|
||
| // ensureHeaders closes gotHeaders if it has not already been closed, with the | ||
| // current status. Used to unblock RoundTrip on handler return-without-write. | ||
| func (w *pipeResponseWriter) ensureHeaders() { | ||
| w.once.Do(func() { | ||
| close(w.gotHeaders) | ||
| }) | ||
| } | ||
|
Comment on lines
+161
to
+167
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Bit of a gotcha: after calling
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Right now this only happens on
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yep, non-blocking but just calling out. |
||
|
|
||
| var ( | ||
| _ http.ResponseWriter = (*pipeResponseWriter)(nil) | ||
| _ http.Flusher = (*pipeResponseWriter)(nil) | ||
| ) |
Uh oh!
There was an error while loading. Please reload this page.