Implementing Chat Streaming

app.MapPost("/api/chat/stream", async (
    ChatRequest request,
    AgentFrameworkService agentService,
    HttpContext httpContext,
    CancellationToken cancellationToken) =>
{
    httpContext.Response.Headers.Append("Content-Type", "text/event-stream");
    httpContext.Response.Headers.Append("Cache-Control", "no-cache");
    
    var conversationId = request.ConversationId 
        ?? await agentService.CreateConversationAsync(request.Message, cancellationToken);
    
    // Send conversation ID first
    await httpContext.Response.WriteAsync(
        $"data: {{\"type\":\"conversationId\",\"conversationId\":\"{conversationId}\"}}\n\n", 
        cancellationToken);
    await httpContext.Response.Body.FlushAsync(cancellationToken);
    
    // Stream chunks
    await foreach (var chunk in agentService.StreamMessageAsync(
        conversationId, request.Message, request.ImageDataUris, cancellationToken))
    {
        var json = JsonSerializer.Serialize(new { type = "chunk", content = chunk });
        await httpContext.Response.WriteAsync($"data: {json}\n\n", cancellationToken);
        await httpContext.Response.Body.FlushAsync(cancellationToken);
    }
    
    await httpContext.Response.WriteAsync("data: {\"type\":\"done\"}\n\n", cancellationToken);
})
.RequireAuthorization("RequireChatScope");

public async IAsyncEnumerable<StreamChunk> StreamMessageAsync(
    string conversationId,
    string message,
    List<string>? imageDataUris = null,
    [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
    ObjectDisposedException.ThrowIf(_disposed, this);
    
    // Stream response - yields StreamChunk with text deltas OR annotations
    await foreach (var update in responsesClient.CreateResponseStreamingAsync(...))
    {
        if (update is StreamingResponseOutputTextDeltaUpdate deltaUpdate)
        {
            yield return StreamChunk.Text(deltaUpdate.Delta);
        }
        else if (update is StreamingResponseOutputItemDoneUpdate itemDoneUpdate)
        {
            var annotations = ExtractAnnotations(itemDoneUpdate.Item, fileSearchQuotes);
            if (annotations.Count > 0)
            {
                yield return StreamChunk.WithAnnotations(annotations);
            }
        }
    }
}

CHAT_SEND_MESSAGE 
  → CHAT_ADD_ASSISTANT_MESSAGE 
  → CHAT_START_STREAM 
  → (repeat CHAT_STREAM_CHUNK) 
  → CHAT_STREAM_ANNOTATIONS (optional, for citations)
  → CHAT_STREAM_COMPLETE (with usage metrics)

app.MapPost("/api/chat/stream", async (
    ChatRequest request,
    AgentFrameworkService agentService,
    HttpContext httpContext,
    IHostEnvironment env,
    CancellationToken cancellationToken) =>
{
    httpContext.Response.Headers.Append("Content-Type", "text/event-stream");
    httpContext.Response.Headers.Append("Cache-Control", "no-cache");
    
    var conversationId = request.ConversationId 
        ?? await agentService.CreateConversationAsync(request.Message, cancellationToken);
    
    await httpContext.Response.WriteAsync(
        $"data: {{\"type\":\"conversationId\",\"conversationId\":\"{conversationId}\"}}\n\n", 
        cancellationToken);
    await httpContext.Response.Body.FlushAsync(cancellationToken);
    
    await foreach (var chunk in agentService.StreamMessageAsync(
        conversationId, request.Message, request.ImageDataUris, cancellationToken))
    {
        var json = System.Text.Json.JsonSerializer.Serialize(new { type = "chunk", content = chunk });
        await httpContext.Response.WriteAsync($"data: {json}\n\n", cancellationToken);
        await httpContext.Response.Body.FlushAsync(cancellationToken);
    }
    
    await httpContext.Response.WriteAsync("data: {\"type\":\"done\"}\n\n", cancellationToken);
})
.RequireAuthorization("RequireChatScope")
.WithName("StreamChatMessage");

CHAT_SEND_MESSAGE 
  → CHAT_ADD_ASSISTANT_MESSAGE 
  → CHAT_START_STREAM 
  → (repeat CHAT_STREAM_CHUNK) 
  → CHAT_STREAM_ANNOTATIONS (optional, for citations)
  → CHAT_STREAM_COMPLETE (with usage: promptTokens, completionTokens, totalTokens, duration)

🔄 [HH:MM:SS] ACTION_TYPE
Action: { … }
Changes: { field: before → after }