WIP: Implement streaming completions

lib/llm/providers/openai.rb

@@ -30,8 +30,20 @@ def complete(message, **params)
       params = DEFAULT_PARAMS.merge(params)
       body = {messages: messages.map(&:to_h)}.merge!(params)
       req = preflight(req, body)
-      res = request(@http, req)
-      Response::Completion.new(res.body, self).extend(response_parser)
+      if params[:stream]
+        Fiber.new do
+          @http.request(req) do |res|
+            res.read_body do |chunk|
+              chunk.scan(/^data:(.+)$/).each do |match|
+                Fiber.yield Response::Chunk.new(match[0], self).extend(response_parser)
+              end
+            end
+          end
+        end
+      else
+        res = request(@http, req)
+        Response::Completion.new(res.body, self).extend(response_parser)
+      end
     end
 
     private

lib/llm/providers/openai/response_parser.rb

@@ -26,5 +26,18 @@ def parse_completion(raw)
         total_tokens: raw.dig("usage", "total_tokens")
       }
     end
+
+    ##
+    # @param [Hash] raw
+    #  The raw response from the LLM provider
+    # @return [Hash]
+    def parse_completion_chunk(raw)
+      {
+        model: raw["model"],
+        choices: raw["choices"].map do
+          LLM::Message.new(*_1["delta"].values_at("role", "content"))
+        end
+      }
+    end
   end
 end

lib/llm/response.rb

@@ -4,6 +4,7 @@ module LLM
   class Response
     require "json"
     require_relative "response/completion"
+    require_relative "response/chunk"
     require_relative "response/embedding"
 
     ##

lib/llm/response/chunk.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module LLM
+  class Response::Chunk < Response::Completion
+    private
+
+    ##
+    # @private
+    # @return [Hash]
+    #  Returns the parsed completion response from the provider
+    def parsed
+      @parsed ||= parse_completion_chunk(raw)
+    end
+  end
+end