Using custom LLM driver

Overview

You can register and use your own LLM driver, either using a new driver name or overriding an existing driver bundled with Polyglot.

Example

<?php
require 'examples/boot.php';

use Cognesy\Config\Env;
use Cognesy\Http\Data\HttpRequest;
use Cognesy\Http\Data\HttpResponse;
use Cognesy\Messages\Messages;
use Cognesy\Polyglot\Inference\Config\LLMConfig;
use Cognesy\Polyglot\Inference\Creation\BundledInferenceDrivers;
use Cognesy\Polyglot\Inference\Drivers\InferenceDriverSpec;
use Cognesy\Polyglot\Inference\Drivers\OpenAI\OpenAIBodyFormat;
use Cognesy\Polyglot\Inference\Drivers\SpecifiedInferenceDriver;
use Cognesy\Polyglot\Inference\Inference;
use Cognesy\Polyglot\Inference\InferenceRuntime;
use Cognesy\Utils\Str;

// A provider that speaks the OpenAI wire protocol needs no driver class of its own -- an
// InferenceDriverSpec names the pieces and the runtime assembles them. To change *behaviour*
// rather than composition, subclass SpecifiedInferenceDriver and name it in the spec.

final class EchoingDriver extends SpecifiedInferenceDriver
{
    #[\Override]
    protected function makeHttpResponse(HttpRequest $request): HttpResponse
    {
        // some extra functionality to demonstrate our driver is being used
        echo ">>> Handling request...\n";

        return parent::makeHttpResponse($request);
    }
}

$drivers = BundledInferenceDrivers::registry()->withDriver(
    name: 'custom-driver',
    driver: new InferenceDriverSpec(
        bodyFormat: OpenAIBodyFormat::class,
        driverClass: EchoingDriver::class,
    ),
);

// withDriver() still accepts a class-string or a closure, exactly as before -- a spec is a
// third option, not a replacement:
//
//     driver: fn($config, $httpClient, $events) => new MyDriver($config, $httpClient, $events),
//     driver: MyDriver::class,

// Create instance of LLM client initialized with custom parameters
$config = new LLMConfig(
    apiUrl          : 'https://api.openai.com/v1',
    apiKey          : (string) Env::get('OPENAI_API_KEY', ''),
    endpoint        : '/chat/completions', model: 'gpt-4o-mini', maxTokens: 128,
    driver          : 'custom-driver',
);

$answer = Inference::fromRuntime(InferenceRuntime::fromConfig($config, drivers: $drivers))
    ->withMessages(Messages::fromString('What is the capital of France'))
    ->withOptions(['max_tokens' => 64])
    ->get();

echo "USER: What is capital of France\n";
echo "ASSISTANT: $answer\n";

assert(Str::contains($answer, 'Paris'));
?>