Testing AI Features: Strategies for Non-Deterministic Systems

The Testing Challenge

Traditional tests expect deterministic outputs. AI systems are inherently non-deterministic—the same input can produce different outputs. Testing requires different strategies.

Property-Based Testing

class AIOutputTest extends TestCase
{
    public function test_product_description_has_required_properties(): void
    {
        $product = Product::factory()->create();
        $description = $this->generator->generate($product);

        // Test properties, not exact content
        $this->assertGreaterThan(100, str_word_count($description));
        $this->assertLessThan(300, str_word_count($description));
        $this->assertStringContainsString($product->name, $description);
        $this->assertDoesNotMatchRegularExpression('/[\[\]{}]/', $description); // No artifacts
    }

    public function test_sentiment_analysis_categorizes_correctly(): void
    {
        $positive = "I absolutely love this product! Best purchase ever!";
        $negative = "Terrible quality, complete waste of money.";

        $this->assertEquals('positive', $this->analyzer->analyze($positive)['sentiment']);
        $this->assertEquals('negative', $this->analyzer->analyze($negative)['sentiment']);
    }
}

Mocking AI Services

class ChatbotTest extends TestCase
{
    public function test_chatbot_handles_greeting(): void
    {
        $this->mock(AIClient::class)
            ->shouldReceive('chat')
            ->andReturn('Hello! How can I help you today?');

        $response = $this->chatbot->respond('Hi there!');

        $this->assertStringContainsString('help', strtolower($response));
    }
}

Golden Set Testing

class GoldenSetTest extends TestCase
{
    private array $goldenSet = [
        [
            'input' => 'What are your business hours?',
            'expected_contains' => ['9', '5', 'Monday', 'Friday'],
        ],
        [
            'input' => 'How do I return an item?',
            'expected_contains' => ['return', '30 days', 'refund'],
        ],
    ];

    public function test_ai_responses_cover_key_information(): void
    {
        foreach ($this->goldenSet as $case) {
            $response = $this->ai->generate($case['input']);

            foreach ($case['expected_contains'] as $expected) {
                $this->assertStringContainsStringIgnoringCase(
                    $expected,
                    $response,
                    "Response to '{$case['input']}' should contain '{$expected}'"
                );
            }
        }
    }
}

Regression Testing

class AIRegressionTest extends TestCase
{
    public function test_model_quality_above_baseline(): void
    {
        $testSet = TestCase::all();
        $scores = [];

        foreach ($testSet as $case) {
            $response = $this->ai->generate($case->input);
            $scores[] = $this->evaluator->score($response, $case->expected);
        }

        $average = array_sum($scores) / count($scores);
        $baseline = config('ai.quality_baseline', 0.85);

        $this->assertGreaterThanOrEqual($baseline, $average);
    }
}

Snapshot Testing

public function test_prompt_template_produces_expected_structure(): void
{
    $prompt = PromptBuilder::productDescription(
        Product::factory()->make(['name' => 'Test Product'])
    );

    $this->assertMatchesSnapshot($prompt);
}

public function test_ai_config_unchanged(): void
{
    $config = [
        'model' => config('ai.model'),
        'temperature' => config('ai.temperature'),
        'max_tokens' => config('ai.max_tokens'),
    ];

    $this->assertMatchesSnapshot($config);
}

Conclusion

Testing AI features requires testing properties, not exact outputs. Use mocks for unit tests, property tests for behavior, golden sets for critical paths, and regression tests to prevent quality degradation.