feat: expose methods, add grok, add deepseek

Author: Ma11hewThomas

README.md

@@ -2,16 +2,28 @@
 
 Generate AI test summaries using leading AI models from OpenAI and Anthropic Claude. Integrate with Developer tooling to provide AI summaries where you need them.
 
-## **⭐⭐ If you find this project useful, consider giving it a GitHub star ⭐⭐**
-
-## You can help us grow
-
-Support our mission to enhance test reporting with AI summaries by:
-
-- **⭐ Starring this repository to show your support. ⭐**
-- **🙌 Following our [GitHub page here](https://github.com/ctrf-io) 🙌**
-
-Thank you! Your support is invaluable to us! 💙
+<div align="center">
+<div style="padding: 1.5rem; border-radius: 8px; margin: 1rem 0; border: 1px solid #30363d;">
+<span style="font-size: 23px;">💚</span>
+<h3 style="margin: 1rem 0;">CTRF tooling is open source and free to use</h3>
+<p style="font-size: 16px;">You can support the project with a follow and a star</p>
+
+<div style="margin-top: 1.5rem;">
+<a href="https://github.com/ctrf-io/ai-test-reporter">
+<img src="https://img.shields.io/github/stars/ctrf-io/ai-test-reporter?style=for-the-badge&color=2ea043" alt="GitHub stars">
+</a>
+<a href="https://github.com/ctrf-io">
+<img src="https://img.shields.io/github/followers/ctrf-io?style=for-the-badge&color=2ea043" alt="GitHub followers">
+</a>
+</div>
+</div>
+
+<p style="font-size: 14px; margin: 1rem 0;">
+Maintained by <a href="https://github.com/ma11hewthomas">Matthew Thomas</a><br/>
+Contributions are very welcome! <br/>
+Explore more <a href="https://www.ctrf.io/integrations">integrations</a>
+</p>
+</div>
 
 ## Key Features
 
@@ -140,6 +152,71 @@ A message is sent to Claude for each failed test.
 
 `consolidate`: Consolidate and summarize multiple AI summaries into a higher-level overview (default: true)
 
+## Grok
+
+Run the following command:
+
+```bash
+npx ai-ctrf grok <path-to-ctrf-report>
+```
+
+
+An AI summary for each failed test will be added to your test report.
+
+The package interacts with the Grok API, you must set `GROK_API_KEY` and optionally `GROK_API_BASE_URL` environment variables.
+
+You will be responsible for any charges incurred from using Grok. Make sure you are aware of the associated costs.
+
+A message is sent to Grok for each failed test.
+
+### Grok Options
+
+`--model`: Grok model to use (default: grok-1).
+
+`--systemPrompt`: Custom system prompt to guide the AI response.
+
+`--maxTokens`: Maximum number of tokens for the response.
+
+`--temperature`: Sampling temperature.
+
+`--log`: Whether to log the AI responses to the console (default: true).
+
+`--maxMessages`: Limit the number of failing tests to send for summarization in the LLM request. This helps avoid overwhelming the model when dealing with reports that have many failing tests. (default: 10)
+
+`consolidate`: Consolidate and summarize multiple AI summaries into a higher-level overview (default: true)
+
+## DeepSeek
+
+Run the following command:
+
+```bash
+npx ai-ctrf deepseek <path-to-ctrf-report>
+```
+
+An AI summary for each failed test will be added to your test report.
+
+The package interacts with the DeepSeek API, you must set `DEEPSEEK_API_KEY` and optionally `DEEPSEEK_API_BASE_URL` environment variables.
+
+You will be responsible for any charges incurred from using DeepSeek. Make sure you are aware of the associated costs.
+
+A message is sent to DeepSeek for each failed test.
+
+### DeepSeek Options
+
+`--model`: DeepSeek model to use (default: deepseek-coder).
+
+`--systemPrompt`: Custom system prompt to guide the AI response.
+
+`--maxTokens`: Maximum number of tokens for the response.
+
+`--temperature`: Sampling temperature.
+
+`--log`: Whether to log the AI responses to the console (default: true).
+
+`--maxMessages`: Limit the number of failing tests to send for summarization in the LLM request. This helps avoid overwhelming the model when dealing with reports that have many failing tests. (default: 10)
+
+`consolidate`: Consolidate and summarize multiple AI summaries into a higher-level overview (default: true)
+
 ## CTRF Report Example
 
 ``` json

package-lock.json

@@ -1,8 +1,9 @@
 {
   "name": "ai-ctrf",
-  "version": "0.0.5",
+  "version": "0.0.6",
   "description": "AI Test Reporter - Create human-readable summaries of test results with LLMs like OpenAI GPT",
-  "main": "index.js",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
     "lint": "eslint . --ext .ts --fix",
@@ -39,5 +40,12 @@
     "@anthropic-ai/sdk": "^0.27.2",
     "openai": "^4.57.0",
     "yargs": "^17.7.2"
+  },
+  "exports": {
+    ".": {
+      "require": "./dist/index.js",
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
   }
 }

package.json

@@ -42,3 +42,19 @@ export function stripAnsi(message: string) {
 
 	return message.replace(ansiRegex(), '');
 }
+
+export function generateFailedTestPrompt(test: any, report: CtrfReport): string {
+    return `Analyze this test failure:
+
+Test Name: ${test.name}
+Test Tool: ${report.results.tool.name}
+${report.results.environment ? `Environment: ${report.results.environment}` : ""}
+
+Failure Details:
+${JSON.stringify(test, null, 2)}
+
+What I need:
+1. What specifically failed in this test
+2. The likely root cause based on the error messages and context
+3. The potential impact of this failure on the system`;
+}

src/common.ts

@@ -1,9 +1,11 @@
 import { CtrfReport } from "../types/ctrf";
 import { Arguments } from "./index";
 import { openAI } from "./models/openai";
-import { saveUpdatedReport } from "./common";
 import { claudeAI } from "./models/claude";
 import { azureOpenAI } from "./models/azure-openai";
+import { grokAI } from "./models/grok";
+import { deepseekAI } from "./models/deepseek";
+// import { CONSOLIDATED_SUMMARY_SYSTEM_PROMPT } from "./constants";
 
 export async function generateConsolidatedSummary(report: CtrfReport, file: string, model: string, args: Arguments) {
     const failedTests = report.results.tests.filter(test => test.status === 'failed');
@@ -21,6 +23,25 @@ export async function generateConsolidatedSummary(report: CtrfReport, file: stri
                  - Adding generic conclusions or advice such as "By following these steps..."
                  - headings, bullet points, or special formatting.`
     const consolidatedPrompt = `The following tests failed in the suite:\n\n${aiSummaries.join("\n")}\n\nA total of ${failedTests.length} tests failed in this test suite. Please provide a high-level summary of what went wrong across the suite and suggest what might be the root causes or patterns.`;
+
+    //     const systemPrompt = CONSOLIDATED_SUMMARY_SYSTEM_PROMPT;
+    //     const consolidatedPrompt = `Analyze these ${failedTests.length} test failures from our test suite:
+
+    // Test Environment: ${report.results.environment || 'Not specified'}
+    // Test Tool: ${report.results.tool.name}
+    // Total Tests Run: ${report.results.tests.length}
+    // Failed Tests: ${failedTests.length}
+
+    // Failed Test Details:
+    // ${aiSummaries.join("\n")}
+
+    // Key Questions to Address:
+    // 1. What patterns or common themes exist across these failures?
+    // 3. Is there evidence of a broader system change causing these failures?
+    // 4. How should I prioritize the issues to address based on the failure patterns?
+
+    // Please provide a high-level analysis of the test suite failures, focusing on systemic issues and patterns.`;
+
     let consolidatedSummary = ""
 
     if (model === 'openai') {
@@ -29,6 +50,10 @@ export async function generateConsolidatedSummary(report: CtrfReport, file: stri
         consolidatedSummary = await claudeAI(systemPrompt, consolidatedPrompt, args) || ""
     } else if (model === 'azure') {
         consolidatedSummary = await azureOpenAI(systemPrompt, consolidatedPrompt, args) || ""
+    } else if (model === 'grok') {
+        consolidatedSummary = await grokAI(systemPrompt, consolidatedPrompt, args) || ""
+    } else if (model === 'deepseek') {
+        consolidatedSummary = await deepseekAI(systemPrompt, consolidatedPrompt, args) || ""
     }
 
     if (consolidatedSummary) {

src/consolidated-summary.ts

@@ -0,0 +1,23 @@
+export const CONSOLIDATED_SUMMARY_SYSTEM_PROMPT = `You are tasked with analyzing multiple test failures across a test run. Your goal is to provide a concise, high-level synthesis that identifies common patterns, potential root causes, and system-wide issues. Focus on correlations between failures and broader system implications.
+
+Avoid:
+ - Including code snippets or technical implementation details
+ - Generic testing advice or best practices
+ - Bullet points, headings, or special formatting
+ - Repeating individual test failure details`
+
+export const FAILED_TEST_SUMMARY_SYSTEM_PROMPT = `You are tasked with analyzing a specific test failure from a CTRF report. Your goal is to generate a clear, actionable summary that helps developers understand and fix the issue quickly.
+
+When analyzing the failure:
+- Start your response with "The test failed because"
+- Keep your explanation conversational and natural
+- Focus on the exact error message and stack trace provided without reinterpreting them
+- Identify the specific root cause based on the provided information
+- Suggest concrete steps for resolution that directly relate to the failure
+
+Avoid:
+- Including code snippets or stack traces in your response
+- Adding generic conclusions or advice
+- Using bullet points, headings, or special formatting
+- Making assumptions beyond the provided information
+- Including implementation details or debugging steps`

src/constants.ts

@@ -5,7 +5,9 @@ import { openAIFailedTestSummary } from './models/openai';
 import { azureFailedTestSummary } from './models/azure-openai';
 import { validateCtrfFile } from './common';
 import { claudeFailedTestSummary } from './models/claude';
-import { generateConsolidatedSummary } from './consolidated-summary';
+import { grokFailedTestSummary } from './models/grok';
+import { deepseekFailedTestSummary } from './models/deepseek';
+import { FAILED_TEST_SUMMARY_SYSTEM_PROMPT } from './constants';
 
 export interface Arguments {
     _: Array<string | number>;
@@ -73,6 +75,36 @@ const argv: Arguments = yargs(hideBin(process.argv))
                 });
         }
     )
+    .command(
+        'grok <file>',
+        'Generate test summary from a CTRF report using Grok',
+        (yargs) => {
+            return yargs.positional('file', {
+                describe: 'Path to the CTRF file',
+                type: 'string',
+            })
+            .option('model', {
+                describe: 'Grok model to use',
+                type: 'string',
+                default: 'grok-2-latest',
+            });
+        }
+    )
+    .command(
+        'deepseek <file>',
+        'Generate test summary from a CTRF report using DeepSeek',
+        (yargs) => {
+            return yargs.positional('file', {
+                describe: 'Path to the CTRF file',
+                type: 'string',
+            })
+            .option('model', {
+                describe: 'DeepSeek model to use',
+                type: 'string',
+                default: 'deepseek-reasoner',
+            });
+        }
+    )
     .option('systemPrompt', {
         describe: 'System prompt to guide the AI',
         type: 'string',
@@ -159,4 +191,24 @@ if (argv._.includes('openai') && argv.file) {
     } catch (error) {
         console.error('Failed to read file:', error);
     }
-}
+} else if (argv._.includes('grok') && argv.file) {
+    try {
+        const report = validateCtrfFile(argv.file);
+        if (report !== null) {
+            grokFailedTestSummary(report, file, argv);
+        }
+    } catch (error) {
+        console.error('Failed to read file:', error);
+    }
+} else if (argv._.includes('deepseek') && argv.file) {
+    try {
+        const report = validateCtrfFile(argv.file);
+        if (report !== null) {
+            deepseekFailedTestSummary(report, file, argv);
+        }
+    } catch (error) {
+        console.error('Failed to read file:', error);
+    }
+} 
+
+export { openAIFailedTestSummary, claudeFailedTestSummary, azureFailedTestSummary, grokFailedTestSummary, deepseekFailedTestSummary };

src/index.ts

@@ -37,7 +37,7 @@ export async function azureOpenAI(systemPrompt: string, prompt: string, args: Ar
     }
 }
 
-export async function azureFailedTestSummary(report: CtrfReport, file: string, args: Arguments) {
+export async function azureFailedTestSummary(report: CtrfReport, file: string, args: Arguments): Promise<CtrfReport> {
     const failedTests = report.results.tests.filter(test => test.status === 'failed');
 
     let logged = false;
@@ -71,4 +71,5 @@ export async function azureFailedTestSummary(report: CtrfReport, file: string, a
         await generateConsolidatedSummary(report, file, "azure", args)
     }
     saveUpdatedReport(file, report);
+    return report;
 }

src/models/azure-openai.ts

@@ -33,7 +33,7 @@ export async function claudeAI(systemPrompt: string, prompt: string, args: Argum
     }
 }
 
-export async function claudeFailedTestSummary(report: CtrfReport, file: string, args: Arguments) {
+export async function claudeFailedTestSummary(report: CtrfReport, file: string, args: Arguments): Promise<CtrfReport> {
     const failedTests = report.results.tests.filter(test => test.status === 'failed');
 
     let logged = false;
@@ -67,4 +67,5 @@ export async function claudeFailedTestSummary(report: CtrfReport, file: string,
         await generateConsolidatedSummary(report, file, "claude", args)
     }
     saveUpdatedReport(file, report);
+    return report;
 }