From 816208fd9c119bab27a9e93b8d59e4e988f6aed5 Mon Sep 17 00:00:00 2001 From: Yashwin Date: Mon, 4 May 2026 15:42:36 +0530 Subject: [PATCH] feat: add prompt test case management and integrate AI provider model resolution logic --- src/components/layout/Layout.tsx | 2 +- src/components/shared/FormTagInput.tsx | 66 ++++ src/components/shared/MarkdownViewer.tsx | 96 +++++ src/components/shared/PageHeader.tsx | 2 +- src/components/shared/index.ts | 2 + .../tenant/PromptTestCaseResultModal.tsx | 117 ++++++ .../tenant/PromptTestCaseResultsListModal.tsx | 151 ++++++++ src/pages/tenant/CompletionCreate.tsx | 51 ++- src/pages/tenant/CreateDocument.tsx | 65 +--- src/pages/tenant/PromptCreate.tsx | 101 +++--- src/pages/tenant/PromptEdit.tsx | 78 ++-- src/pages/tenant/PromptManagement.tsx | 7 +- src/pages/tenant/PromptTestCaseCreate.tsx | 342 ++++++++++++++++++ src/pages/tenant/PromptTestCases.tsx | 326 +++++++++++++++++ src/routes/tenant-admin-routes.tsx | 24 +- src/services/ai-service.ts | 29 ++ src/types/ai.ts | 15 + 17 files changed, 1319 insertions(+), 155 deletions(-) create mode 100644 src/components/shared/FormTagInput.tsx create mode 100644 src/components/shared/MarkdownViewer.tsx create mode 100644 src/components/tenant/PromptTestCaseResultModal.tsx create mode 100644 src/components/tenant/PromptTestCaseResultsListModal.tsx create mode 100644 src/pages/tenant/PromptTestCaseCreate.tsx create mode 100644 src/pages/tenant/PromptTestCases.tsx diff --git a/src/components/layout/Layout.tsx b/src/components/layout/Layout.tsx index aed4a8f..e0f3868 100644 --- a/src/components/layout/Layout.tsx +++ b/src/components/layout/Layout.tsx @@ -9,7 +9,7 @@ interface LayoutProps { currentPage: string; breadcrumbs?: Array<{ label: string; path?: string }>; pageHeader?: { - title: string; + title: React.ReactNode; description?: string; tabs?: TabItem[]; action?: React.ReactNode; diff --git a/src/components/shared/FormTagInput.tsx b/src/components/shared/FormTagInput.tsx new file mode 100644 index 0000000..4304703 --- /dev/null +++ b/src/components/shared/FormTagInput.tsx @@ -0,0 +1,66 @@ +import type { ReactElement } from "react"; +import { X } from "lucide-react"; + +interface FormTagInputProps { + label?: string; + value: string[]; + onChange: (value: string[]) => void; + error?: string; + placeholder?: string; + className?: string; +} + +export const FormTagInput = ({ + label, + value = [], + onChange, + error, + placeholder = "Type and press enter...", +}: FormTagInputProps): ReactElement => { + return ( +
+ {label && ( + + )} +
+ {value.map((tag, tagIdx) => ( + + {tag} + + + ))} + { + if (e.key === "Enter") { + e.preventDefault(); + const val = e.currentTarget.value.trim(); + if (val && !value.includes(val)) { + onChange([...value, val]); + e.currentTarget.value = ""; + } + } + }} + /> +
+ {error && ( +

{error}

+ )} +
+ ); +}; diff --git a/src/components/shared/MarkdownViewer.tsx b/src/components/shared/MarkdownViewer.tsx new file mode 100644 index 0000000..ef45ae4 --- /dev/null +++ b/src/components/shared/MarkdownViewer.tsx @@ -0,0 +1,96 @@ +import type { ReactElement } from "react"; + +interface MarkdownViewerProps { + content: string; + className?: string; +} + +export const MarkdownViewer = ({ + content, + className = "", +}: MarkdownViewerProps): ReactElement => { + if (!content) return <>; + + const parseBold = (text: string) => { + if (!text) return ""; + const parts = text.split("**"); + return parts.map((part, i) => { + if (i % 2 === 1) { + return ( + + {part} + + ); + } + return part; + }); + }; + + const parseLine = (line: string, lineIndex: number) => { + // 1. Check for headings + if (line.startsWith("### ")) { + return ( +

+ {parseBold(line.slice(4))} +

+ ); + } + if (line.startsWith("## ")) { + return ( +

+ {parseBold(line.slice(3))} +

+ ); + } + if (line.startsWith("# ")) { + return ( +

+ {parseBold(line.slice(2))} +

+ ); + } + + // 2. Check for bullet lists + if (line.trim().startsWith("- ") || line.trim().startsWith("* ")) { + const cleanLine = line.trim().startsWith("- ") + ? line.trim().slice(2) + : line.trim().slice(2); + return ( +
  • + {parseBold(cleanLine)} +
    • + ); + } + + // 3. Regular paragraph + return ( +

    + {parseBold(line)} +

    + ); + }; + + // Split content by newline and process each line + const lines = content.split("\n"); + + return ( +
    + {lines.map((line, index) => parseLine(line, index))} +
    + ); +}; diff --git a/src/components/shared/PageHeader.tsx b/src/components/shared/PageHeader.tsx index 478d9c2..5a4fda8 100644 --- a/src/components/shared/PageHeader.tsx +++ b/src/components/shared/PageHeader.tsx @@ -10,7 +10,7 @@ export interface TabItem { } interface PageHeaderProps { - title: string; + title: React.ReactNode; description?: string; tabs?: TabItem[]; action?: React.ReactNode; diff --git a/src/components/shared/index.ts b/src/components/shared/index.ts index 374bd16..4f3e6a6 100644 --- a/src/components/shared/index.ts +++ b/src/components/shared/index.ts @@ -41,3 +41,5 @@ export type { FileUploadModalProps } from './FileUploadModal'; export { FileShareModal } from './FileShareModal'; export { ActiveOnlyToggle } from './ActiveOnlyToggle'; export { SearchBox } from './SearchBox'; +export { FormTagInput } from './FormTagInput'; +export { MarkdownViewer } from './MarkdownViewer'; diff --git a/src/components/tenant/PromptTestCaseResultModal.tsx b/src/components/tenant/PromptTestCaseResultModal.tsx new file mode 100644 index 0000000..210fd05 --- /dev/null +++ b/src/components/tenant/PromptTestCaseResultModal.tsx @@ -0,0 +1,117 @@ +import type { ReactElement } from "react"; +import { Modal, MarkdownViewer } from "@/components/shared"; +import { Cpu, DollarSign, Clock, CheckCircle } from "lucide-react"; + +interface PromptTestCaseResultModalProps { + isOpen: boolean; + onClose: () => void; + result: any; +} + +export const PromptTestCaseResultModal = ({ + isOpen, + onClose, + result, +}: PromptTestCaseResultModalProps): ReactElement | null => { + if (!result) return null; + + return ( + +
    + {/* Performance & Usage Metrics Cards */} +
    + {/* Provider Card */} +
    + + Provider & Model + + + {result.provider || "N/A"} + + + {result.model || "N/A"} + +
    + + {/* Latency Card */} +
    + + Latency + + + {result.latency_ms ? `${(result.latency_ms / 1000).toFixed(2)}s` : "N/A"} + + Response time +
    + + {/* Tokens Card */} +
    + + Token Usage + + + {result.usage?.total_tokens || result.usage?.totalTokens || 0} + + + In: {result.usage?.prompt_tokens || result.usage?.promptTokens || 0} • Out:{" "} + {result.usage?.completion_tokens || result.usage?.completionTokens || 0} + +
    + + {/* Cost Card */} +
    + + Cost + + + {result.cost !== undefined ? `$${result.cost}` : "$0.00"} + + Total API cost +
    +
    + + {/* Prompt Input / Rendered Prompt */} +
    + + Rendered Prompt + +
    + {result.renderedPrompt || result.prompt || "No rendered prompt available."} +
    +
    + + {/* Expected Output */} + {result.expectedOutput && ( +
    + + Expected Output + +
    + +
    +
    + )} + + {/* Response Content */} +
    + + LLM Response Content + +
    + {result.content || result.response ? ( + + ) : ( + "No output response content." + )} +
    +
    +
    +     + ); +}; diff --git a/src/components/tenant/PromptTestCaseResultsListModal.tsx b/src/components/tenant/PromptTestCaseResultsListModal.tsx new file mode 100644 index 0000000..1082d4e --- /dev/null +++ b/src/components/tenant/PromptTestCaseResultsListModal.tsx @@ -0,0 +1,151 @@ +import { useEffect, useState, type ReactElement } from "react"; +import { Modal, DataTable, type Column, MarkdownViewer } from "@/components/shared"; +import { aiService } from "@/services/ai-service"; +import { formatDate } from "@/utils/format-date"; +import { showToast } from "@/utils/toast"; + +interface PromptTestCaseResultsListModalProps { + isOpen: boolean; + onClose: () => void; + testCaseId: string; + testCaseName: string; +} + +export const PromptTestCaseResultsListModal = ({ + isOpen, + onClose, + testCaseId, + testCaseName, +}: PromptTestCaseResultsListModalProps): ReactElement | null => { + const [results, setResults] = useState([]); + const [isLoading, setIsLoading] = useState(false); + const [error, setError] = useState(null); + const [expandedRows, setExpandedRows] = useState>({}); + + const loadResults = async () => { + if (!testCaseId) return; + setIsLoading(true); + setError(null); + try { + const data = await aiService.listPromptTestCaseResults(testCaseId, 10); + setResults(data || []); + } catch (err: any) { + const message = + err?.response?.data?.error?.message || "Failed to load test case results"; + setError(message); + showToast.error(message); + } finally { + setIsLoading(false); + } + }; + + useEffect(() => { + if (isOpen && testCaseId) { + void loadResults(); + setExpandedRows({}); + } + }, [isOpen, testCaseId]); + + const toggleRowExpansion = (rowId: string) => { + setExpandedRows((prev) => ({ + ...prev, + [rowId]: !prev[rowId], + })); + }; + + const columns: Column[] = [ + { + key: "created_at", + label: "Timestamp", + render: (row) => ( + + {formatDate(row.created_at)} + + ), + }, + { + key: "provider_model", + label: "Provider / Model", + render: (row) => ( +
    + + {row.provider || "N/A"} + + + {row.model || "N/A"} + +
    + ), + }, + { + key: "usage", + label: "Usage & Cost", + render: (row) => ( +
    + + {row.tokens_used ? `${row.tokens_used} tokens` : "0 tokens"} + + + {row.cost_usd && Number(row.cost_usd) > 0 ? `$${row.cost_usd}` : "$0.00"} + +
    + ), + }, + { + key: "latency", + label: "Latency", + render: (row) => ( + + {row.latency_ms ? `${(row.latency_ms / 1000).toFixed(2)}s` : "N/A"} + + ), + }, + { + key: "run_by", + label: "Run By", + render: (row) => ( + + {row.run_by_email || row.run_by || "System"} + + ), + }, + ]; + + return ( + +
    + item.id} + isLoading={isLoading} + error={error} + emptyMessage="No test results recorded yet." + expandableRows={true} + isRowExpanded={(item) => !!expandedRows[item.id]} + onRowExpandToggle={(item) => toggleRowExpansion(item.id)} + showExpandColumn={true} + expandedColSpan={columns.length + 1} + renderExpandedRow={(item) => ( +
    + + Output + + {item.output ? ( + + ) : ( + "No output generated." + )} +
    + )} + /> +
    +     + ); +}; diff --git a/src/pages/tenant/CompletionCreate.tsx b/src/pages/tenant/CompletionCreate.tsx index 1010028..dd84976 100644 --- a/src/pages/tenant/CompletionCreate.tsx +++ b/src/pages/tenant/CompletionCreate.tsx @@ -20,8 +20,7 @@ const playgroundSchema = z.object({ type PlaygroundFormData = z.infer; const CompletionCreate = (): ReactElement => { - const [providers, setProviders] = useState([]); - const [models, setModels] = useState>([]); + const [apiProviders, setApiProviders] = useState([]); const [isLoading, setIsLoading] = useState(false); const [isSending, setIsSending] = useState(false); const [isPlaygroundMode, setIsPlaygroundMode] = useState(true); @@ -43,8 +42,6 @@ const CompletionCreate = (): ReactElement => { }, }); - const formValues = watch(); - const [lastSentUserMessage, setLastSentUserMessage] = useState(""); const [displayedResponse, setDisplayedResponse] = useState(""); const typingIntervalRef = useRef | null>(null); @@ -61,30 +58,41 @@ const CompletionCreate = (): ReactElement => { fallbackUsed: false, }); + const selectedProvider = watch("provider"); + const providerOptions = useMemo( - () => providers.map((p) => ({ value: p.name, label: p.displayName || p.name })), - [providers], + () => apiProviders.map((p) => ({ value: p.name, label: p.displayName || p.name })), + [apiProviders], ); + const providerDetail = apiProviders.find((p) => p.name === selectedProvider); const modelOptions = useMemo( () => - models - .filter((m) => !formValues.provider || m.provider === formValues.provider) - .map((m) => ({ - value: m.id, - label: `${m.id}${m.isDefault ? " • default" : ""}`, - })), - [models, formValues.provider], + (providerDetail?.models || []).map((m) => ({ + value: m, + label: `${m}${m === providerDetail?.defaultModel ? " • default" : ""}`, + })), + [providerDetail], ); const loadOptions = async (): Promise => { setIsLoading(true); try { - const [providerData, modelData] = await Promise.all([aiService.getProviders(), aiService.getModels()]); - setProviders(providerData); - setModels(modelData); + const providerData = await aiService.getProviders(); + setApiProviders(providerData); + + if (providerData.length > 0) { + const initialProvider = "gemini"; + const matchedProvider = providerData.find((p) => p.name === initialProvider) || providerData[0]; + setValue("provider", matchedProvider.name); + if (matchedProvider.defaultModel) { + setValue("model", matchedProvider.defaultModel); + } else if (matchedProvider.models && matchedProvider.models.length > 0) { + setValue("model", matchedProvider.models[0]); + } + } } catch (err: any) { - showToast.error(err?.response?.data?.error?.message || "Failed to load provider/model options"); + showToast.error(err?.response?.data?.error?.message || "Failed to load provider options"); } finally { setIsLoading(false); } @@ -310,7 +318,14 @@ const CompletionCreate = (): ReactElement => { options={providerOptions} onValueChange={(value) => { field.onChange(value); - setValue("model", ""); + const pDetail = apiProviders.find((p) => p.name === value); + if (pDetail && pDetail.defaultModel) { + setValue("model", pDetail.defaultModel); + } else if (pDetail && pDetail.models && pDetail.models.length > 0) { + setValue("model", pDetail.models[0]); + } else { + setValue("model", ""); + } }} /> )} diff --git a/src/pages/tenant/CreateDocument.tsx b/src/pages/tenant/CreateDocument.tsx index 1d6a3f6..e768ae4 100644 --- a/src/pages/tenant/CreateDocument.tsx +++ b/src/pages/tenant/CreateDocument.tsx @@ -10,6 +10,7 @@ import { FormTextArea, PrimaryButton, RichTextEditor, + FormTagInput, } from "@/components/shared"; import { documentService, @@ -17,7 +18,7 @@ import { } from "@/services/document-service"; import type { DocumentCategory } from "@/types/document"; import { showToast } from "@/utils/toast"; -import { ArrowLeft, FileText, Info, Paperclip, X } from "lucide-react"; +import { ArrowLeft, FileText, Info, Paperclip } from "lucide-react"; import { moduleService } from "@/services/module-service"; import type { MyModule } from "@/types/module"; @@ -292,58 +293,18 @@ const CreateDocument = (): ReactElement => { error={errors.department?.message} {...register("department")} /> -
    - - ( -
    - {(field.value || []).map((tag, tagIdx) => ( - - {tag} - - - ))} - { - if (e.key === "Enter") { - e.preventDefault(); - const val = e.currentTarget.value.trim(); - if (val && !(field.value || []).includes(val)) { - field.onChange([...(field.value || []), val]); - e.currentTarget.value = ""; - } - } - }} - /> -
    - )} - /> - {errors.tags && ( -

    {errors.tags.message}

    + ( + )} -
    + /> ; const PromptCreate = (): ReactElement => { const navigate = useNavigate(); const [isSubmitting, setIsSubmitting] = useState(false); - const [providers, setProviders] = useState< - Array<{ value: string; label: string }> - >([]); - const [models, setModels] = useState>( - [], - ); + const [apiProviders, setApiProviders] = useState([]); const { control, handleSubmit, + setValue, + watch, formState: { errors }, } = useForm({ resolver: zodResolver(promptSchema), @@ -67,7 +66,7 @@ const PromptCreate = (): ReactElement => { model: "", temperature: 0.7, max_tokens: 2048, - tags: "", + tags: [], is_default: true, variables: [ { @@ -85,25 +84,24 @@ const PromptCreate = (): ReactElement => { name: "variables", }); + const selectedProvider = watch("provider"); + + const providersOptions = apiProviders.map((p) => ({ + value: p.name, + label: p.displayName || p.name, + })); + + const providerDetail = apiProviders.find((p) => p.name === selectedProvider); + const modelsOptions = (providerDetail?.models || []).map((m) => ({ + value: m, + label: m, + })); + useEffect(() => { const loadMeta = async (): Promise => { try { - const [providerData, modelData] = await Promise.all([ - aiService.getProviders(), - aiService.getModels(), - ]); - setProviders( - providerData.map((p) => ({ - value: p.name, - label: p.displayName || p.name, - })), - ); - setModels( - modelData.map((m) => ({ - value: m.id, - label: `${m.id} (${m.provider})`, - })), - ); + const providerData = await aiService.getProviders(); + setApiProviders(providerData); } catch (err: unknown) { showToast.error( (err as { response?: { data?: { error?: { message?: string } } } }) @@ -116,12 +114,7 @@ const PromptCreate = (): ReactElement => { }, []); const onFormSubmit = async (data: PromptFormData): Promise => { - const parsedTags = data.tags - ? data.tags - .split(",") - .map((tag) => tag.trim()) - .filter(Boolean) - : []; + const parsedTags = data.tags || []; const sanitizedVariables = data.variables ?.filter((v) => v.name.trim()) @@ -168,8 +161,15 @@ const PromptCreate = (): ReactElement => { + Create Prompt + + Draft + + + ), + description: "Manage and reuse prompts for different use cases", action: (
    navigate("/tenant/ai/prompts")}> @@ -440,7 +440,9 @@ const PromptCreate = (): ReactElement => { />
    - +
    {
    )} /> - Make default for this use case + + Make default for this use case +
    @@ -480,8 +486,18 @@ const PromptCreate = (): ReactElement => { { + field.onChange(val); + const pDetail = apiProviders.find((p) => p.name === val); + if (pDetail && pDetail.defaultModel) { + setValue("model", pDetail.defaultModel); + } else if (pDetail && pDetail.models && pDetail.models.length > 0) { + setValue("model", pDetail.models[0]); + } else { + setValue("model", ""); + } + }} + options={providersOptions} error={errors.provider?.message} /> )} @@ -495,7 +511,7 @@ const PromptCreate = (): ReactElement => { label="Model" value={field.value || ""} onValueChange={field.onChange} - options={models} + options={modelsOptions} error={errors.model?.message} /> )} @@ -543,11 +559,10 @@ const PromptCreate = (): ReactElement => { name="tags" control={control} render={({ field }) => ( - )} diff --git a/src/pages/tenant/PromptEdit.tsx b/src/pages/tenant/PromptEdit.tsx index 98e523d..994d7c2 100644 --- a/src/pages/tenant/PromptEdit.tsx +++ b/src/pages/tenant/PromptEdit.tsx @@ -8,6 +8,7 @@ import { PrimaryButton, SecondaryButton, FormSlider, + FormTagInput } from "@/components/shared"; import { Plus, Trash2, ArrowLeft } from "lucide-react"; import { aiService } from "@/services/ai-service"; @@ -16,6 +17,7 @@ import { useForm, useFieldArray, Controller } from "react-hook-form"; import { zodResolver } from "@hookform/resolvers/zod"; import { z } from "zod"; import { cn } from "@/lib/utils"; +import type { AIProviderInfo } from "@/types/ai"; const variableSchema = z.object({ name: z.string().min(1, "Variable name is required").max(100), @@ -34,7 +36,7 @@ const promptSchema = z.object({ model: z.string().optional(), temperature: z.number().min(0).max(2), max_tokens: z.number().int().min(1).max(128000), - tags: z.string().optional(), + tags: z.array(z.string()), is_default: z.boolean(), variables: z.array(variableSchema), change_notes: z.string().optional(), @@ -47,17 +49,14 @@ const PromptEdit = (): ReactElement => { const navigate = useNavigate(); const [isLoading, setIsLoading] = useState(true); const [isSubmitting, setIsSubmitting] = useState(false); - const [providers, setProviders] = useState< - Array<{ value: string; label: string }> - >([]); - const [models, setModels] = useState>( - [], - ); + const [apiProviders, setApiProviders] = useState([]); const [currentVersion, setCurrentVersion] = useState(1); const { control, handleSubmit, + setValue, + watch, reset, formState: { errors }, } = useForm({ @@ -72,7 +71,7 @@ const PromptEdit = (): ReactElement => { model: "", temperature: 0.7, max_tokens: 2048, - tags: "", + tags: [], is_default: false, variables: [], change_notes: "", @@ -84,29 +83,30 @@ const PromptEdit = (): ReactElement => { name: "variables", }); + const selectedProvider = watch("provider"); + + const providersOptions = apiProviders.map((p) => ({ + value: p.name, + label: p.displayName || p.name, + })); + + const providerDetail = apiProviders.find((p) => p.name === selectedProvider); + const modelsOptions = (providerDetail?.models || []).map((m) => ({ + value: m, + label: m, + })); + useEffect(() => { const loadData = async (): Promise => { if (!id) return; try { setIsLoading(true); - const [providerData, modelData, promptData] = await Promise.all([ + const [providerData, promptData] = await Promise.all([ aiService.getProviders(), - aiService.getModels(), aiService.getPrompt(id), ]); - setProviders( - providerData.map((p) => ({ - value: p.name, - label: p.displayName || p.name, - })), - ); - setModels( - modelData.map((m) => ({ - value: m.id, - label: `${m.id} (${m.provider})`, - })), - ); + setApiProviders(providerData); reset({ name: promptData.name, @@ -118,7 +118,7 @@ const PromptEdit = (): ReactElement => { model: promptData.defaultParameters?.model || "", temperature: promptData.defaultParameters?.temperature ?? 0.7, max_tokens: promptData.defaultParameters?.max_tokens ?? 2048, - tags: (promptData.tags || []).join(", "), + tags: promptData.tags || [], is_default: promptData.isDefault || false, variables: (promptData.variables || []).map((v) => ({ name: v.name, @@ -145,12 +145,7 @@ const PromptEdit = (): ReactElement => { const onFormSubmit = async (data: PromptFormData): Promise => { if (!id) return; - const parsedTags = data.tags - ? data.tags - .split(",") - .map((tag) => tag.trim()) - .filter(Boolean) - : []; + const parsedTags = data.tags || []; const sanitizedVariables = data.variables ?.filter((v) => v.name.trim()) @@ -543,8 +538,18 @@ const PromptEdit = (): ReactElement => { { + field.onChange(val); + const pDetail = apiProviders.find((p) => p.name === val); + if (pDetail && pDetail.defaultModel) { + setValue("model", pDetail.defaultModel); + } else if (pDetail && pDetail.models && pDetail.models.length > 0) { + setValue("model", pDetail.models[0]); + } else { + setValue("model", ""); + } + }} + options={providersOptions} error={errors.provider?.message} /> )} @@ -558,7 +563,7 @@ const PromptEdit = (): ReactElement => { label="Model" value={field.value || ""} onValueChange={field.onChange} - options={models} + options={modelsOptions} error={errors.model?.message} /> )} @@ -606,11 +611,10 @@ const PromptEdit = (): ReactElement => { name="tags" control={control} render={({ field }) => ( - )} diff --git a/src/pages/tenant/PromptManagement.tsx b/src/pages/tenant/PromptManagement.tsx index 81c5a56..b2553cf 100644 --- a/src/pages/tenant/PromptManagement.tsx +++ b/src/pages/tenant/PromptManagement.tsx @@ -1,6 +1,6 @@ import { useEffect, useMemo, useState, type ReactElement } from "react"; import { useNavigate } from "react-router-dom"; -import { Plus, Search, Copy, History, Trash2, Edit3 } from "lucide-react"; +import { Plus, Search, Copy, History, Trash2, Edit3, TestTube } from "lucide-react"; import { Layout } from "@/components/layout/Layout"; import { DataTable, @@ -224,6 +224,11 @@ const PromptManagement = (): ReactElement => {
    , + label: "Prompt Test Cases", + onClick: () => navigate(`/tenant/ai/prompts/${row.id}/test-cases`), + }, { icon: , label: "Edit Prompt", diff --git a/src/pages/tenant/PromptTestCaseCreate.tsx b/src/pages/tenant/PromptTestCaseCreate.tsx new file mode 100644 index 0000000..deb6182 --- /dev/null +++ b/src/pages/tenant/PromptTestCaseCreate.tsx @@ -0,0 +1,342 @@ +import { useEffect, useState, type ReactElement } from "react"; +import { useNavigate, useParams } from "react-router-dom"; +import { Layout } from "@/components/layout/Layout"; +import { + FormField, + FormTextArea, + FormTagInput, + PrimaryButton, + SecondaryButton, +} from "@/components/shared"; +import { ArrowLeft } from "lucide-react"; +import { aiService } from "@/services/ai-service"; +import { showToast } from "@/utils/toast"; +import { useForm, useFieldArray, Controller } from "react-hook-form"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { z } from "zod"; + +const variableSchema = z.object({ + name: z.string().min(1, "Variable name is required").max(255), + value: z.string().optional(), + required: z.boolean().optional(), +}); + +const testCaseSchema = z.object({ + name: z.string().min(1, "Name is required").max(255), + description: z.string().optional(), + expected_output: z.string().optional(), + tags: z.array(z.string()), + variables: z.array(variableSchema), +}).superRefine((data, ctx) => { + (data.variables || []).forEach((v, index) => { + if (v.required && (!v.value || !v.value.trim())) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `${v.name} is a required variable`, + path: ["variables", index, "value"], + }); + } + }); +}); + +type TestCaseFormData = z.infer; + +export const PromptTestCaseCreate = (): ReactElement => { + const { id: promptId } = useParams<{ id: string }>(); + const navigate = useNavigate(); + + const [isLoading, setIsLoading] = useState(true); + const [isSubmitting, setIsSubmitting] = useState(false); + + const { + control, + handleSubmit, + reset, + formState: { errors }, + } = useForm({ + resolver: zodResolver(testCaseSchema), + defaultValues: { + name: "", + description: "", + expected_output: "", + tags: [], + variables: [], + }, + }); + + const { fields } = useFieldArray({ + control, + name: "variables", + }); + + // Fetch prompt metadata to autofill initial variable keys + useEffect(() => { + const loadPromptDetails = async () => { + if (!promptId) return; + try { + setIsLoading(true); + const prompt = await aiService.getPrompt(promptId); + let initialVariables: Array<{ name: string; value: string; required?: boolean }> = []; + + if (prompt && Array.isArray(prompt.variables)) { + initialVariables = prompt.variables.map((v: any) => ({ + name: v.name || "", + value: v.default || "", + required: !!v.required, + })); + } + + reset({ + name: "", + description: "", + expected_output: "", + tags: prompt.tags || [], + variables: initialVariables, + }); + } catch (err: any) { + console.warn("Failed to prefetch prompt variables", err); + reset({ + name: "", + description: "", + expected_output: "", + tags: [], + variables: [], + }); + } finally { + setIsLoading(false); + } + }; + void loadPromptDetails(); + }, [promptId, reset]); + + const onFormSubmit = async (data: TestCaseFormData) => { + if (!promptId) return; + + // Convert array to Record + const variablesObject: Record = {}; + (data.variables || []).forEach((v) => { + if (v.name.trim()) { + variablesObject[v.name.trim()] = v.value || ""; + } + }); + + setIsSubmitting(true); + try { + await aiService.createPromptTestCase(promptId, { + name: data.name.trim(), + description: data.description?.trim() || undefined, + variables: variablesObject, + expected_output: data.expected_output?.trim() || undefined, + tags: data.tags && data.tags.length > 0 ? data.tags : undefined, + }); + + showToast.success("Test case created successfully!"); + navigate(`/tenant/ai/prompts/${promptId}/test-cases`); + } catch (err: any) { + const msg = + err?.response?.data?.error?.message || "Failed to create test case."; + showToast.error(msg); + } finally { + setIsSubmitting(false); + } + }; + + if (isLoading) { + return ( + +
    +
    +
    +

    + Loading prompt template details... +

    +
    +
    +     + ); + } + + return ( + + navigate(`/tenant/ai/prompts/${promptId}/test-cases`)} + className="h-10 px-5 min-w-[120px]" + > + Cancel + + + {isSubmitting ? "Creating..." : "Save Test Case"} + +
    + ), + }} + > +
    + +
    + +
    + {/* Left column containing Test Case Information */} +
    + {/* General Information Card */} +
    +

    + General Information +

    + ( + + )} + /> + ( + + )} + /> +
    + + {/* Input Variables Section */} +
    +
    +

    + Input Variables +

    +
    + + {/* Column Header Row matching the image */} +
    +
    Variable Name
    +
    Value
    +
    Required
    +
    + +
    +
    + {fields.map((field, index) => ( +
    +
    + + {field.name} + +
    +
    + ( + + )} + /> +
    +
    + + {field.required ? "Required" : "Optional"} + +
    +
    + ))} + {fields.length === 0 && ( +
    +

    + No input variables configured. +

    +
    + )} +
    +
    +
    + + {/* Expected Output Section */} +
    +

    + Expected Output +

    + ( + + )} + /> +
    +
    + + {/* Right column containing Organization & Advanced */} +
    + {/* Organization / Tags Card */} +
    +

    Organization

    + ( + + )} + /> +
    +
    +
    +     + ); +}; + +export default PromptTestCaseCreate; diff --git a/src/pages/tenant/PromptTestCases.tsx b/src/pages/tenant/PromptTestCases.tsx new file mode 100644 index 0000000..81eb082 --- /dev/null +++ b/src/pages/tenant/PromptTestCases.tsx @@ -0,0 +1,326 @@ +import { useEffect, useMemo, useState, type ReactElement } from "react"; +import { useNavigate, useSearchParams, useParams } from "react-router-dom"; +import { + Plus, + Play, + ArrowLeft, + Eye, + Loader2, +} from "lucide-react"; +import { Layout } from "@/components/layout/Layout"; +import { + DataTable, + type Column, + PrimaryButton, + ActionDropdown, +} from "@/components/shared"; +import { aiService } from "@/services/ai-service"; +import type { AIPrompt, AIPromptTestCase } from "@/types/ai"; +import { showToast } from "@/utils/toast"; +import { formatDate } from "@/utils/format-date"; +import { PromptTestCaseResultModal } from "@/components/tenant/PromptTestCaseResultModal"; +import { PromptTestCaseResultsListModal } from "@/components/tenant/PromptTestCaseResultsListModal"; + +const PromptTestCases = (): ReactElement => { + const navigate = useNavigate(); + const { id } = useParams(); + const [searchParams] = useSearchParams(); + const promptId = id || searchParams.get("promptId") || ""; + + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + const [testCases, setTestCases] = useState([]); + const [prompt, setPrompt] = useState(null); + + const [runningCases, setRunningCases] = useState>({}); + const [expandedRows, setExpandedRows] = useState>({}); + + const toggleRowExpansion = (id: string) => { + setExpandedRows((prev) => ({ + ...prev, + [id]: !prev[id], + })); + }; + + const [selectedResult, setSelectedResult] = useState(null); + const [isResultModalOpen, setIsResultModalOpen] = useState(false); + + const [selectedTestCaseId, setSelectedTestCaseId] = useState(""); + const [selectedTestCaseName, setSelectedTestCaseName] = useState(""); + const [isResultsListModalOpen, setIsResultsListModalOpen] = useState(false); + + const handleRunTestCase = async (testCaseId: string) => { + setRunningCases((prev) => ({ ...prev, [testCaseId]: true })); + try { + const response = await aiService.runPromptTestCase(testCaseId); + showToast.success("Test case executed successfully!"); + if (response) { + setSelectedResult(response); + setIsResultModalOpen(true); + } + void loadData(); + } catch (err: any) { + const message = + err?.response?.data?.error?.message || "Failed to run test case"; + showToast.error(message); + } finally { + setRunningCases((prev) => ({ ...prev, [testCaseId]: false })); + } + }; + + // Load Prompt Details and Test Cases + const loadData = async (): Promise => { + if (!promptId) { + setError("Prompt ID is missing."); + setIsLoading(false); + return; + } + + setIsLoading(true); + setError(null); + try { + // 1. Fetch prompt details + try { + const p = await aiService.getPrompt(promptId); + setPrompt(p); + } catch (err) { + console.warn("Failed to load prompt details", err); + } + + // 2. Fetch test cases + const cases = await aiService.listPromptTestCases(promptId); + setTestCases(cases || []); + } catch (err: unknown) { + const message = + (err as { response?: { data?: { error?: { message?: string } } } }) + ?.response?.data?.error?.message || "Failed to load prompt test cases"; + setError(message); + showToast.error(message); + } finally { + setIsLoading(false); + } + }; + + useEffect(() => { + void loadData(); + }, [promptId]); + + + const columns: Column[] = useMemo( + () => [ + { + key: "name", + label: "Name & Description", + render: (row) => ( +
    +

    + {row.name} +

    +

    + {row.description || "No description"} +

    +
    + ), + }, + { + key: "template", + label: "Template", + render: () => ( +
    +

    + {prompt?.name || "AI Prompt"} +

    +

    + v{prompt?.version || 1}.0 +

    +
    + ), + }, + { + key: "tags", + label: "Tags", + render: (row) => { + const tags = row.tags || []; + if (!tags.length) { + return —; + } + return ( +
    + {tags.map((tag, index) => ( + + {tag} + + ))} +
    + ); + }, + }, + { + key: "updatedAt", + label: "Last Updated", + render: (row) => ( + + {formatDate(row.updated_at || row.created_at || "")} + + ), + }, + { + key: "actions", + label: "Actions", + align: "right", + render: (row) => { + const isRunning = runningCases[row.id] || false; + return ( +
    + + , + label: "View Test Results", + onClick: () => { + setSelectedTestCaseId(row.id); + setSelectedTestCaseName(row.name); + setIsResultsListModalOpen(true); + }, + }, + // { + // icon: , + // label: "View Test Case", + // onClick: () => showToast.info("View Test Case details..."), + // }, + // { + // icon: , + // label: "Edit Test Case", + // onClick: () => showToast.info("Edit Test Case..."), + // }, + // { + // icon: , + // label: "Delete Test Case", + // variant: "danger", + // onClick: () => showToast.info("Delete Test Case..."), + // }, + ]} + /> +
    + ); + }, + }, + ], + [prompt, testCases, runningCases] + ); + + + return ( + navigate(`/tenant/ai/prompts/${promptId}/test-cases/create`)} + className="flex items-center gap-2 h-10 shadow-sm bg-[#112868] text-white hover:bg-[#112868]/90" + > + + Create Test Cases + + ), + }} + > +
    + {/* Back navigation link */} +
    + +
    + +
    + item.id} + isLoading={isLoading} + error={error} + emptyMessage="No test cases found." + expandableRows={true} + isRowExpanded={(item) => !!expandedRows[item.id]} + onRowExpandToggle={(item) => toggleRowExpansion(item.id)} + showExpandColumn={true} + expandedColSpan={columns.length + 1} + renderExpandedRow={(item: any) => ( +
    +
    + + Input Variables + + {item.input_variables && Object.keys(item.input_variables).length > 0 ? ( +
    + {Object.entries(item.input_variables).map(([key, value]) => ( +
    + {key}: + + {String(value)} + +
    + ))} +
    + ) : ( + No input variables defined. + )} +
    + +
    + + Expected Output + + {item.expected_output ? ( +
    + {item.expected_output} +
    + ) : ( + No expected output defined. + )} +
    +
    + )} + /> +
    +
    + + setIsResultModalOpen(false)} + result={selectedResult} + /> + + setIsResultsListModalOpen(false)} + testCaseId={selectedTestCaseId} + testCaseName={selectedTestCaseName} + /> +     + ); +}; + +export default PromptTestCases; diff --git a/src/routes/tenant-admin-routes.tsx b/src/routes/tenant-admin-routes.tsx index 9691417..2453dc9 100644 --- a/src/routes/tenant-admin-routes.tsx +++ b/src/routes/tenant-admin-routes.tsx @@ -30,19 +30,27 @@ const NotificationSettings = lazy( () => import("@/pages/tenant/NotificationSettings"), ); const Notifications = lazy(() => import("@/pages/tenant/Notifications")); -const NotificationTemplates = lazy(() => import("@/pages/tenant/NotificationTemplates")); +const NotificationTemplates = lazy( + () => import("@/pages/tenant/NotificationTemplates"), +); const FilesList = lazy(() => import("@/pages/tenant/FilesList")); const FileView = lazy(() => import("@/pages/tenant/FileView")); const StorageDashboard = lazy(() => import("@/pages/tenant/StorageDashboard")); const SmtpSettings = lazy(() => import("@/pages/tenant/SmtpSettings")); const FailedEmails = lazy(() => import("@/pages/tenant/FailedEmails")); const AIGateway = lazy(() => import("@/pages/tenant/AIGateway")); -const CompletionHistory = lazy(() => import("@/pages/tenant/CompletionHistory")); +const CompletionHistory = lazy( + () => import("@/pages/tenant/CompletionHistory"), +); const CompletionCreate = lazy(() => import("@/pages/tenant/CompletionCreate")); const CompletionDetail = lazy(() => import("@/pages/tenant/CompletionDetail")); const PromptManagement = lazy(() => import("@/pages/tenant/PromptManagement")); const PromptCreate = lazy(() => import("@/pages/tenant/PromptCreate")); const PromptEdit = lazy(() => import("@/pages/tenant/PromptEdit")); +const PromptTestCases = lazy(() => import("@/pages/tenant/PromptTestCases")); +const PromptTestCaseCreate = lazy( + () => import("@/pages/tenant/PromptTestCaseCreate"), +); // Loading fallback component const RouteLoader = (): ReactElement => ( @@ -205,6 +213,18 @@ export const tenantAdminRoutes: RouteConfig[] = [ path: "/tenant/ai/prompts/:id/edit", element: , }, + { + path: "/tenant/ai/prompts/:id/test-cases", + element: , + }, + { + path: "/tenant/ai/prompts/:id/test-cases/create", + element: , + }, + { + path: "/tenant/ai/prompt-test-cases", + element: , + }, { path: "/tenant/ai/knowledge", element: , diff --git a/src/services/ai-service.ts b/src/services/ai-service.ts index d9ccf15..f6651c9 100644 --- a/src/services/ai-service.ts +++ b/src/services/ai-service.ts @@ -9,6 +9,7 @@ import type { KnowledgeCollection, KnowledgeSearchItem, TenantAIConfig, + AIPromptTestCase, } from "@/types/ai"; const unwrap = (response: any): T => { @@ -268,6 +269,34 @@ class AIService { const response = await apiClient.post("/ai/knowledge/search/context", payload); return unwrap<{ context?: string; matches?: KnowledgeSearchItem[] }>(response); } + + async createPromptTestCase(promptId: string, payload: { + name: string; + description?: string; + variables?: Record; + expected_output?: string; + expected_contains?: string; + max_tokens?: number; + tags?: string[]; + }): Promise { + const response = await apiClient.post(`/ai/prompts/${promptId}/test-cases`, payload); + return unwrap(response); + } + + async listPromptTestCases(promptId: string): Promise { + const response = await apiClient.get(`/ai/prompts/${promptId}/test-cases`); + return unwrap(response); + } + + async runPromptTestCase(testCaseId: string): Promise { + const response = await apiClient.post(`/ai/prompts/test-cases/${testCaseId}/run`, {}); + return unwrap(response); + } + + async listPromptTestCaseResults(testCaseId: string, limit = 10): Promise { + const response = await apiClient.get(`/ai/prompts/test-cases/${testCaseId}/results`, { params: { limit } }); + return unwrap(response); + } } export const aiService = new AIService(); diff --git a/src/types/ai.ts b/src/types/ai.ts index a8396b5..f3089e3 100644 --- a/src/types/ai.ts +++ b/src/types/ai.ts @@ -163,3 +163,18 @@ export interface AICostSummary { cost: number; }>; } + +export interface AIPromptTestCase { + id: string; + prompt_id: string; + tenant_id: string; + name: string; + description?: string; + input_variables?: Record; + expected_output?: string; + tags?: string[]; + created_by?: string; + created_at?: string; + updated_at?: string; +} +