803 lines
40 KiB
TypeScript
803 lines
40 KiB
TypeScript
import fs from 'fs';
|
||
import path from 'path';
|
||
import { VertexAI } from '@google-cloud/vertexai';
|
||
import { calculateMatch, digitsOnly, normalizeMoney } from './utils';
|
||
import { getCriteriaLabel } from './CpcHistoryService';
|
||
import logger from '@utils/logger';
|
||
|
||
/** Vertex SDK does not read `GCP_KEY_FILE` by itself — must pass keyFilename (critical in Docker). */
|
||
function resolveVertexServiceAccountPath(): string | undefined {
|
||
const fromAdc = (process.env.GOOGLE_APPLICATION_CREDENTIALS || '').trim();
|
||
const fromKeyFile = (process.env.GCP_KEY_FILE || '').trim();
|
||
const candidates = [
|
||
fromAdc,
|
||
fromKeyFile ? path.resolve(process.cwd(), fromKeyFile) : ''
|
||
].filter(Boolean);
|
||
for (const p of candidates) {
|
||
try {
|
||
if (fs.existsSync(p)) return path.resolve(p);
|
||
} catch {
|
||
/* ignore */
|
||
}
|
||
}
|
||
return undefined;
|
||
}
|
||
|
||
/**
|
||
* Decide which printed script Gemini should prefer when the document shows the same field in English and Hindi.
|
||
* Driven by the user's MSD/form string (Devanagari vs Latin letter counts).
|
||
*/
|
||
function preferScriptForMsdFieldValue(value: unknown): 'Devanagari' | 'Latin' {
|
||
const s = String(value ?? '').trim();
|
||
if (!s) return 'Latin';
|
||
try {
|
||
const dev = (s.match(/\p{Script=Devanagari}/gu) || []).length;
|
||
const lat = (s.match(/\p{Script=Latin}/gu) || []).length;
|
||
if (dev === 0 && lat === 0) return 'Latin';
|
||
return dev >= lat ? 'Devanagari' : 'Latin';
|
||
} catch {
|
||
return /[\u0900-\u097F]/.test(s) ? 'Devanagari' : 'Latin';
|
||
}
|
||
}
|
||
|
||
/** JSON block appended to Vertex prompt: per-field prefer_script from MSD input language. */
|
||
function buildMsdScriptPreferenceBlock(
|
||
expectedFields: string[],
|
||
msdReferencePayload?: Record<string, unknown>
|
||
): string {
|
||
if (!msdReferencePayload || typeof msdReferencePayload !== 'object') return '';
|
||
const uniq = [...new Set((expectedFields || []).map((f) => String(f || '').trim()).filter(Boolean))];
|
||
const keys =
|
||
uniq.length > 0
|
||
? uniq
|
||
: Object.keys(msdReferencePayload).filter((k) => {
|
||
const v = msdReferencePayload[k];
|
||
return v !== undefined && v !== null && String(v).trim() !== '';
|
||
});
|
||
if (keys.length === 0) return '';
|
||
const hints: Record<string, { prefer_script: 'Devanagari' | 'Latin' }> = {};
|
||
for (const key of keys) {
|
||
const raw = msdReferencePayload[key];
|
||
if (raw === undefined || raw === null) continue;
|
||
const str = String(raw).trim();
|
||
if (!str) continue;
|
||
hints[key] = { prefer_script: preferScriptForMsdFieldValue(raw) };
|
||
}
|
||
if (Object.keys(hints).length === 0) return '';
|
||
return `MSD_SCRIPT_PREFERENCE (per field: infer input language from MSD; when the document shows the same field in both English and Hindi, extract ONLY the on-page text whose script matches prefer_script for that key — do not translate; do not swap languages):\n${JSON.stringify(hints, null, 2)}\n`;
|
||
}
|
||
|
||
const VALID_DOC_TYPES = ['CSD_PO', 'CPC_AUTH', 'AADHAAR', 'RETAIL_INVOICE'] as const;
|
||
|
||
/**
|
||
* Field rules aligned with RE / Softude mail (Feb–Apr 2026):
|
||
* - Rahul: CSD PO # 100% exact, amounts ±₹5, per-field all-pass (no average-based gate).
|
||
* - Rohit table: customer / order (where fuzzy) ≥95%, invoice ≥98% OR ±₹5, stamp ≥85% fuzzy,
|
||
* Aadhaar 12-digit 100%, retail invoice # ≥95%, document date ≥90%.
|
||
*/
|
||
const DOCUMENT_RULES: any = {
|
||
/** CPC claim doc 2 */
|
||
'AADHAAR': {
|
||
'name': { threshold: 90, method: 'fuzzy' },
|
||
'customer_name': { threshold: 90, method: 'fuzzy' },
|
||
'aadhaar_number': { threshold: 100, method: 'exact_length_12' },
|
||
'aadhar_number': { threshold: 100, method: 'exact_length_12' },
|
||
'gender': { threshold: 100, method: 'exact' },
|
||
'mail_extraction': { threshold: 90, method: 'fuzzy' }
|
||
},
|
||
/** CPC claim doc 1 — authorization letter */
|
||
'CPC_AUTH': {
|
||
'authorized_person_name': { threshold: 90, method: 'fuzzy' },
|
||
'customer_name': { threshold: 90, method: 'fuzzy' },
|
||
'authority_grantor_name': { threshold: 90, method: 'fuzzy' },
|
||
'letter_number': { threshold: 90, method: 'fuzzy' },
|
||
'invoice_value': { threshold: null, method: 'range_5_or_fuzzy_98' },
|
||
'letter_amount': { threshold: null, method: 'range_5_or_fuzzy_98' },
|
||
'amount': { threshold: null, method: 'range_5_or_fuzzy_98' },
|
||
'pan_number': { threshold: 95, method: 'fuzzy' },
|
||
'order_or_authorisation_number': { threshold: 95, method: 'fuzzy' },
|
||
'stamp_sign_present': { threshold: 85, method: 'boolean_fuzzy_85' },
|
||
'govt_signatory_and_stamp_present': { threshold: 85, method: 'boolean_fuzzy_85' },
|
||
'signature_and_stamp': { threshold: 85, method: 'boolean_fuzzy_85' },
|
||
'mail_extraction': { threshold: 90, method: 'fuzzy' }
|
||
},
|
||
/** CSD — Purchase order: PO# remains exact 100% per Rahul; other fuzzy thresholds per Rohit table. */
|
||
'CSD_PO': {
|
||
'customer_name': { threshold: 90, method: 'fuzzy' },
|
||
'name': { threshold: 90, method: 'fuzzy' },
|
||
'order_or_authorisation_number': { threshold: 100, method: 'exact' },
|
||
'po_number': { threshold: 100, method: 'exact' },
|
||
'invoice_value': { threshold: null, method: 'range_5_or_fuzzy_98' },
|
||
'po_amount': { threshold: null, method: 'range_5_or_fuzzy_98' },
|
||
'vendor_name': { threshold: 95, method: 'fuzzy' },
|
||
'govt_signatory_and_stamp_present': { threshold: 85, method: 'boolean_fuzzy_85' },
|
||
'signature_and_stamp': { threshold: 85, method: 'boolean_fuzzy_85' },
|
||
'mail_extraction': { threshold: 90, method: 'fuzzy' }
|
||
},
|
||
'RETAIL_INVOICE': {
|
||
'customer_name': { threshold: 95, method: 'fuzzy' },
|
||
'order_or_authorisation_number': { threshold: 95, method: 'fuzzy' },
|
||
'invoice_value': { threshold: null, method: 'range_5_or_fuzzy_98' },
|
||
'invoice_date': { threshold: 90, method: 'fuzzy' },
|
||
'vendor_name': { threshold: 95, method: 'fuzzy' },
|
||
'mail_extraction': { threshold: 90, method: 'fuzzy' }
|
||
},
|
||
'GENERIC': {
|
||
'default': { threshold: 95, method: 'fuzzy' }
|
||
}
|
||
};
|
||
|
||
/** Human-readable `field_results.threshold` for API/UI (no percentage figures). */
|
||
function apiThresholdLabel(rule: { method?: string; threshold?: number | null }): string {
|
||
const m = rule?.method;
|
||
if (m === 'range_5_or_fuzzy_98' || m === 'range_5') return 'Amount comparison';
|
||
if (m === 'boolean_fuzzy_85' || m === 'boolean') return 'Stamp / signature';
|
||
if (m === 'exact_length_12') return 'Aadhaar number';
|
||
if (m === 'exact' || m === 'exact_numeric') return 'Exact match';
|
||
if (m === 'fuzzy') return 'Text match';
|
||
return 'N/A';
|
||
}
|
||
|
||
function msdFieldDisplayName(fieldKey: string, docType?: string): string {
|
||
if (fieldKey === 'invoice_value' || fieldKey === 'po_amount') {
|
||
if (docType === 'CSD_PO') return 'PO Amount';
|
||
if (docType === 'CPC_AUTH') return 'Letter Amount';
|
||
}
|
||
if (fieldKey === 'letter_amount') return 'Letter Amount';
|
||
const map: Record<string, string> = {
|
||
authorized_person_name: 'Customer Name',
|
||
customer_name: 'Customer Name',
|
||
name: 'Customer Name',
|
||
letter_number: 'Letter Number',
|
||
po_number: 'PO Number',
|
||
order_or_authorisation_number: 'PO Number',
|
||
invoice_value: 'Document Amount',
|
||
po_amount: 'PO Amount',
|
||
amount: 'Letter Amount',
|
||
aadhaar_number: 'Aadhaar Number',
|
||
aadhar_number: 'Aadhaar Number',
|
||
govt_signatory_and_stamp_present: 'Signature & Stamp',
|
||
signature_and_stamp: 'Signature & Stamp',
|
||
stamp_sign_present: 'Signature & Stamp',
|
||
mail_extraction: 'Mail extraction',
|
||
pan_number: 'PAN',
|
||
vendor_name: 'Supplier Name',
|
||
authority_grantor_name: 'Authority Grantor',
|
||
gender: 'Gender'
|
||
};
|
||
return map[fieldKey] || fieldKey.replace(/_/g, ' ');
|
||
}
|
||
|
||
function buildMsdStyleMessage(fieldKey: string, status: string, docType?: string): string {
|
||
const label = msdFieldDisplayName(fieldKey, docType);
|
||
if (status === 'MISSING') {
|
||
return `According to the expected record and the document, the "${label}" could not be read from the document.\nKindly upload the document again or update the expected value.`;
|
||
}
|
||
return `According to the expected record and the document, the "${label}" does not match.\nKindly upload the document again or update the expected value.`;
|
||
}
|
||
|
||
function pickRuleForKey(rules: Record<string, unknown>, key: string): string {
|
||
const k = key.toLowerCase();
|
||
const candidates = Object.keys(rules)
|
||
.filter((rk) => rk !== 'default')
|
||
.sort((a, b) => b.length - a.length);
|
||
const hit = candidates.find((rk) => k.includes(rk.toLowerCase()));
|
||
return hit || 'default';
|
||
}
|
||
|
||
function isWithinRange(valA: any, valB: any, diff: number = 5): boolean {
|
||
const a = parseFloat(String(valA).replace(/[^0-9.]/g, ""));
|
||
const b = parseFloat(String(valB).replace(/[^0-9.]/g, ""));
|
||
if (isNaN(a) || isNaN(b)) return false;
|
||
return Math.abs(a - b) <= diff;
|
||
}
|
||
|
||
function isVertexModelAccessIssue(err: unknown): boolean {
|
||
const e = err as { message?: string; name?: string; code?: number | string };
|
||
const blob = `${e?.name || ''} ${e?.message || ''} ${String(e?.code || '')}`.toLowerCase();
|
||
return (
|
||
blob.includes('publisher model') ||
|
||
blob.includes('model') && blob.includes('not found') ||
|
||
blob.includes('does not have access') ||
|
||
blob.includes('status: 404') ||
|
||
blob.includes('code":404')
|
||
);
|
||
}
|
||
|
||
export class CpcValidationService {
|
||
/**
|
||
* @param expectedFieldKeys When set (e.g. from UI row order), every listed key is validated — MSD values may be empty (fails with clear reason) and keys are not dropped. When omitted, keys come from `msdPayload` (non-blank key names only).
|
||
*/
|
||
static validateSrs(
|
||
msdPayload: any,
|
||
extractedFields: any,
|
||
fieldConfidence: any = {},
|
||
docTypeAttr: string = 'generic_invoice',
|
||
claimId: string | null = null,
|
||
attemptNo: number = 1,
|
||
expectedFieldKeys?: string[] | null
|
||
) {
|
||
let normalizedDocType = (docTypeAttr || "generic_invoice").toUpperCase();
|
||
if (normalizedDocType === 'AADHAAR_CARD' || normalizedDocType === 'ADHAAR') normalizedDocType = 'AADHAAR';
|
||
if (normalizedDocType === 'AUTHORITY_LETTER' || normalizedDocType === 'CPC_LETTER') normalizedDocType = 'CPC_AUTH';
|
||
if (normalizedDocType === 'PURCHASE_ORDER' || normalizedDocType === 'PO') normalizedDocType = 'CSD_PO';
|
||
if (normalizedDocType === 'INVOICE' || normalizedDocType === 'GENERIC_INVOICE') normalizedDocType = 'RETAIL_INVOICE';
|
||
if (!VALID_DOC_TYPES.includes(normalizedDocType as any) && normalizedDocType !== 'GENERIC') {
|
||
logger.warn(`[CpcValidation] Unknown doc type "${docTypeAttr}" → falling back to GENERIC`);
|
||
}
|
||
|
||
const rules = DOCUMENT_RULES[normalizedDocType] || DOCUMENT_RULES.GENERIC;
|
||
|
||
const fieldResults: any[] = [];
|
||
const mismatchReasons: string[] = [];
|
||
let totalMatchPercent = 0;
|
||
let totalFields = 0;
|
||
let matchedCount = 0;
|
||
let mismatchedCount = 0;
|
||
let missingCount = 0;
|
||
|
||
const globalThreshold = 95;
|
||
|
||
const findNormalizedValue = (obj: any, targetKey: string) => {
|
||
const norm = (k: string) => k.toLowerCase().replace(/[\s_]/g, '');
|
||
const normTarget = norm(targetKey);
|
||
if (obj[targetKey] !== undefined) return obj[targetKey];
|
||
|
||
/** MSD field → alternate keys produced by rules / Gemini */
|
||
const synonymSources: Record<string, string[]> = {
|
||
authorized_person_name: ['customer_name', 'name', 'authorized_person_name', 'account_holder_name'],
|
||
customer_name: ['customer_name', 'name', 'authorized_person_name', 'account_holder_name', 'customername'],
|
||
name: ['authorized_person_name', 'customer_name', 'customername'],
|
||
pan_number: ['pan_number', 'pan', 'panno'],
|
||
invoice_value: ['invoice_value', 'amount', 'total_amount', 'total_value', 'po_amount', 'letter_amount'],
|
||
po_amount: ['po_amount', 'invoice_value', 'amount', 'total_amount', 'total_value'],
|
||
letter_amount: ['letter_amount', 'invoice_value', 'amount', 'total_amount', 'total_value'],
|
||
aadhaar_number: ['aadhaar_number', 'aadhar_number', 'aadhaar', 'aadhaarnumber', 'id_number'],
|
||
aadhar_number: ['aadhar_number', 'aadhaar_number', 'aadhaar', 'aadhaarnumber', 'id_number'],
|
||
letter_number: ['letter_number', 'order_or_auth_number', 'auth_number', 'auth_no'],
|
||
order_or_authorisation_number: ['order_or_authorisation_number', 'order_or_auth_number', 'po_number', 'order_number'],
|
||
po_number: ['po_number', 'order_or_authorisation_number', 'order_or_auth_number', 'order_number'],
|
||
govt_signatory_and_stamp_present: [
|
||
'govt_signatory_and_stamp_present',
|
||
'signature_and_stamp',
|
||
'stamp_sign_present',
|
||
'stamp_or_signatory_present'
|
||
],
|
||
signature_and_stamp: [
|
||
'signature_and_stamp',
|
||
'govt_signatory_and_stamp_present',
|
||
'stamp_sign_present',
|
||
'stamp_or_signatory_present'
|
||
],
|
||
mail_extraction: ['mail_extraction', 'email', 'registered_email', 'contact_email', 'buyer_email', 'correspondence_email']
|
||
};
|
||
for (const alt of synonymSources[targetKey] || []) {
|
||
if (obj[alt] !== undefined && obj[alt] !== null && String(obj[alt]).trim() !== '') {
|
||
return obj[alt];
|
||
}
|
||
}
|
||
|
||
const aliases: any = {
|
||
name: ['customername', 'customer_name', 'full_name', 'authorized_person_name', 'account_holder_name'],
|
||
customer_name: ['customername', 'name', 'full_name', 'authorized_person_name', 'account_holder_name'],
|
||
aadhaar_number: ['aadhaarnumber', 'aadhar_number', 'aadhar', 'aadhaar', 'id_number'],
|
||
aadhar_number: ['aadhaarnumber', 'aadhaar_number', 'aadhaar', 'id_number'],
|
||
invoice_value: ['total_amount', 'amount', 'total_value', 'po_amount', 'letter_amount'],
|
||
po_amount: ['invoice_value', 'total_amount', 'amount', 'total_value'],
|
||
letter_amount: ['invoice_value', 'amount', 'total_value'],
|
||
letter_number: ['order_or_auth_number', 'auth_number', 'auth_no'],
|
||
order_or_authorisation_number: ['order_or_auth_number', 'po_number', 'order_number'],
|
||
po_number: ['order_or_authorisation_number', 'order_or_auth_number', 'order_number'],
|
||
govt_signatory_and_stamp_present: ['stamp_sign_present', 'stamp_or_signatory_present', 'signature_and_stamp'],
|
||
signature_and_stamp: ['govt_signatory_and_stamp_present', 'stamp_sign_present', 'stamp_or_signatory_present'],
|
||
mail_extraction: ['email', 'e_mail', 'contactemail', 'correspondenceemail']
|
||
};
|
||
|
||
for (const k of Object.keys(obj)) {
|
||
const normKey = norm(k);
|
||
if (normKey === normTarget) return obj[k];
|
||
for (const [canonical, list] of Object.entries(aliases)) {
|
||
if (
|
||
norm(canonical) === normTarget &&
|
||
(list as string[]).some((a) => norm(a) === normKey)
|
||
) {
|
||
return obj[k];
|
||
}
|
||
}
|
||
}
|
||
return undefined;
|
||
};
|
||
|
||
const fromUi = Array.isArray(expectedFieldKeys)
|
||
? [...new Set(expectedFieldKeys.map((k) => String(k || '').trim()).filter(Boolean))]
|
||
: [];
|
||
|
||
const expectedKeys =
|
||
fromUi.length > 0
|
||
? fromUi
|
||
: Object.keys(msdPayload || {}).filter((k) => k && String(k).trim() !== '');
|
||
|
||
for (const key of expectedKeys) {
|
||
totalFields++;
|
||
const rawExpected = msdPayload?.[key];
|
||
const expectedStr =
|
||
rawExpected === null || rawExpected === undefined ? '' : String(rawExpected);
|
||
const msdValueEmpty =
|
||
expectedStr.trim() === '' || expectedStr.trim().toLowerCase() === 'null';
|
||
|
||
if (msdValueEmpty) {
|
||
const foundPeek = findNormalizedValue(extractedFields, key);
|
||
const confidence = fieldConfidence[key] || 0;
|
||
const label = msdFieldDisplayName(key, normalizedDocType);
|
||
mismatchReasons.push(
|
||
`According to the expected record, "${label}" was not provided. Enter the expected value to validate against the document.`
|
||
);
|
||
fieldResults.push({
|
||
field: key,
|
||
expected: '(not provided)',
|
||
extracted: foundPeek ?? null,
|
||
status: 'UNSUCCESSFUL',
|
||
match_percentage: 0,
|
||
threshold: 'N/A',
|
||
match_method: 'n/a',
|
||
extraction_confidence: confidence,
|
||
reason: 'Expected value was empty — enter a value to compare with the document.',
|
||
criteria: getCriteriaLabel(key, normalizedDocType)
|
||
});
|
||
mismatchedCount++;
|
||
continue;
|
||
}
|
||
|
||
const expected = rawExpected;
|
||
const found = findNormalizedValue(extractedFields, key);
|
||
const confidence = fieldConfidence[key] || 0;
|
||
|
||
const ruleKey = pickRuleForKey(rules as Record<string, unknown>, key);
|
||
const rule = rules[ruleKey] || rules.default || DOCUMENT_RULES.GENERIC.default;
|
||
|
||
let matchPercent = 0;
|
||
let isPass = false;
|
||
let status = "UNSUCCESSFUL";
|
||
let reason = null;
|
||
|
||
if (found === undefined || found === null || String(found).trim() === "" || String(found).toLowerCase() === "null") {
|
||
status = "MISSING";
|
||
reason = "Field not found in document";
|
||
missingCount++;
|
||
} else {
|
||
if (rule.method === 'exact_numeric') {
|
||
const numExp = parseFloat(String(expected).replace(/[^0-9.]/g, ''));
|
||
const numFnd = parseFloat(String(found).replace(/[^0-9.]/g, ''));
|
||
isPass = !isNaN(numExp) && !isNaN(numFnd) && Math.round(numExp) === Math.round(numFnd);
|
||
matchPercent = isPass ? 100 : 0;
|
||
} else if (rule.method === 'exact') {
|
||
const normExp = String(expected).trim().toLowerCase().replace(/[\s\-\/]+/g, '');
|
||
const normFnd = String(found).trim().toLowerCase().replace(/[\s\-\/]+/g, '');
|
||
isPass = normExp === normFnd;
|
||
matchPercent = isPass ? 100 : 0;
|
||
} else if (rule.method === 'range_5') {
|
||
isPass = isWithinRange(expected, found, 5);
|
||
matchPercent = isPass ? 100 : 0;
|
||
} else if (rule.method === 'range_5_or_fuzzy_98') {
|
||
const inRange = isWithinRange(expected, found, 5);
|
||
const expM = normalizeMoney(String(expected));
|
||
const fndM = normalizeMoney(String(found));
|
||
const fuzzyMoney =
|
||
expM && fndM ? calculateMatch(expM, fndM, key) : calculateMatch(String(expected), String(found), key);
|
||
isPass = inRange || fuzzyMoney >= 98;
|
||
matchPercent = inRange ? 100 : fuzzyMoney;
|
||
} else if (rule.method === 'boolean') {
|
||
const normBool = (v: unknown) => {
|
||
const t = String(v ?? '')
|
||
.toLowerCase()
|
||
.trim();
|
||
if (/\b(yes|true|1|present|available|signed)\b/.test(t)) return 'pos';
|
||
if (/\b(no|false|0|absent|not\s*available|unavailable|n\/a)\b/.test(t)) return 'neg';
|
||
return 'unk';
|
||
};
|
||
const ePol = normBool(expected);
|
||
const fPol = normBool(found);
|
||
if (ePol !== 'unk' && fPol !== 'unk') {
|
||
isPass = ePol === fPol;
|
||
} else {
|
||
isPass =
|
||
String(expected).trim().toLowerCase() === String(found).trim().toLowerCase();
|
||
}
|
||
matchPercent = isPass ? 100 : 0;
|
||
} else if (rule.method === 'boolean_fuzzy_85') {
|
||
const expand = (v: unknown) => {
|
||
const t = String(v ?? '').toLowerCase();
|
||
if (/\b(yes|true|1|present|available|signed)\b/.test(t)) return 'available';
|
||
if (/\b(no|false|0|absent|not\s*available|unavailable|n\/a)\b/.test(t)) return 'not available';
|
||
return String(v ?? '')
|
||
.trim()
|
||
.toLowerCase();
|
||
};
|
||
const ex = expand(expected);
|
||
const fd = expand(found);
|
||
matchPercent = calculateMatch(ex, fd, key);
|
||
isPass = matchPercent >= 85;
|
||
} else if (rule.method === 'exact_length_12') {
|
||
const dExp = String(expected).replace(/\D/g, "");
|
||
const dFnd = String(found).replace(/\D/g, "");
|
||
isPass = (dExp === dFnd && dFnd.length === 12);
|
||
matchPercent = isPass ? 100 : 0;
|
||
} else if (rule.threshold === 100) {
|
||
matchPercent = String(expected).trim().toLowerCase() === String(found).trim().toLowerCase() ? 100 : 0;
|
||
isPass = (matchPercent === 100);
|
||
} else {
|
||
matchPercent = calculateMatch(expected, found, key);
|
||
isPass = (matchPercent >= (rule.threshold || globalThreshold));
|
||
}
|
||
|
||
if (isPass) {
|
||
status = "SUCCESSFUL";
|
||
matchedCount++;
|
||
} else {
|
||
status = "UNSUCCESSFUL";
|
||
reason = 'Value does not match expected';
|
||
mismatchedCount++;
|
||
}
|
||
}
|
||
|
||
totalMatchPercent += matchPercent;
|
||
|
||
if (status !== "SUCCESSFUL") {
|
||
mismatchReasons.push(buildMsdStyleMessage(key, status, normalizedDocType));
|
||
}
|
||
|
||
fieldResults.push({
|
||
field: key,
|
||
expected: expected,
|
||
extracted: found || null,
|
||
status: status,
|
||
match_percentage: matchPercent,
|
||
threshold: apiThresholdLabel(rule),
|
||
match_method: rule.method,
|
||
extraction_confidence: confidence,
|
||
reason: reason,
|
||
criteria: getCriteriaLabel(key, normalizedDocType)
|
||
});
|
||
}
|
||
|
||
/** MSD: success only if every expected field passes its own rule (no averaging). */
|
||
const allFieldsPass =
|
||
totalFields > 0 && mismatchedCount === 0 && missingCount === 0 && matchedCount === totalFields;
|
||
const overallAccuracy = totalFields > 0 ? Math.round(totalMatchPercent / totalFields) : 0;
|
||
const displayMatchPercent = allFieldsPass ? 100 : overallAccuracy;
|
||
const hasMissing = missingCount > 0;
|
||
const overallValidationStatus = hasMissing
|
||
? "NEED_MANUAL"
|
||
: allFieldsPass
|
||
? "MATCH"
|
||
: "MISMATCH";
|
||
const overallStatus = overallValidationStatus === "MATCH" ? "SUCCESSFUL" : "UNSUCCESSFUL";
|
||
|
||
return {
|
||
claim_id: claimId,
|
||
attempt_no: attemptNo,
|
||
status: overallStatus,
|
||
validation_status: overallValidationStatus,
|
||
match_percentage: displayMatchPercent,
|
||
overall_match_percentage: displayMatchPercent,
|
||
threshold: 100,
|
||
all_fields_passed: allFieldsPass,
|
||
mismatch_summary: {
|
||
total_expected_fields: totalFields,
|
||
matched: matchedCount,
|
||
mismatched: mismatchedCount,
|
||
missing: missingCount,
|
||
all_fields_passed: allFieldsPass
|
||
},
|
||
mismatch_reasons: mismatchReasons,
|
||
field_results: fieldResults
|
||
};
|
||
}
|
||
|
||
static async extractWithGemini(params: {
|
||
projectId: string;
|
||
location: string;
|
||
modelName?: string;
|
||
documentType: string;
|
||
ocrText?: string;
|
||
fileBuffer?: Buffer;
|
||
mimeType?: string;
|
||
expectedFields?: string[];
|
||
/** MSD / form values — passed into prompt so Gemini aligns labels with user input (no secrets; same as document check). */
|
||
msdReferencePayload?: Record<string, unknown>;
|
||
}) {
|
||
const {
|
||
projectId,
|
||
location,
|
||
modelName,
|
||
documentType,
|
||
ocrText,
|
||
fileBuffer,
|
||
mimeType,
|
||
expectedFields = [],
|
||
msdReferencePayload
|
||
} = params;
|
||
|
||
const saPath = resolveVertexServiceAccountPath();
|
||
const vertexInit: ConstructorParameters<typeof VertexAI>[0] = {
|
||
project: projectId,
|
||
location
|
||
};
|
||
if (saPath) {
|
||
(vertexInit as { googleAuthOptions?: { keyFilename: string } }).googleAuthOptions = {
|
||
keyFilename: saPath
|
||
};
|
||
logger.info(`[CpcValidation] Vertex AI using service account file: ${saPath}`);
|
||
} else {
|
||
logger.warn(
|
||
'[CpcValidation] No GCP_KEY_FILE / GOOGLE_APPLICATION_CREDENTIALS on disk — Vertex uses ADC only (often empty inside Docker).'
|
||
);
|
||
}
|
||
|
||
const usedModel =
|
||
(modelName && String(modelName).trim()) ||
|
||
process.env.GEMINI_MODEL?.trim() ||
|
||
process.env.VERTEX_AI_MODEL?.trim() ||
|
||
'gemini-1.5-flash';
|
||
|
||
const promptText = this.buildPrompt(documentType, ocrText || "", expectedFields, params.msdReferencePayload);
|
||
const parts: any[] = [{ text: promptText }];
|
||
|
||
if (fileBuffer) {
|
||
parts.push({
|
||
inlineData: {
|
||
mimeType: mimeType || "application/pdf",
|
||
data: fileBuffer.toString("base64")
|
||
}
|
||
});
|
||
}
|
||
|
||
const fallbackLocation = (process.env.CPC_VERTEX_FALLBACK_LOCATION || 'us-central1').trim();
|
||
const fallbackModel = (process.env.CPC_VERTEX_FALLBACK_MODEL || 'gemini-2.0-flash-lite').trim();
|
||
const attempts = [
|
||
{ location, model: usedModel, label: 'primary' },
|
||
{ location: fallbackLocation, model: fallbackModel, label: 'fallback' }
|
||
].filter((a, i, arr) => arr.findIndex((x) => x.location === a.location && x.model === a.model) === i);
|
||
|
||
let lastErr: unknown;
|
||
for (let idx = 0; idx < attempts.length; idx++) {
|
||
const attempt = attempts[idx];
|
||
const attemptVertexInit: ConstructorParameters<typeof VertexAI>[0] = {
|
||
project: projectId,
|
||
location: attempt.location
|
||
};
|
||
if (saPath) {
|
||
(attemptVertexInit as { googleAuthOptions?: { keyFilename: string } }).googleAuthOptions = {
|
||
keyFilename: saPath
|
||
};
|
||
}
|
||
const vertexAI = new VertexAI(attemptVertexInit);
|
||
const model = vertexAI.getGenerativeModel({ model: attempt.model });
|
||
try {
|
||
if (idx > 0) {
|
||
logger.warn(
|
||
`[CpcValidation] Retrying Vertex extraction using ${attempt.label} model/location (${attempt.model} @ ${attempt.location})`
|
||
);
|
||
}
|
||
const resp = await model.generateContent({
|
||
contents: [{ role: 'user', parts }],
|
||
generationConfig: {
|
||
temperature: 0.1,
|
||
maxOutputTokens: Math.min(
|
||
8192,
|
||
parseInt(process.env.CPC_VERTEX_MAX_OUTPUT_TOKENS || '8192', 10) || 8192
|
||
)
|
||
}
|
||
});
|
||
|
||
const cand = resp?.response?.candidates?.[0] as { finishReason?: string; content?: { parts?: unknown[] } } | undefined;
|
||
if (cand?.finishReason && cand.finishReason !== 'STOP') {
|
||
logger.warn(`[CpcValidation] Gemini finishReason=${cand.finishReason}`);
|
||
}
|
||
|
||
const out =
|
||
cand?.content?.parts?.map((p: any) => (typeof p?.text === 'string' ? p.text : '')).join('') || '';
|
||
|
||
if (!out) throw new Error('EMPTY_AI_RESPONSE');
|
||
|
||
const parsed = this.parseJsonLoose(out);
|
||
const merged: Record<string, unknown> = { ...(parsed.extracted_fields || {}) };
|
||
const lockKeys = [...new Set(expectedFields.map((k) => String(k || '').trim()).filter(Boolean))];
|
||
for (const k of lockKeys) {
|
||
if (!(k in merged)) merged[k] = null;
|
||
}
|
||
parsed.extracted_fields = merged;
|
||
const keys = Object.keys(parsed.extracted_fields || {});
|
||
if (keys.length === 0) {
|
||
logger.warn('[CpcValidation] Gemini returned empty extracted_fields; raw head: ' + out.slice(0, 400));
|
||
}
|
||
return parsed;
|
||
} catch (error) {
|
||
lastErr = error;
|
||
const shouldRetry = idx < attempts.length - 1 && isVertexModelAccessIssue(error);
|
||
if (shouldRetry) {
|
||
logger.warn(
|
||
`[CpcValidation] Vertex attempt failed for ${attempt.model} @ ${attempt.location}. Trying fallback...`,
|
||
error
|
||
);
|
||
continue;
|
||
}
|
||
logger.error("Gemini Extraction Error:", error);
|
||
throw error;
|
||
}
|
||
}
|
||
throw lastErr || new Error('AI_EXTRACTION_FAILED: Vertex extraction failed');
|
||
}
|
||
|
||
private static buildPrompt(
|
||
documentType: string,
|
||
ocrText: string,
|
||
expectedFields: string[] = [],
|
||
msdReferencePayload?: Record<string, unknown>
|
||
) {
|
||
const dt = documentType.toLowerCase();
|
||
const rawDocType = String(documentType || '');
|
||
const isAadhaar = dt.includes('aadhaar');
|
||
const isInvoice = dt.includes('invoice') || dt.includes('retail');
|
||
/** Avoid `includes('po')` — false positives on unrelated doc type strings. */
|
||
const isCsdPo =
|
||
/\bcsd[_\s-]*po\b/i.test(rawDocType) ||
|
||
/\bpurchase[_\s-]*order\b/i.test(rawDocType) ||
|
||
/^\s*PO\s*$/i.test(rawDocType.trim());
|
||
const isAuthorityDoc =
|
||
dt.includes('authority') ||
|
||
dt.includes('cpc_auth') ||
|
||
dt.includes('auth_letter') ||
|
||
dt.includes('authority_letter') ||
|
||
dt.includes('cpc_letter');
|
||
|
||
const schema: any = {
|
||
extracted_fields: {},
|
||
field_confidence: {}
|
||
};
|
||
|
||
const userLockedKeys = [...new Set((expectedFields || []).map((f) => String(f || '').trim()).filter(Boolean))];
|
||
|
||
if (userLockedKeys.length > 0) {
|
||
userLockedKeys.forEach((f) => {
|
||
schema.extracted_fields[f] = 'string|null';
|
||
});
|
||
} else if (isAadhaar) {
|
||
schema.extracted_fields = {
|
||
customer_name: 'string',
|
||
aadhar_number: 'string',
|
||
name: 'string|null',
|
||
dob: 'string',
|
||
gender: 'string',
|
||
address: 'string',
|
||
aadhaar_number: 'string|null'
|
||
};
|
||
} else if (isCsdPo) {
|
||
schema.extracted_fields = {
|
||
customer_name: 'string',
|
||
po_number: 'string',
|
||
po_amount: 'string',
|
||
signature_and_stamp: 'string|boolean',
|
||
vendor_name: 'string',
|
||
invoice_date: 'string',
|
||
order_or_authorisation_number: 'string|null',
|
||
invoice_value: 'string|null',
|
||
govt_signatory_and_stamp_present: 'string|boolean|null'
|
||
};
|
||
} else if (isInvoice) {
|
||
schema.extracted_fields = {
|
||
customer_name: 'string',
|
||
order_or_authorisation_number: 'string',
|
||
invoice_value: 'string',
|
||
invoice_date: 'string',
|
||
vendor_name: 'string'
|
||
};
|
||
} else if (isAuthorityDoc) {
|
||
schema.extracted_fields = {
|
||
customer_name: 'string',
|
||
letter_number: 'string|null',
|
||
letter_amount: 'string|null',
|
||
signature_and_stamp: 'string|boolean|null',
|
||
authorized_person_name: 'string|null',
|
||
authority_grantor_name: 'string',
|
||
valid_until: 'string',
|
||
purpose: 'string',
|
||
date_of_issue: 'string',
|
||
pan_number: 'string|null',
|
||
order_or_authorisation_number: 'string|null',
|
||
amount: 'string|null',
|
||
invoice_value: 'string|null',
|
||
stamp_sign_present: 'string|boolean|null',
|
||
govt_signatory_and_stamp_present: 'string|boolean|null'
|
||
};
|
||
}
|
||
|
||
Object.keys(schema.extracted_fields).forEach(key => {
|
||
schema.field_confidence[key] = "number (0-1)";
|
||
});
|
||
|
||
const msdRef =
|
||
msdReferencePayload &&
|
||
typeof msdReferencePayload === 'object' &&
|
||
Object.keys(msdReferencePayload).length > 0
|
||
? JSON.stringify(msdReferencePayload, null, 2)
|
||
: '';
|
||
|
||
const scriptPrefBlock = buildMsdScriptPreferenceBlock(userLockedKeys, msdReferencePayload);
|
||
|
||
return `
|
||
Return ONLY valid JSON (no markdown).
|
||
Schema:
|
||
${JSON.stringify(schema, null, 2)}
|
||
|
||
Instructions:
|
||
Extract fields based on the provided document_type.
|
||
${userLockedKeys.length > 0
|
||
? `MANDATORY_KEYS: Your JSON property "extracted_fields" MUST contain exactly these keys (same spelling, no extras): ${userLockedKeys.join(', ')}. Use null only when that value is not visible on the document image/PDF.`
|
||
: ''}
|
||
${userLockedKeys.length > 0
|
||
? `EXTRACTION REQUEST: Extract only what is needed for those keys; do not invent keys outside the list.`
|
||
: ''}
|
||
${msdRef ? `REFERENCE_VALUES (from the user's form — use to locate the correct rows/labels on the document; values in extracted_fields must match what is visibly printed on the PDF/image, not invented):\n${msdRef}\n` : ''}
|
||
${scriptPrefBlock}
|
||
BILINGUAL_FORMS: Indian CPC/CSD forms often print the same label in English and Hindi. For each key in MSD_SCRIPT_PREFERENCE (if present), the MSD value shows which language the user entered — prefer_script is Devanagari (Hindi script) vs Latin (English). When both languages appear for that field on the image/PDF, copy the value whose script matches prefer_script. When only one script is visible, extract that visible value. Never return the other language if both are printed and MSD is clearly single-script. Numeric-only fields (amounts, IDs): use digits as printed; script rule applies mainly to name and free-text fields.
|
||
|
||
For Aadhaar: customer_name (holder name), aadhar_number (12 digits, no spaces preferred), optional dob (DDMMYYYY), gender, address. You may also populate legacy keys name and aadhaar_number if visible.
|
||
CRITICAL: For 'address', extract ONLY the physical location details.
|
||
${isCsdPo
|
||
? `For CSD Purchase Order: extract po_number (PO reference — exact text), po_amount (digits only, rupees), vendor_name (supplier/dealer company from letterhead or From/Supplier block), customer_name (the human buyer / beneficiary — NOT the dealer company name), invoice_date, signature_and_stamp as yes/no (official stamp or authorized signatory visible). Legacy keys order_or_authorisation_number, invoice_value, govt_signatory_and_stamp_present may be filled with the same values if present.
|
||
For customer_name, read the value beside or under labels such as: Sold To, Bill To, Ship To, Consignee, Buyer, Purchaser, Customer, CSD Card / Card Holder, Beneficiary, Name of Purchaser/Buyer, Ordered By. Do NOT use the first generic "Name:" on the page if it sits under supplier/dealer details or is clearly a sales contact.
|
||
Many CSD PO line tables put the beneficiary in the Description column as: a 16-digit number (card/UIN style) immediately followed by the person's name (then often a house/plot number and address). Prefer that name for customer_name when present.
|
||
${expectedFields.some((f) => String(f).toLowerCase() === 'customer_name') ? "CRITICAL: The JSON key customer_name must hold the printed buyer/beneficiary person name from the PO (what the user typed in customer_name). Put the supplying company's legal name only under vendor_name when that key exists; never put the dealer letterhead name in customer_name." : ''}`
|
||
: ''}
|
||
${isInvoice ? 'For Retail Invoice: customer name, invoice amount (numeric only, exclude currency symbol), order/authorisation number, vendor name, and date.' : ''}
|
||
${isAuthorityDoc
|
||
? 'For CPC / Authorization Letter: extract customer_name (person being authorized), letter_number, letter_amount (numeric), signature_and_stamp yes/no (stamp/signature visible). Also extract authority grantor, dates, purpose, PAN if visible when those keys exist in the schema. Legacy keys authorized_person_name, invoice_value, govt_signatory_and_stamp_present may mirror the same values.'
|
||
: ''}
|
||
${userLockedKeys.some((f) => String(f).toLowerCase() === 'mail_extraction')
|
||
? "If 'mail_extraction' is requested: extract the email address or mail reference line visible on the document (official correspondence / contact email). Put the primary value in extracted_fields.mail_extraction."
|
||
: ''}
|
||
If a field name like 'pan_number' is requested, look for a 10-character alphanumeric string (5 letters, 4 digits, 1 letter).
|
||
For 'govt_signatory_and_stamp_present' or 'signature_and_stamp', check if the document has an official stamp or authorized signatory mark and return "yes" or "no".
|
||
|
||
document_type: ${documentType}
|
||
|
||
OCR_TEXT:
|
||
"""${ocrText ? ocrText.slice(0, 20000) : "No OCR text provided. Please extract directly from the provided document image/PDF."}"""
|
||
`;
|
||
}
|
||
|
||
private static parseJsonLoose(text: string): { extracted_fields: Record<string, unknown>; field_confidence: Record<string, unknown> } {
|
||
let s = String(text || '').trim();
|
||
s = s.replace(/^```(?:json)?\s*/i, '').replace(/\s*```\s*$/i, '');
|
||
const a = s.indexOf('{');
|
||
const b = s.lastIndexOf('}');
|
||
if (a === -1) throw new Error('AI_EXTRACTION_FAILED: No JSON object found in LLM response');
|
||
let parsed: Record<string, unknown>;
|
||
try {
|
||
parsed = JSON.parse(s.slice(a, b + 1)) as Record<string, unknown>;
|
||
} catch {
|
||
throw new Error('AI_EXTRACTION_FAILED: Invalid JSON from model');
|
||
}
|
||
const nested = parsed.extracted_fields;
|
||
if (nested && typeof nested === 'object' && !Array.isArray(nested)) {
|
||
return {
|
||
extracted_fields: nested as Record<string, unknown>,
|
||
field_confidence:
|
||
parsed.field_confidence && typeof parsed.field_confidence === 'object'
|
||
? (parsed.field_confidence as Record<string, unknown>)
|
||
: {}
|
||
};
|
||
}
|
||
// Model sometimes returns flat keys instead of { extracted_fields: { ... } }
|
||
const fc =
|
||
parsed.field_confidence && typeof parsed.field_confidence === 'object'
|
||
? (parsed.field_confidence as Record<string, unknown>)
|
||
: {};
|
||
const ef: Record<string, unknown> = { ...parsed };
|
||
delete ef.field_confidence;
|
||
delete ef.extracted_fields;
|
||
return { extracted_fields: ef, field_confidence: fc };
|
||
}
|
||
}
|