diff --git a/src/jobs/form16SapResponseJob.ts b/src/jobs/form16SapResponseJob.ts index 2a53284..7f9b69c 100644 --- a/src/jobs/form16SapResponseJob.ts +++ b/src/jobs/form16SapResponseJob.ts @@ -2,123 +2,265 @@ import fs from 'fs'; import path from 'path'; import logger from '../utils/logger'; import { wfmFileService } from '../services/wfmFile.service'; -import { Form16CreditNote, Form16DebitNote, Form16SapResponse, Form16DebitNoteSapResponse, Form16aSubmission, WorkflowRequest } from '../models'; +import { + Form16CreditNote, + Form16DebitNote, + Form16SapResponse, + Form16DebitNoteSapResponse, + Form16aSubmission, + WorkflowRequest, +} from '../models'; import { gcsStorageService } from '../services/gcsStorage.service'; -type ResponseRow = Record; +// ─── Helpers ───────────────────────────────────────────────────────────────── function safeFileName(name: string): string { return (name || '').trim().replace(/[\\\/:*?"<>|]+/g, '-').slice(0, 180) || 'form16-sap-response.csv'; } -async function processOutgoingFile(fileName: string, type: 'credit' | 'debit', resolvedOutgoingDir?: string): Promise { - // Idempotency by file name. Credit uses form16_sap_responses; debit uses form16_debit_note_sap_responses. - const CreditModel = Form16SapResponse as any; - const DebitModel = Form16DebitNoteSapResponse as any; - const existingCredit = type === 'credit' ? await CreditModel.findOne({ where: { fileName }, attributes: ['id', 'creditNoteId', 'sapDocumentNumber', 'storageUrl'] }) : null; - const existingDebit = type === 'debit' ? await DebitModel.findOne({ where: { fileName }, attributes: ['id', 'debitNoteId', 'sapDocumentNumber', 'storageUrl'] }) : null; - const existing = existingCredit || existingDebit; - if (existing && (existing.creditNoteId ?? existing.debitNoteId) && (existing.storageUrl || existing.sapDocumentNumber)) { - return; - } +/** Columns we store in dedicated DB fields. Everything else goes into raw_row. */ +const KNOWN_CSV_COLUMNS = new Set([ + 'TRNS_UNIQ_NO', 'TRNSUNIQNO', 'DMS_UNIQ_NO', 'DMSUNIQNO', + 'TDS_TRNS_ID', + 'CLAIM_NUMBER', + 'DOC_NO', 'DOCNO', 'SAP_DOC_NO', 'SAPDOC', + 'MSG_TYP', 'MSGTYP', 'MSG_TYPE', + 'MESSAGE', 'MSG', + 'DOC_DATE', 'DOCDATE', + 'TDS_AMT', 'TDSAMT', +]); - const rows = resolvedOutgoingDir - ? await wfmFileService.readForm16OutgoingResponseByPath(path.join(resolvedOutgoingDir, fileName)) - : await wfmFileService.readForm16OutgoingResponse(fileName, type); - if (!rows || rows.length === 0) { - // Still record as processed so we don't retry empty/invalid files forever - if (existing) { - if (type === 'credit') await CreditModel.update({ rawRow: null, updatedAt: new Date() }, { where: { id: existing.id } }); - else await DebitModel.update({ rawRow: null, updatedAt: new Date() }, { where: { id: existing.id } }); - } else { - if (type === 'credit') await CreditModel.create({ type, fileName, rawRow: null, createdAt: new Date(), updatedAt: new Date() }); - else await DebitModel.create({ fileName, rawRow: null, createdAt: new Date(), updatedAt: new Date() }); - } - return; - } +/** + * Parse all columns from one CSV data row. + * Returns { known fields } + rawRow (only the columns NOT in KNOWN_CSV_COLUMNS). + */ +function extractCsvFields(r: Record) { + const trnsUniqNo = (r.TRNS_UNIQ_NO || r.TRNSUNIQNO || r.DMS_UNIQ_NO || r.DMSUNIQNO || '').trim() || null; + const tdsTransId = (r.TDS_TRNS_ID || '').trim() || null; + const claimNumber = (r.CLAIM_NUMBER || '').trim() || null; + const sapDocNo = (r.DOC_NO || r.DOCNO || r.SAP_DOC_NO || r.SAPDOC || '').trim() || null; + const msgTyp = (r.MSG_TYP || r.MSGTYP || r.MSG_TYPE || '').trim() || null; + const message = (r.MESSAGE || r.MSG || '').trim() || null; + const docDate = (r.DOC_DATE || r.DOCDATE || '').trim() || null; + const tdsAmt = (r.TDS_AMT || r.TDSAMT || '').trim() || null; - // Choose the first "real" row. Some SAP/WFM exports include an extra line like "|MSG_TYP|MESSAGE|" - // after the header; in that case rows[0] will not have TRNS_UNIQ_NO and mapping will fail. - const normalizedRows = rows as ResponseRow[]; - const pick = - // Prefer proper transaction id rows - normalizedRows.find((row) => { - const id = (row.TRNS_UNIQ_NO || row.TRNSUNIQNO || row.DMS_UNIQ_NO || row.DMSUNIQNO || '')?.toString().trim(); - return Boolean(id); - }) || - // Fallback: require BOTH a claim ref and a doc/status field to avoid picking the "|MSG_TYP|MESSAGE|" line - normalizedRows.find((row) => { - const claimRef = (row.TDS_TRNS_ID || row.CLAIM_NUMBER || '')?.toString().trim(); - const docNo = (row.DOC_NO || row.DOCNO || row.SAP_DOC_NO || row.SAPDOC || '')?.toString().trim(); - const msgTyp = (row.MSG_TYP || row.MSGTYP || row.MSG_TYPE || '')?.toString().trim(); - if (!claimRef) return false; - if (!docNo && !msgTyp) return false; - // guard against the "MSG_TYP" literal being inside claimRef column - if (claimRef.toUpperCase() === 'MSG_TYP' || claimRef.toUpperCase() === 'MESSAGE') return false; - return true; - }); - const r = (pick || (rows[0] as ResponseRow)) as ResponseRow; - const trnsUniqNo = (r.TRNS_UNIQ_NO || r.TRNSUNIQNO || r.DMS_UNIQ_NO || r.DMS_UNIQ_NO || '')?.toString().trim() || null; - // SAP claim number: credit uses TDS_TRNS_ID; debit uses CLAIM_NUMBER - const claimNumber = ( - (type === 'credit' ? r.TDS_TRNS_ID : r.CLAIM_NUMBER) || - r.CLAIM_NUMBER || - r.TDS_TRNS_ID || - '' - )?.toString().trim() || null; - const sapDocNo = (r.DOC_NO || r.DOCNO || r.SAP_DOC_NO || r.SAPDOC || '')?.toString().trim() || null; - const msgTyp = (r.MSG_TYP || r.MSGTYP || r.MSG_TYPE || '')?.toString().trim() || null; - const message = (r.MESSAGE || r.MSG || '')?.toString().trim() || null; - - let creditNoteId: number | null = null; - let debitNoteId: number | null = null; - let requestId: string | null = null; - let requestNumber: string | null = null; - - if (type === 'credit' && trnsUniqNo) { - const cn = await (Form16CreditNote as any).findOne({ where: { trnsUniqNo }, attributes: ['id', 'submissionId'] }); - if (cn) { - creditNoteId = cn.id; - if (sapDocNo) await (Form16CreditNote as any).update({ sapDocumentNumber: sapDocNo, status: 'completed' }, { where: { id: cn.id } }); - const submission = await (Form16aSubmission as any).findByPk(cn.submissionId, { attributes: ['requestId'] }); - requestId = submission?.requestId ?? null; - } - } - // Backward compatibility: old credit notes may not have trnsUniqNo stored. Match by creditNoteNumber (SAP sends it as TDS_TRNS_ID). - if (type === 'credit' && !creditNoteId && claimNumber) { - const cn = await (Form16CreditNote as any).findOne({ where: { creditNoteNumber: claimNumber }, attributes: ['id', 'submissionId'] }); - if (cn) { - creditNoteId = cn.id; - if (sapDocNo) await (Form16CreditNote as any).update({ sapDocumentNumber: sapDocNo, status: 'completed' }, { where: { id: cn.id } }); - const submission = await (Form16aSubmission as any).findByPk(cn.submissionId, { attributes: ['requestId'] }); - requestId = submission?.requestId ?? null; + // Extra columns → raw_row (so nothing is ever lost) + const rawRow: Record = {}; + for (const [key, val] of Object.entries(r)) { + if (!KNOWN_CSV_COLUMNS.has(key.trim().toUpperCase()) && !KNOWN_CSV_COLUMNS.has(key.trim())) { + rawRow[key.trim()] = val || ''; } } - if (type === 'debit' && trnsUniqNo) { - const dn = await (Form16DebitNote as any).findOne({ where: { trnsUniqNo }, attributes: ['id'] }); - if (dn) { - debitNoteId = dn.id; - if (sapDocNo) await (Form16DebitNote as any).update({ sapDocumentNumber: sapDocNo, status: 'completed' }, { where: { id: dn.id } }); + return { trnsUniqNo, tdsTransId, claimNumber, sapDocNo, msgTyp, message, docDate, tdsAmt, rawRow }; +} + +// ─── Credit note matching ───────────────────────────────────────────────────── + +async function findCreditNoteId( + trnsUniqNo: string | null, + tdsTransId: string | null, + fileName: string, +): Promise<{ creditNoteId: number | null; requestId: string | null }> { + const CN = Form16CreditNote as any; + let cn: any = null; + + // 1. Primary: TDS_TRNS_ID in SAP response = credit note number we sent + if (tdsTransId) { + cn = await CN.findOne({ where: { creditNoteNumber: tdsTransId }, attributes: ['id', 'submissionId'] }); + if (cn) logger.info(`[Form16 SAP Job] Credit match via TDS_TRNS_ID=${tdsTransId} → credit_note id=${cn.id}`); + } + + // 2. TRNS_UNIQ_NO (format: F16-CN-{submissionId}-{creditNoteId}-{ts}) + if (!cn && trnsUniqNo) { + const m = trnsUniqNo.match(/^F16-CN-(\d+)-(\d+)-/); + if (m) { + cn = await CN.findByPk(parseInt(m[2]), { attributes: ['id', 'submissionId'] }); + if (cn) logger.info(`[Form16 SAP Job] Credit match via TRNS_UNIQ_NO id-parse=${m[2]} → credit_note id=${cn.id}`); + } + if (!cn) { + cn = await CN.findOne({ where: { trnsUniqNo }, attributes: ['id', 'submissionId'] }); + if (cn) logger.info(`[Form16 SAP Job] Credit match via trns_uniq_no=${trnsUniqNo} → credit_note id=${cn.id}`); } } - if (type === 'debit' && !debitNoteId && claimNumber) { - const dn = await (Form16DebitNote as any).findOne({ where: { debitNoteNumber: claimNumber }, attributes: ['id'] }); - if (dn) { - debitNoteId = dn.id; - if (sapDocNo) await (Form16DebitNote as any).update({ sapDocumentNumber: sapDocNo, status: 'completed' }, { where: { id: dn.id } }); - } - } - // Fallback: match by filename (without .csv) to debit_note_number when SAP uses different TRNS_UNIQ_NO/CLAIM_NUMBER - if (type === 'debit' && !debitNoteId) { + + // 3. Filename (without .csv) = credit note number + if (!cn) { const baseName = fileName.replace(/\.csv$/i, '').trim(); if (baseName) { - const dn = await (Form16DebitNote as any).findOne({ where: { debitNoteNumber: baseName }, attributes: ['id'] }); - if (dn) { - debitNoteId = dn.id; - if (sapDocNo) await (Form16DebitNote as any).update({ sapDocumentNumber: sapDocNo, status: 'completed' }, { where: { id: dn.id } }); + cn = await CN.findOne({ where: { creditNoteNumber: baseName }, attributes: ['id', 'submissionId'] }); + if (cn) logger.info(`[Form16 SAP Job] Credit match via filename=${baseName} → credit_note id=${cn.id}`); + } + } + + if (!cn) return { creditNoteId: null, requestId: null }; + + const submission = await (Form16aSubmission as any).findByPk(cn.submissionId, { attributes: ['requestId'] }); + return { creditNoteId: cn.id, requestId: submission?.requestId ?? null }; +} + +// ─── Debit note matching ────────────────────────────────────────────────────── + +async function findDebitNoteId( + trnsUniqNo: string | null, + tdsTransId: string | null, + claimNumber: string | null, + fileName: string, +): Promise { + const DN = Form16DebitNote as any; + const CN = Form16CreditNote as any; + let dn: any = null; + + // 1. Primary: TRNS_UNIQ_NO (format: F16-DN-{creditNoteId}-{debitNoteId}-{ts}) + if (trnsUniqNo) { + const m = trnsUniqNo.match(/^F16-DN-(\d+)-(\d+)-/); + if (m) { + dn = await DN.findByPk(parseInt(m[2]), { attributes: ['id'] }); + if (dn) logger.info(`[Form16 SAP Job] Debit match via TRNS_UNIQ_NO id-parse=${m[2]} → debit_note id=${dn.id}`); + } + if (!dn) { + dn = await DN.findOne({ where: { trnsUniqNo }, attributes: ['id'] }); + if (dn) logger.info(`[Form16 SAP Job] Debit match via trns_uniq_no=${trnsUniqNo} → debit_note id=${dn.id}`); + } + } + + // 2. TDS_TRNS_ID = credit note number → find linked debit note + if (!dn && tdsTransId) { + const cn = await CN.findOne({ where: { creditNoteNumber: tdsTransId }, attributes: ['id'] }); + if (cn) { + dn = await DN.findOne({ + where: { creditNoteId: cn.id }, + order: [['createdAt', 'DESC']], + attributes: ['id'], + }); + if (dn) logger.info(`[Form16 SAP Job] Debit match via TDS_TRNS_ID=${tdsTransId} → credit_note id=${cn.id} → debit_note id=${dn.id}`); + } + } + + // 3. CLAIM_NUMBER = debit note number + if (!dn && claimNumber) { + dn = await DN.findOne({ where: { debitNoteNumber: claimNumber }, attributes: ['id'] }); + if (dn) logger.info(`[Form16 SAP Job] Debit match via CLAIM_NUMBER=${claimNumber} → debit_note id=${dn.id}`); + } + + // 4. Filename (without .csv) = debit note number + if (!dn) { + const baseName = fileName.replace(/\.csv$/i, '').trim(); + if (baseName) { + dn = await DN.findOne({ where: { debitNoteNumber: baseName }, attributes: ['id'] }); + if (dn) logger.info(`[Form16 SAP Job] Debit match via filename=${baseName} → debit_note id=${dn.id}`); + } + } + + return dn ? dn.id : null; +} + +// ─── Core processor ─────────────────────────────────────────────────────────── + +async function processOutgoingFile( + fileName: string, + type: 'credit' | 'debit', + resolvedOutgoingDir: string, +): Promise { + const CreditModel = Form16SapResponse as any; + const DebitModel = Form16DebitNoteSapResponse as any; + + // Idempotency: skip if already fully linked + const existing = + type === 'credit' + ? await CreditModel.findOne({ where: { fileName }, attributes: ['id', 'creditNoteId', 'sapDocumentNumber', 'storageUrl'] }) + : await DebitModel.findOne({ where: { fileName }, attributes: ['id', 'debitNoteId', 'sapDocumentNumber', 'storageUrl'] }); + + if (existing && (existing.creditNoteId ?? existing.debitNoteId) && (existing.storageUrl || existing.sapDocumentNumber)) { + logger.debug(`[Form16 SAP Job] Skipping already-processed ${type} file: ${fileName}`); + return; + } + + // ── Read CSV ── + const rows = await wfmFileService.readForm16OutgoingResponseByPath(path.join(resolvedOutgoingDir, fileName)); + if (!rows || rows.length === 0) { + logger.warn(`[Form16 SAP Job] ${type} file ${fileName}: empty or unreadable CSV`); + const emptyPayload = { rawRow: null, updatedAt: new Date() }; + if (existing) { + type === 'credit' ? await CreditModel.update(emptyPayload, { where: { id: existing.id } }) + : await DebitModel.update(emptyPayload, { where: { id: existing.id } }); + } else { + type === 'credit' ? await CreditModel.create({ type, fileName, ...emptyPayload, createdAt: new Date() }) + : await DebitModel.create({ fileName, ...emptyPayload, createdAt: new Date() }); + } + return; + } + + // ── Pick the best data row ── + // Skip the degenerate "|MSG_TYP|MESSAGE|" lines that some SAP exports include after the header. + type CsvRow = Record; + const normalizedRows = rows as CsvRow[]; + const pick = + normalizedRows.find((row) => { + const trns = (row.TRNS_UNIQ_NO || row.TRNSUNIQNO || row.DMS_UNIQ_NO || '').trim(); + return Boolean(trns); + }) || + normalizedRows.find((row) => { + const tdsId = (row.TDS_TRNS_ID || '').trim(); + const docNo = (row.DOC_NO || row.DOCNO || '').trim(); + const msgTyp = (row.MSG_TYP || '').trim(); + if (!tdsId) return false; + if (!docNo && !msgTyp) return false; + if (['MSG_TYP', 'MESSAGE', 'TDS_TRNS_ID'].includes(tdsId.toUpperCase())) return false; + return true; + }) || + normalizedRows[0]; + + const r = pick as CsvRow; + const { trnsUniqNo, tdsTransId, claimNumber, sapDocNo, msgTyp, message, docDate, tdsAmt, rawRow } = extractCsvFields(r); + + logger.info( + `[Form16 SAP Job] Processing ${type} file ${fileName}: TRNS_UNIQ_NO=${trnsUniqNo ?? '—'}, TDS_TRNS_ID=${tdsTransId ?? '—'}, CLAIM_NUMBER=${claimNumber ?? '—'}, DOC_NO=${sapDocNo ?? '—'}` + ); + + // ── Match to a note in DB ── + let creditNoteId: number | null = null; + let debitNoteId: number | null = null; + let requestId: string | null = null; + let requestNumber: string | null = null; + + if (type === 'credit') { + const res = await findCreditNoteId(trnsUniqNo, tdsTransId, fileName); + creditNoteId = res.creditNoteId; + requestId = res.requestId; + if (creditNoteId && sapDocNo) { + await (Form16CreditNote as any).update( + { sapDocumentNumber: sapDocNo, status: 'completed' }, + { where: { id: creditNoteId } } + ); + } + if (!creditNoteId) { + logger.warn( + `[Form16 SAP Job] Credit file ${fileName}: no matching credit note. TDS_TRNS_ID=${tdsTransId ?? '—'}, TRNS_UNIQ_NO=${trnsUniqNo ?? '—'}.` + ); + } + } else { + debitNoteId = await findDebitNoteId(trnsUniqNo, tdsTransId, claimNumber, fileName); + if (debitNoteId && sapDocNo) { + await (Form16DebitNote as any).update( + { sapDocumentNumber: sapDocNo, status: 'completed' }, + { where: { id: debitNoteId } } + ); + // Fetch requestId from linked credit note → submission + const dn = await (Form16DebitNote as any).findByPk(debitNoteId, { attributes: ['creditNoteId'] }); + if (dn?.creditNoteId) { + const cn = await (Form16CreditNote as any).findByPk(dn.creditNoteId, { attributes: ['submissionId'] }); + if (cn?.submissionId) { + const sub = await (Form16aSubmission as any).findByPk(cn.submissionId, { attributes: ['requestId'] }); + requestId = sub?.requestId ?? null; + } } } + if (!debitNoteId) { + logger.warn( + `[Form16 SAP Job] Debit file ${fileName}: no matching debit note. TRNS_UNIQ_NO=${trnsUniqNo ?? '—'}, TDS_TRNS_ID=${tdsTransId ?? '—'}, CLAIM_NUMBER=${claimNumber ?? '—'}.` + ); + } } if (requestId) { @@ -126,8 +268,8 @@ async function processOutgoingFile(fileName: string, type: 'credit' | 'debit', r requestNumber = req?.requestNumber ?? null; } - // Read the raw file bytes and upload to storage so it can be downloaded later - const absPath = resolvedOutgoingDir ? path.join(resolvedOutgoingDir, fileName) : wfmFileService.getForm16OutgoingPath(fileName, type); + // ── Upload raw CSV to storage ── + const absPath = path.join(resolvedOutgoingDir, fileName); let storageUrl: string | null = null; try { if (fs.existsSync(absPath)) { @@ -142,99 +284,118 @@ async function processOutgoingFile(fileName: string, type: 'credit' | 'debit', r storageUrl = upload.storageUrl || null; } } catch (e) { - logger.error('[Form16 SAP Job] Failed to upload outgoing response file:', fileName, e); + logger.error('[Form16 SAP Job] Failed to upload response file:', fileName, e); } + // ── Persist to DB ── + const commonFields = { + trnsUniqNo, + tdsTransId, + claimNumber, + sapDocumentNumber: sapDocNo, + msgTyp, + message, + docDate, + tdsAmt, + rawRow: Object.keys(rawRow).length ? rawRow : null, + storageUrl, + updatedAt: new Date(), + }; + if (type === 'credit') { - const payload = { - type: 'credit' as const, - fileName, - creditNoteId, - claimNumber, - sapDocumentNumber: sapDocNo, - msgTyp, - message, - rawRow: r as any, - storageUrl, - updatedAt: new Date(), - }; + const payload = { type: 'credit' as const, fileName, creditNoteId, ...commonFields }; if (existing) await CreditModel.update(payload, { where: { id: existing.id } }); - else await CreditModel.create({ ...payload, createdAt: new Date() }); + else await CreditModel.create({ ...payload, createdAt: new Date() }); } else { - if (debitNoteId == null && (trnsUniqNo || claimNumber)) { - logger.warn(`[Form16 SAP Job] Debit file ${fileName}: no matching debit note in DB. TRNS_UNIQ_NO=${trnsUniqNo ?? '—'}, CLAIM_NUMBER=${claimNumber ?? '—'}. Ensure a debit note exists with matching trns_uniq_no or debit_note_number.`); - } - const payload = { - fileName, - debitNoteId, - claimNumber, - sapDocumentNumber: sapDocNo, - msgTyp, - message, - rawRow: r as any, - storageUrl, - updatedAt: new Date(), - }; + const payload = { fileName, debitNoteId, ...commonFields }; if (existing) await DebitModel.update(payload, { where: { id: existing.id } }); - else await DebitModel.create({ ...payload, createdAt: new Date() }); + else await DebitModel.create({ ...payload, createdAt: new Date() }); } + + logger.info( + `[Form16 SAP Job] Saved ${type} SAP response for file ${fileName} → ${type === 'credit' ? `credit_note_id=${creditNoteId}` : `debit_note_id=${debitNoteId}`}, storage_url=${storageUrl ? 'yes' : 'no'}` + ); } +// ─── Public API (called by Pull button controller) ──────────────────────────── + +/** + * Scan both OUTGOING dirs, read every CSV, match to a DB note via TDS_TRNS_ID (primary), + * TRNS_UNIQ_NO, CLAIM_NUMBER, or filename (fallbacks), save all known CSV columns to their + * own DB columns and any extra columns to raw_row. + * + * Called by POST /form16/sap/pull – no scheduler, Pull button is the only trigger. + */ export async function runForm16SapResponseIngestionOnce(): Promise<{ processed: number; creditProcessed: number; debitProcessed: number; }> { let creditProcessed = 0; - let debitProcessed = 0; + let debitProcessed = 0; + + const RELATIVE_CREDIT_OUT = path.join('WFM-QRE', 'OUTGOING', 'WFM_SAP_MAIN', 'FORM16_CRDT'); + const RELATIVE_DEBIT_OUT = path.join('WFM-QRE', 'OUTGOING', 'WFM_SAP_MAIN', 'FORM16_DBT'); + + const dirs: Array<{ dir: string; type: 'credit' | 'debit'; relSubdir: string }> = [ + { + dir: path.dirname(wfmFileService.getForm16OutgoingPath('__probe__.csv', 'credit')), + type: 'credit', + relSubdir: RELATIVE_CREDIT_OUT, + }, + { + dir: path.dirname(wfmFileService.getForm16OutgoingPath('__probe__.csv', 'debit')), + type: 'debit', + relSubdir: RELATIVE_DEBIT_OUT, + }, + ]; + try { const base = process.env.WFM_BASE_PATH || 'C:\\WFM'; - const creditDir = path.dirname(wfmFileService.getForm16OutgoingPath('__probe__.csv', 'credit')); - const debitDir = path.dirname(wfmFileService.getForm16OutgoingPath('__probe__.csv', 'debit')); - const dirs: Array<{ dir: string; type: 'credit' | 'debit' }> = [ - { dir: creditDir, type: 'credit' }, - { dir: debitDir, type: 'debit' }, - ]; - const RELATIVE_DEBIT_OUT = path.join('WFM-QRE', 'OUTGOING', 'WFM_SAP_MAIN', 'FORM16_DBT'); - const RELATIVE_CREDIT_OUT = path.join('WFM-QRE', 'OUTGOING', 'WFM_SAP_MAIN', 'FORM16_CRDT'); - for (const { dir, type } of dirs) { + for (const { dir, type, relSubdir } of dirs) { let abs = path.isAbsolute(dir) ? dir : path.join(base, dir); + if (!fs.existsSync(abs)) { - const relSubdir = type === 'debit' ? RELATIVE_DEBIT_OUT : RELATIVE_CREDIT_OUT; const cwdFallback = path.join(process.cwd(), relSubdir); if (fs.existsSync(cwdFallback)) { abs = cwdFallback; - logger.info(`[Form16 SAP Job] Using ${type} outgoing dir from current working directory: ${abs}`); + logger.info(`[Form16 SAP Job] ${type} OUTGOING dir resolved via cwd: ${abs}`); } else { - logger.warn(`[Form16 SAP Job] ${type} outgoing dir does not exist, skipping. Tried: ${abs} and ${cwdFallback}. Set WFM_BASE_PATH to the folder that contains WFM-QRE, or place WFM-QRE under project root.`); + logger.warn( + `[Form16 SAP Job] ${type} OUTGOING dir not found. Tried: ${abs} | ${cwdFallback}. ` + + `Set WFM_BASE_PATH to the folder containing WFM-QRE.` + ); continue; } } + const files = fs.readdirSync(abs).filter((f) => f.toLowerCase().endsWith('.csv')); - logger.info(`[Form16 SAP Job] ${type} outgoing dir: ${abs}, found ${files.length} CSV file(s): ${files.length ? files.join(', ') : '(none)'}`); + logger.info( + `[Form16 SAP Job] ${type} OUTGOING dir: ${abs} → ${files.length} CSV file(s)${files.length ? ': ' + files.join(', ') : ''}` + ); + for (const f of files) { - await processOutgoingFile(f, type, abs); - if (type === 'credit') creditProcessed++; - else debitProcessed++; + try { + await processOutgoingFile(f, type, abs); + if (type === 'credit') creditProcessed++; + else debitProcessed++; + } catch (e) { + logger.error(`[Form16 SAP Job] Error processing ${type} file ${f}:`, e); + } } } } catch (e) { - logger.error('[Form16 SAP Job] Tick error:', e); + logger.error('[Form16 SAP Job] Ingestion error:', e); } + + logger.info( + `[Form16 SAP Job] Pull complete – credit: ${creditProcessed}, debit: ${debitProcessed}, total: ${creditProcessed + debitProcessed}` + ); + return { processed: creditProcessed + debitProcessed, creditProcessed, debitProcessed, }; } - -/** Start scheduler that ingests SAP response files every 5 minutes. */ -export function startForm16SapResponseJob(): void { - const cron = require('node-cron'); - cron.schedule('*/5 * * * *', () => { - runForm16SapResponseIngestionOnce(); - }); - logger.info('[Form16 SAP Job] Scheduled SAP response ingestion (every 5 minutes)'); -} - diff --git a/src/migrations/20260318200001-add-sap-response-csv-fields.ts b/src/migrations/20260318200001-add-sap-response-csv-fields.ts new file mode 100644 index 0000000..3a4ed72 --- /dev/null +++ b/src/migrations/20260318200001-add-sap-response-csv-fields.ts @@ -0,0 +1,41 @@ +import type { QueryInterface } from 'sequelize'; +import { DataTypes } from 'sequelize'; + +/** + * Add explicit CSV-column fields to both SAP response tables. + * Previously everything was dumped into raw_row; now each well-known SAP CSV column + * has its own DB column, and raw_row holds only unexpected/extra columns. + * + * New columns (both tables): + * trns_uniq_no – TRNS_UNIQ_NO from SAP response (our unique ID echoed back) + * tds_trns_id – TDS_TRNS_ID from SAP response (= credit note number we sent) + * doc_date – DOC_DATE (SAP document date) + * tds_amt – TDS_AMT (amount confirmed by SAP) + */ +module.exports = { + up: async (queryInterface: QueryInterface) => { + const commonColumns = [ + ['trns_uniq_no', { type: DataTypes.STRING(200), allowNull: true }], + ['tds_trns_id', { type: DataTypes.STRING(200), allowNull: true }], + ['doc_date', { type: DataTypes.STRING(20), allowNull: true }], + ['tds_amt', { type: DataTypes.STRING(50), allowNull: true }], + ] as const; + + for (const [col, def] of commonColumns) { + await queryInterface.addColumn('form16_sap_responses', col, def).catch(() => {/* already exists */}); + await queryInterface.addColumn('form16_debit_note_sap_responses', col, def).catch(() => {/* already exists */}); + } + + await queryInterface.addIndex('form16_sap_responses', ['trns_uniq_no'], { name: 'idx_f16_sap_resp_trns_uniq_no' }).catch(() => {}); + await queryInterface.addIndex('form16_sap_responses', ['tds_trns_id'], { name: 'idx_f16_sap_resp_tds_trns_id' }).catch(() => {}); + await queryInterface.addIndex('form16_debit_note_sap_responses', ['trns_uniq_no'], { name: 'idx_f16_dbt_sap_trns_uniq_no' }).catch(() => {}); + await queryInterface.addIndex('form16_debit_note_sap_responses', ['tds_trns_id'], { name: 'idx_f16_dbt_sap_tds_trns_id' }).catch(() => {}); + }, + + down: async (queryInterface: QueryInterface) => { + for (const col of ['trns_uniq_no', 'tds_trns_id', 'doc_date', 'tds_amt']) { + await queryInterface.removeColumn('form16_sap_responses', col).catch(() => {}); + await queryInterface.removeColumn('form16_debit_note_sap_responses', col).catch(() => {}); + } + }, +}; diff --git a/src/models/Form16DebitNoteSapResponse.ts b/src/models/Form16DebitNoteSapResponse.ts index 5bcad00..9e708d2 100644 --- a/src/models/Form16DebitNoteSapResponse.ts +++ b/src/models/Form16DebitNoteSapResponse.ts @@ -6,11 +6,16 @@ export interface Form16DebitNoteSapResponseAttributes { id: number; fileName: string; debitNoteId?: number | null; - claimNumber?: string | null; - sapDocumentNumber?: string | null; - msgTyp?: string | null; - message?: string | null; - rawRow?: Record | null; + // Well-known SAP CSV columns stored as individual fields + trnsUniqNo?: string | null; // TRNS_UNIQ_NO – our unique ID echoed back by SAP + tdsTransId?: string | null; // TDS_TRNS_ID – credit note number we sent (primary match key) + claimNumber?: string | null; // CLAIM_NUMBER + sapDocumentNumber?: string | null;// DOC_NO – SAP-generated document number + msgTyp?: string | null; // MSG_TYP + message?: string | null; // MESSAGE + docDate?: string | null; // DOC_DATE + tdsAmt?: string | null; // TDS_AMT + rawRow?: Record | null; // any extra / unknown columns from the CSV storageUrl?: string | null; createdAt: Date; updatedAt: Date; @@ -21,10 +26,14 @@ interface Form16DebitNoteSapResponseCreationAttributes Form16DebitNoteSapResponseAttributes, | 'id' | 'debitNoteId' + | 'trnsUniqNo' + | 'tdsTransId' | 'claimNumber' | 'sapDocumentNumber' | 'msgTyp' | 'message' + | 'docDate' + | 'tdsAmt' | 'rawRow' | 'storageUrl' | 'createdAt' @@ -38,10 +47,14 @@ class Form16DebitNoteSapResponse public id!: number; public fileName!: string; public debitNoteId?: number | null; + public trnsUniqNo?: string | null; + public tdsTransId?: string | null; public claimNumber?: string | null; public sapDocumentNumber?: string | null; public msgTyp?: string | null; public message?: string | null; + public docDate?: string | null; + public tdsAmt?: string | null; public rawRow?: Record | null; public storageUrl?: string | null; public createdAt!: Date; @@ -52,17 +65,21 @@ class Form16DebitNoteSapResponse Form16DebitNoteSapResponse.init( { - id: { type: DataTypes.INTEGER, autoIncrement: true, primaryKey: true }, - fileName: { type: DataTypes.STRING(255), allowNull: false, unique: true, field: 'file_name' }, - debitNoteId: { type: DataTypes.INTEGER, allowNull: true, field: 'debit_note_id' }, - claimNumber: { type: DataTypes.STRING(100), allowNull: true, field: 'claim_number' }, - sapDocumentNumber: { type: DataTypes.STRING(100), allowNull: true, field: 'sap_document_number' }, - msgTyp: { type: DataTypes.STRING(20), allowNull: true, field: 'msg_typ' }, - message: { type: DataTypes.TEXT, allowNull: true }, - rawRow: { type: DataTypes.JSONB, allowNull: true, field: 'raw_row' }, - storageUrl: { type: DataTypes.STRING(500), allowNull: true, field: 'storage_url' }, - createdAt: { type: DataTypes.DATE, allowNull: false, field: 'created_at' }, - updatedAt: { type: DataTypes.DATE, allowNull: false, field: 'updated_at' }, + id: { type: DataTypes.INTEGER, autoIncrement: true, primaryKey: true }, + fileName: { type: DataTypes.STRING(255), allowNull: false, unique: true, field: 'file_name' }, + debitNoteId: { type: DataTypes.INTEGER, allowNull: true, field: 'debit_note_id' }, + trnsUniqNo: { type: DataTypes.STRING(200), allowNull: true, field: 'trns_uniq_no' }, + tdsTransId: { type: DataTypes.STRING(200), allowNull: true, field: 'tds_trns_id' }, + claimNumber: { type: DataTypes.STRING(200), allowNull: true, field: 'claim_number' }, + sapDocumentNumber:{ type: DataTypes.STRING(100), allowNull: true, field: 'sap_document_number' }, + msgTyp: { type: DataTypes.STRING(20), allowNull: true, field: 'msg_typ' }, + message: { type: DataTypes.TEXT, allowNull: true }, + docDate: { type: DataTypes.STRING(20), allowNull: true, field: 'doc_date' }, + tdsAmt: { type: DataTypes.STRING(50), allowNull: true, field: 'tds_amt' }, + rawRow: { type: DataTypes.JSONB, allowNull: true, field: 'raw_row' }, + storageUrl: { type: DataTypes.STRING(500), allowNull: true, field: 'storage_url' }, + createdAt: { type: DataTypes.DATE, allowNull: false, field: 'created_at' }, + updatedAt: { type: DataTypes.DATE, allowNull: false, field: 'updated_at' }, }, { sequelize, diff --git a/src/models/Form16SapResponse.ts b/src/models/Form16SapResponse.ts index 13ac4ae..386037c 100644 --- a/src/models/Form16SapResponse.ts +++ b/src/models/Form16SapResponse.ts @@ -1,19 +1,22 @@ import { DataTypes, Model, Optional } from 'sequelize'; import { sequelize } from '@config/database'; import { Form16CreditNote } from './Form16CreditNote'; -import { Form16DebitNote } from './Form16DebitNote'; export interface Form16SapResponseAttributes { id: number; - type: 'credit' | 'debit'; + type: 'credit'; fileName: string; creditNoteId?: number | null; - debitNoteId?: number | null; - claimNumber?: string | null; - sapDocumentNumber?: string | null; - msgTyp?: string | null; - message?: string | null; - rawRow?: Record | null; + // Well-known SAP CSV columns stored as individual fields + trnsUniqNo?: string | null; // TRNS_UNIQ_NO – our unique ID echoed back by SAP + tdsTransId?: string | null; // TDS_TRNS_ID – credit note number echoed back (primary match key) + claimNumber?: string | null; // CLAIM_NUMBER (alias / fallback) + sapDocumentNumber?: string | null;// DOC_NO – SAP-generated document number + msgTyp?: string | null; // MSG_TYP + message?: string | null; // MESSAGE + docDate?: string | null; // DOC_DATE + tdsAmt?: string | null; // TDS_AMT + rawRow?: Record | null; // any extra / unknown columns from the CSV storageUrl?: string | null; createdAt: Date; updatedAt: Date; @@ -24,11 +27,14 @@ interface Form16SapResponseCreationAttributes Form16SapResponseAttributes, | 'id' | 'creditNoteId' - | 'debitNoteId' + | 'trnsUniqNo' + | 'tdsTransId' | 'claimNumber' | 'sapDocumentNumber' | 'msgTyp' | 'message' + | 'docDate' + | 'tdsAmt' | 'rawRow' | 'storageUrl' | 'createdAt' @@ -40,38 +46,43 @@ class Form16SapResponse implements Form16SapResponseAttributes { public id!: number; - public type!: 'credit' | 'debit'; + public type!: 'credit'; public fileName!: string; public creditNoteId?: number | null; - public debitNoteId?: number | null; + public trnsUniqNo?: string | null; + public tdsTransId?: string | null; public claimNumber?: string | null; public sapDocumentNumber?: string | null; public msgTyp?: string | null; public message?: string | null; + public docDate?: string | null; + public tdsAmt?: string | null; public rawRow?: Record | null; public storageUrl?: string | null; public createdAt!: Date; public updatedAt!: Date; public creditNote?: Form16CreditNote; - public debitNote?: Form16DebitNote; } Form16SapResponse.init( { - id: { type: DataTypes.INTEGER, autoIncrement: true, primaryKey: true }, - type: { type: DataTypes.STRING(10), allowNull: false }, - fileName: { type: DataTypes.STRING(255), allowNull: false, unique: true, field: 'file_name' }, - creditNoteId: { type: DataTypes.INTEGER, allowNull: true, field: 'credit_note_id' }, - debitNoteId: { type: DataTypes.INTEGER, allowNull: true, field: 'debit_note_id' }, - claimNumber: { type: DataTypes.STRING(100), allowNull: true, field: 'claim_number' }, - sapDocumentNumber: { type: DataTypes.STRING(100), allowNull: true, field: 'sap_document_number' }, - msgTyp: { type: DataTypes.STRING(20), allowNull: true, field: 'msg_typ' }, - message: { type: DataTypes.TEXT, allowNull: true }, - rawRow: { type: DataTypes.JSONB, allowNull: true, field: 'raw_row' }, - storageUrl: { type: DataTypes.STRING(500), allowNull: true, field: 'storage_url' }, - createdAt: { type: DataTypes.DATE, allowNull: false, field: 'created_at' }, - updatedAt: { type: DataTypes.DATE, allowNull: false, field: 'updated_at' }, + id: { type: DataTypes.INTEGER, autoIncrement: true, primaryKey: true }, + type: { type: DataTypes.STRING(10), allowNull: false }, + fileName: { type: DataTypes.STRING(255), allowNull: false, unique: true, field: 'file_name' }, + creditNoteId: { type: DataTypes.INTEGER, allowNull: true, field: 'credit_note_id' }, + trnsUniqNo: { type: DataTypes.STRING(200), allowNull: true, field: 'trns_uniq_no' }, + tdsTransId: { type: DataTypes.STRING(200), allowNull: true, field: 'tds_trns_id' }, + claimNumber: { type: DataTypes.STRING(200), allowNull: true, field: 'claim_number' }, + sapDocumentNumber:{ type: DataTypes.STRING(100), allowNull: true, field: 'sap_document_number' }, + msgTyp: { type: DataTypes.STRING(20), allowNull: true, field: 'msg_typ' }, + message: { type: DataTypes.TEXT, allowNull: true }, + docDate: { type: DataTypes.STRING(20), allowNull: true, field: 'doc_date' }, + tdsAmt: { type: DataTypes.STRING(50), allowNull: true, field: 'tds_amt' }, + rawRow: { type: DataTypes.JSONB, allowNull: true, field: 'raw_row' }, + storageUrl: { type: DataTypes.STRING(500), allowNull: true, field: 'storage_url' }, + createdAt: { type: DataTypes.DATE, allowNull: false, field: 'created_at' }, + updatedAt: { type: DataTypes.DATE, allowNull: false, field: 'updated_at' }, }, { sequelize, @@ -89,11 +100,4 @@ Form16SapResponse.belongsTo(Form16CreditNote, { targetKey: 'id', }); -Form16SapResponse.belongsTo(Form16DebitNote, { - as: 'debitNote', - foreignKey: 'debitNoteId', - targetKey: 'id', -}); - export { Form16SapResponse }; - diff --git a/src/scripts/auto-setup.ts b/src/scripts/auto-setup.ts index aec5a81..94cb546 100644 --- a/src/scripts/auto-setup.ts +++ b/src/scripts/auto-setup.ts @@ -179,6 +179,7 @@ async function runMigrations(): Promise { const m62 = require('../migrations/20260317100001-create-form16-sap-responses'); const m63 = require('../migrations/20260317120001-add-form16-trns-uniq-no'); const m64 = require('../migrations/20260318100001-create-form16-debit-note-sap-responses'); + const m65 = require('../migrations/20260318200001-add-sap-response-csv-fields'); const migrations = [ { name: '2025103000-create-users', module: m0 }, @@ -250,6 +251,7 @@ async function runMigrations(): Promise { { name: '20260317100001-create-form16-sap-responses', module: m62 }, { name: '20260317120001-add-form16-trns-uniq-no', module: m63 }, { name: '20260318100001-create-form16-debit-note-sap-responses', module: m64 }, + { name: '20260318200001-add-sap-response-csv-fields', module: m65 }, ]; // Dynamically import sequelize after secrets are loaded diff --git a/src/scripts/migrate.ts b/src/scripts/migrate.ts index aca38db..7fba546 100644 --- a/src/scripts/migrate.ts +++ b/src/scripts/migrate.ts @@ -69,6 +69,7 @@ import * as m61 from '../migrations/20260317-refactor-activity-types-columns'; import * as m62 from '../migrations/20260317100001-create-form16-sap-responses'; import * as m63 from '../migrations/20260317120001-add-form16-trns-uniq-no'; import * as m64 from '../migrations/20260318100001-create-form16-debit-note-sap-responses'; +import * as m65 from '../migrations/20260318200001-add-sap-response-csv-fields'; interface Migration { name: string; @@ -145,6 +146,7 @@ const migrations: Migration[] = [ { name: '20260317100001-create-form16-sap-responses', module: m62 }, { name: '20260317120001-add-form16-trns-uniq-no', module: m63 }, { name: '20260318100001-create-form16-debit-note-sap-responses', module: m64 }, + { name: '20260318200001-add-sap-response-csv-fields', module: m65 }, ]; diff --git a/src/server.ts b/src/server.ts index 5d8e0b3..e963e52 100644 --- a/src/server.ts +++ b/src/server.ts @@ -117,8 +117,6 @@ const startServer = async (): Promise => { startPauseResumeJob(); const { startForm16NotificationJobs } = require('./jobs/form16NotificationJob'); startForm16NotificationJobs(); - const { startForm16SapResponseJob } = require('./jobs/form16SapResponseJob'); - startForm16SapResponseJob(); const { startForm16ArchiveJob } = require('./services/form16Archive.service'); startForm16ArchiveJob(); diff --git a/src/services/form16.service.ts b/src/services/form16.service.ts index f3bcd30..36fca88 100644 --- a/src/services/form16.service.ts +++ b/src/services/form16.service.ts @@ -565,9 +565,9 @@ async function run26asMatchAndCreditNote(submission: Form16aSubmission): Promise ); await submission.update({ validationStatus: 'failed', - validationNotes: 'No 26AS data found for this TAN, financial year and quarter. Please ensure 26AS has been uploaded for this period.', + validationNotes: `No 26AS data found for TAN no - ${tanNumber}, financial year and quarter. Please ensure 26AS has been uploaded for this period.`, }); - return { validationStatus: 'failed', validationNotes: 'No 26AS record found for this TAN, financial year and quarter.' }; + return { validationStatus: 'failed', validationNotes: `No 26AS record found for this TAN no - ${tanNumber}, financial year and quarter.` }; } const amountTolerance = 1; // allow 1 rupee rounding @@ -577,11 +577,11 @@ async function run26asMatchAndCreditNote(submission: Form16aSubmission): Promise ); await submission.update({ validationStatus: 'failed', - validationNotes: `Amount mismatch with latest 26AS. Form 16A TDS amount: ${tdsAmount}. Latest 26AS aggregated amount for this quarter: ${aggregated26as}. Please submit Form 16 with correct data.`, + validationNotes: `Amount mismatch with latest 26AS for TAN no - ${tanNumber}. Form 16A TDS amount: ${tdsAmount}. Latest 26AS aggregated amount for this quarter: ${aggregated26as}. Please submit Form 16 with correct data.`, }); return { validationStatus: 'failed', - validationNotes: 'Amount mismatch with latest 26AS. Please verify the certificate and resubmit.', + validationNotes: `Amount mismatch with latest 26AS for TAN no - ${tanNumber}. Please verify the certificate and resubmit.`, }; } diff --git a/src/services/wfmFile.service.ts b/src/services/wfmFile.service.ts index e3d5fd4..d5e4598 100644 --- a/src/services/wfmFile.service.ts +++ b/src/services/wfmFile.service.ts @@ -6,7 +6,7 @@ import logger from '../utils/logger'; const DEFAULT_CLAIMS_INCOMING = path.join('WFM-QRE', 'INCOMING', 'WFM_MAIN', 'DLR_INC_CLAIMS'); const DEFAULT_CLAIMS_OUTGOING = path.join('WFM-QRE', 'OUTGOING', 'WFM_SAP_MAIN', 'DLR_INC_CLAIMS'); const DEFAULT_FORM16_CREDIT_INCOMING = path.join('WFM-QRE', 'INCOMING', 'WFM_MAIN', 'FORM16_CRDT'); -const DEFAULT_FORM16_DEBIT_INCOMING = path.join('WFM-QRE', 'INCOMING', 'WFM_MAIN', 'FORM16_DEBT'); +const DEFAULT_FORM16_DEBIT_INCOMING = path.join('WFM-QRE', 'INCOMING', 'WFM_MAIN', 'FORM16_DBT'); const DEFAULT_FORM16_CREDIT_OUTGOING = path.join('WFM-QRE', 'OUTGOING', 'WFM_SAP_MAIN', 'FORM16_CRDT'); const DEFAULT_FORM16_DEBIT_OUTGOING = path.join('WFM-QRE', 'OUTGOING', 'WFM_SAP_MAIN', 'FORM16_DBT');