diff --git a/src/controllers/admin.controller.ts b/src/controllers/admin.controller.ts index 840649d..264e504 100644 --- a/src/controllers/admin.controller.ts +++ b/src/controllers/admin.controller.ts @@ -556,12 +556,12 @@ const DEFAULT_FORM16_CONFIG = { alertSubmitForm16FrequencyDays: 0, alertSubmitForm16FrequencyHours: 24, alertSubmitForm16RunAtTime: '09:00', - alertSubmitForm16Template: 'Please submit your Form 16 at your earliest. [Name], due date: [DueDate].', + alertSubmitForm16Template: 'Dear [Name], please submit Form 16A for the pending period. Due: [DueDate].', reminderNotificationEnabled: true, reminderFrequencyDays: 0, reminderFrequencyHours: 12, reminderRunAtTime: '10:00', - reminderNotificationTemplate: 'Reminder: Form 16 submission is pending. [Name], [Request ID]. Please review.', + reminderNotificationTemplate: 'Reminder: Dear [Name], your Form 16A submission is pending for request [Request ID]. Please complete it.', }; /** diff --git a/src/controllers/form16.controller.ts b/src/controllers/form16.controller.ts index e78d3bd..42eedd1 100644 --- a/src/controllers/form16.controller.ts +++ b/src/controllers/form16.controller.ts @@ -16,12 +16,32 @@ import { ResponseHandler } from '../utils/responseHandler'; import logger from '../utils/logger'; import { WorkflowRequest } from '@models/WorkflowRequest'; import { Form16aSubmission } from '@models/Form16aSubmission'; +import { Dealer } from '@models/Dealer'; /** * Form 16 controller: credit notes, OCR extract, and create submission for dealers. */ export class Form16Controller { + private toSapCsv(sap: { + trnsUniqNo?: string | null; + tdsTransId?: string | null; + sapDocumentNumber?: string | null; + msgTyp?: string | null; + message?: string | null; + }): string { + const header = ['TRNS_UNIQ_NO', 'TDS_TRNS_ID', 'DOC_NO', 'MSG_TYP', 'MESSAGE'].join('|'); + const row = [ + sap.trnsUniqNo || '', + sap.tdsTransId || '', + sap.sapDocumentNumber || '', + sap.msgTyp || '', + sap.message || '', + ] + .map((v) => String(v).replace(/\r?\n/g, ' ').replace(/\|/g, ' ')) + .join('|'); + return `${header}\n${row}\n`; + } /** * GET /api/v1/form16/permissions * Returns Form 16 permissions for the current user (API-driven from admin config). @@ -174,12 +194,31 @@ export class Form16Controller { if (!userId) { return ResponseHandler.unauthorized(res, 'Authentication required'); } - const body = (req.body || {}) as { dealerCode?: string; financialYear?: string }; - const dealerCode = (body.dealerCode || '').trim(); + const body = (req.body || {}) as { dealerCode?: string; dealerId?: string; email?: string; financialYear?: string }; + const financialYear = (body.financialYear || '').trim() || undefined; + let dealerCode = (body.dealerCode || '').trim(); + const dealerId = (body.dealerId || '').trim(); + const dealerEmail = (body.email || '').trim().toLowerCase(); + + // Fallback 1: resolve by Dealer PK (when FE sends id but dealerCode is empty). + if (!dealerCode && dealerId) { + const dealer = await Dealer.findByPk(dealerId, { attributes: ['salesCode', 'dlrcode'] }); + dealerCode = String((dealer as any)?.salesCode || (dealer as any)?.dlrcode || '').trim(); + } + + // Fallback 2: resolve from non-submitted list (supports id/email based payloads reliably). + if (!dealerCode) { + const list = await form16Service.listNonSubmittedDealers(financialYear); + const match = list.dealers.find((d) => + (dealerId && String(d.id).trim() === dealerId) || + (dealerEmail && String(d.email || '').trim().toLowerCase() === dealerEmail) + ); + dealerCode = String(match?.dealerCode || '').trim(); + } + if (!dealerCode) { return ResponseHandler.error(res, 'dealerCode is required', 400); } - const financialYear = (body.financialYear || '').trim() || undefined; const updated = await form16Service.recordNonSubmittedDealerNotification(dealerCode, financialYear || '', userId); if (!updated) { return ResponseHandler.error(res, 'Dealer not found in non-submitted list for this financial year', 404); @@ -245,6 +284,7 @@ export class Form16Controller { } const taxDeducted = typeof body.taxDeducted === 'number' ? body.taxDeducted : parseFloat(String(body.taxDeducted ?? 0)); const entry = await form16Service.create26asEntry({ + panNumber: (body.panNumber as string) || undefined, tanNumber, deductorName: (body.deductorName as string) || undefined, quarter, @@ -281,6 +321,7 @@ export class Form16Controller { const body = req.body as Record; const updateData: Record = {}; if (body.tanNumber !== undefined) updateData.tanNumber = body.tanNumber; + if (body.panNumber !== undefined) updateData.panNumber = body.panNumber; if (body.deductorName !== undefined) updateData.deductorName = body.deductorName; if (body.quarter !== undefined) updateData.quarter = body.quarter; if (body.assessmentYear !== undefined) updateData.assessmentYear = body.assessmentYear; @@ -382,7 +423,8 @@ export class Form16Controller { res, { sapResponse, - url: sapResponse.storageUrl || null, + // Use API-backed CSV URL so View works even when local /uploads file is unavailable in UAT. + url: `/api/v1/form16/credit-notes/${id}/sap-response/csv`, }, 'OK' ); @@ -395,8 +437,8 @@ export class Form16Controller { /** * GET /api/v1/form16/credit-notes/:id/download - * Returns a storage URL for the SAP response CSV if available. - * If not yet available, returns 409 so UI can show "being generated, wait". + * Backward-compatible route that now always returns API-backed CSV URL. + * The CSV itself is generated from persisted DB fields (no /uploads dependency). */ async downloadCreditNote(req: Request, res: Response): Promise { try { @@ -408,9 +450,9 @@ export class Form16Controller { if (Number.isNaN(id)) { return ResponseHandler.error(res, 'Invalid credit note id', 400); } - let url: string | null = null; + let sapResponse = null; try { - url = await form16Service.getCreditNoteSapResponseUrlForUser(id, userId); + sapResponse = await form16Service.getCreditNoteSapResponseForUser(id, userId); } catch (e: any) { const msg = String(e?.message || ''); if (msg.toLowerCase().includes('not found')) { @@ -418,10 +460,10 @@ export class Form16Controller { } throw e; } - if (!url) { + if (!sapResponse) { return ResponseHandler.error(res, 'The credit note is being generated. Please wait.', 409); } - return ResponseHandler.success(res, { url }, 'OK'); + return ResponseHandler.success(res, { url: `/api/v1/form16/credit-notes/${id}/sap-response/csv` }, 'OK'); } catch (error) { const errorMessage = error instanceof Error ? error.message : 'Unknown error'; logger.error('[Form16Controller] downloadCreditNote error:', error); @@ -448,7 +490,8 @@ export class Form16Controller { res, { sapResponse, - url: sapResponse.storageUrl || null, + // Use API-backed CSV URL so View works even when local /uploads file is unavailable in UAT. + url: `/api/v1/form16/debit-notes/${id}/sap-response/csv`, }, 'OK' ); @@ -459,6 +502,64 @@ export class Form16Controller { } } + /** + * GET /api/v1/form16/credit-notes/:id/sap-response/csv + * Stream SAP response CSV generated from persisted DB fields. + */ + async downloadCreditNoteSapResponseCsv(req: Request, res: Response): Promise { + try { + const userId = (req as AuthenticatedRequest).user?.userId; + if (!userId) { + return ResponseHandler.unauthorized(res, 'Authentication required'); + } + const id = parseInt((req.params as { id: string }).id, 10); + if (Number.isNaN(id)) { + return ResponseHandler.error(res, 'Invalid credit note id', 400); + } + const sapResponse = await form16Service.getCreditNoteSapResponseForUser(id, userId); + if (!sapResponse) { + return ResponseHandler.error(res, 'The credit note is being generated. Please wait.', 409); + } + const csv = this.toSapCsv(sapResponse); + const fileName = (sapResponse.fileName || `credit-note-${id}.csv`).replace(/[^a-zA-Z0-9._-]/g, '_'); + res.setHeader('Content-Type', 'text/csv; charset=utf-8'); + res.setHeader('Content-Disposition', `attachment; filename="${fileName}"`); + res.status(200).send(csv); + return; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + logger.error('[Form16Controller] downloadCreditNoteSapResponseCsv error:', error); + return ResponseHandler.error(res, 'Failed to fetch credit note SAP response CSV', 500, errorMessage); + } + } + + /** + * GET /api/v1/form16/debit-notes/:id/sap-response/csv + * Stream SAP response CSV generated from persisted DB fields. + */ + async downloadDebitNoteSapResponseCsv(req: Request, res: Response): Promise { + try { + const id = parseInt((req.params as { id: string }).id, 10); + if (Number.isNaN(id)) { + return ResponseHandler.error(res, 'Invalid debit note id', 400); + } + const sapResponse = await form16Service.getDebitNoteSapResponse(id); + if (!sapResponse) { + return ResponseHandler.error(res, 'The debit note is being generated. Please wait.', 409); + } + const csv = this.toSapCsv(sapResponse); + const fileName = (sapResponse.fileName || `debit-note-${id}.csv`).replace(/[^a-zA-Z0-9._-]/g, '_'); + res.setHeader('Content-Type', 'text/csv; charset=utf-8'); + res.setHeader('Content-Disposition', `attachment; filename="${fileName}"`); + res.status(200).send(csv); + return; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + logger.error('[Form16Controller] downloadDebitNoteSapResponseCsv error:', error); + return ResponseHandler.error(res, 'Failed to fetch debit note SAP response CSV', 500, errorMessage); + } + } + /** * GET /api/v1/form16/requests/:requestId/credit-note * Get credit note (if any) linked to a Form 16 request. Used on Form 16 details workflow tab. diff --git a/src/jobs/form16SapResponseJob.ts b/src/jobs/form16SapResponseJob.ts index cb27342..a7172c8 100644 --- a/src/jobs/form16SapResponseJob.ts +++ b/src/jobs/form16SapResponseJob.ts @@ -6,322 +6,97 @@ import { Form16CreditNote, Form16DebitNote, Form16SapResponse, - Form16DebitNoteSapResponse, - Form16aSubmission, - WorkflowRequest, + From16SapReadFile, } from '../models'; -import { gcsStorageService } from '../services/gcsStorage.service'; -// ─── Helpers ───────────────────────────────────────────────────────────────── +type CsvRow = Record; -function safeFileName(name: string): string { - return (name || '').trim().replace(/[\\\/:*?"<>|]+/g, '-').slice(0, 180) || 'form16-sap-response.csv'; +function extractCsvFields(r: CsvRow) { + const trnsUniqNo = (r.TRNS_UNIQ_NO || r.TRNSUNIQNO || '').trim() || null; + const tdsTransId = (r.TDS_TRNS_ID || '').trim() || null; + const docNo = (r.DOC_NO || r.DOCNO || '').trim() || null; + const msgTyp = (r.MSG_TYP || r.MSGTYP || '').trim() || null; + const message = (r.MESSAGE || '').trim() || null; + return { trnsUniqNo, tdsTransId, docNo, msgTyp, message }; } -/** Columns we store in dedicated DB fields. Everything else goes into raw_row. */ -const KNOWN_CSV_COLUMNS = new Set([ - 'TRNS_UNIQ_NO', 'TRNSUNIQNO', 'DMS_UNIQ_NO', 'DMSUNIQNO', - 'TDS_TRNS_ID', - 'CLAIM_NUMBER', - 'DOC_NO', 'DOCNO', 'SAP_DOC_NO', 'SAPDOC', - 'MSG_TYP', 'MSGTYP', 'MSG_TYPE', - 'MESSAGE', 'MSG', - 'DOC_DATE', 'DOCDATE', - 'TDS_AMT', 'TDSAMT', -]); - -/** - * Parse all columns from one CSV data row. - * Returns { known fields } + rawRow (only the columns NOT in KNOWN_CSV_COLUMNS). - */ -function extractCsvFields(r: Record) { - const trnsUniqNo = (r.TRNS_UNIQ_NO || r.TRNSUNIQNO || r.DMS_UNIQ_NO || r.DMSUNIQNO || '').trim() || null; - const tdsTransId = (r.TDS_TRNS_ID || '').trim() || null; - const claimNumber = (r.CLAIM_NUMBER || '').trim() || null; - const sapDocNo = (r.DOC_NO || r.DOCNO || r.SAP_DOC_NO || r.SAPDOC || '').trim() || null; - const msgTyp = (r.MSG_TYP || r.MSGTYP || r.MSG_TYPE || '').trim() || null; - const message = (r.MESSAGE || r.MSG || '').trim() || null; - const docDate = (r.DOC_DATE || r.DOCDATE || '').trim() || null; - const tdsAmt = (r.TDS_AMT || r.TDSAMT || '').trim() || null; - - // Extra columns → raw_row (so nothing is ever lost) - const rawRow: Record = {}; - for (const [key, val] of Object.entries(r)) { - if (!KNOWN_CSV_COLUMNS.has(key.trim().toUpperCase()) && !KNOWN_CSV_COLUMNS.has(key.trim())) { - rawRow[key.trim()] = val || ''; - } - } - - return { trnsUniqNo, tdsTransId, claimNumber, sapDocNo, msgTyp, message, docDate, tdsAmt, rawRow }; +function isUsableRow(r: CsvRow): boolean { + const { tdsTransId } = extractCsvFields(r); + if (!tdsTransId) return false; + const upper = tdsTransId.toUpperCase(); + if (upper === 'TDS_TRNS_ID' || upper === 'MSG_TYP' || upper === 'MESSAGE') return false; + return true; } -// ─── Credit note matching ───────────────────────────────────────────────────── +async function saveRowsAndUpdateNotes(rows: CsvRow[]): Promise<{ totalRecords: number; totalCreditNotes: number; totalDebitNotes: number }> { + let totalRecords = 0; + let totalCreditNotes = 0; + let totalDebitNotes = 0; -async function findCreditNoteId( - trnsUniqNo: string | null, - tdsTransId: string | null, - claimNumber: string | null, - fileName: string, -): Promise<{ creditNoteId: number | null; requestId: string | null }> { - const CN = Form16CreditNote as any; - let cn: any = null; + for (const row of rows) { + if (!isUsableRow(row)) continue; + const parsed = extractCsvFields(row); + if (!parsed.tdsTransId) continue; - // 1. Primary: TDS_TRNS_ID in SAP response = credit note number we sent - if (tdsTransId) { - cn = await CN.findOne({ where: { creditNoteNumber: tdsTransId }, attributes: ['id', 'submissionId'] }); - if (cn) logger.info(`[Form16 SAP Job] Credit match via TDS_TRNS_ID=${tdsTransId} → credit_note id=${cn.id}`); - } + await (Form16SapResponse as any).create({ + trnsUniqNo: parsed.trnsUniqNo, + tdsTransId: parsed.tdsTransId, + docNo: parsed.docNo, + msgTyp: parsed.msgTyp, + message: parsed.message, + createdAt: new Date(), + updatedAt: new Date(), + }); + totalRecords++; - // 2. TRNS_UNIQ_NO (format: F16-CN-{submissionId}-{creditNoteId}-{ts}) - if (!cn && trnsUniqNo) { - const m = trnsUniqNo.match(/^F16-CN-(\d+)-(\d+)-/); - if (m) { - cn = await CN.findByPk(parseInt(m[2]), { attributes: ['id', 'submissionId'] }); - if (cn) logger.info(`[Form16 SAP Job] Credit match via TRNS_UNIQ_NO id-parse=${m[2]} → credit_note id=${cn.id}`); - } - if (!cn) { - cn = await CN.findOne({ where: { trnsUniqNo }, attributes: ['id', 'submissionId'] }); - if (cn) logger.info(`[Form16 SAP Job] Credit match via trns_uniq_no=${trnsUniqNo} → credit_note id=${cn.id}`); - } - } - - // 3. Filename (without .csv) = credit note number - if (!cn) { - const baseName = fileName.replace(/\.csv$/i, '').trim(); - if (baseName) { - cn = await CN.findOne({ where: { creditNoteNumber: baseName }, attributes: ['id', 'submissionId'] }); - if (cn) logger.info(`[Form16 SAP Job] Credit match via filename=${baseName} → credit_note id=${cn.id}`); - } - } - - // 4. CLAIM_NUMBER = credit note number (seen in some SAP/WFM exports) - if (!cn && claimNumber) { - cn = await CN.findOne({ where: { creditNoteNumber: claimNumber }, attributes: ['id', 'submissionId'] }); - if (cn) logger.info(`[Form16 SAP Job] Credit match via CLAIM_NUMBER=${claimNumber} → credit_note id=${cn.id}`); - } - - if (!cn) return { creditNoteId: null, requestId: null }; - - const submission = await (Form16aSubmission as any).findByPk(cn.submissionId, { attributes: ['requestId'] }); - return { creditNoteId: cn.id, requestId: submission?.requestId ?? null }; -} - -// ─── Debit note matching ────────────────────────────────────────────────────── - -async function findDebitNoteId( - trnsUniqNo: string | null, - tdsTransId: string | null, - claimNumber: string | null, - fileName: string, -): Promise { - const DN = Form16DebitNote as any; - const CN = Form16CreditNote as any; - let dn: any = null; - - // 1. Primary: TRNS_UNIQ_NO (format: F16-DN-{creditNoteId}-{debitNoteId}-{ts}) - if (trnsUniqNo) { - const m = trnsUniqNo.match(/^F16-DN-(\d+)-(\d+)-/); - if (m) { - dn = await DN.findByPk(parseInt(m[2]), { attributes: ['id'] }); - if (dn) logger.info(`[Form16 SAP Job] Debit match via TRNS_UNIQ_NO id-parse=${m[2]} → debit_note id=${dn.id}`); - } - if (!dn) { - dn = await DN.findOne({ where: { trnsUniqNo }, attributes: ['id'] }); - if (dn) logger.info(`[Form16 SAP Job] Debit match via trns_uniq_no=${trnsUniqNo} → debit_note id=${dn.id}`); - } - } - - // 2. TDS_TRNS_ID = credit note number → find linked debit note - if (!dn && tdsTransId) { - const cn = await CN.findOne({ where: { creditNoteNumber: tdsTransId }, attributes: ['id'] }); - if (cn) { - dn = await DN.findOne({ - where: { creditNoteId: cn.id }, - order: [['createdAt', 'DESC']], - attributes: ['id'], - }); - if (dn) logger.info(`[Form16 SAP Job] Debit match via TDS_TRNS_ID=${tdsTransId} → credit_note id=${cn.id} → debit_note id=${dn.id}`); - } - } - - // 3. CLAIM_NUMBER = debit note number - if (!dn && claimNumber) { - dn = await DN.findOne({ where: { debitNoteNumber: claimNumber }, attributes: ['id'] }); - if (dn) logger.info(`[Form16 SAP Job] Debit match via CLAIM_NUMBER=${claimNumber} → debit_note id=${dn.id}`); - } - - // 4. Filename (without .csv) = debit note number - if (!dn) { - const baseName = fileName.replace(/\.csv$/i, '').trim(); - if (baseName) { - dn = await DN.findOne({ where: { debitNoteNumber: baseName }, attributes: ['id'] }); - if (dn) logger.info(`[Form16 SAP Job] Debit match via filename=${baseName} → debit_note id=${dn.id}`); - } - } - - return dn ? dn.id : null; -} - -// ─── Core processor ─────────────────────────────────────────────────────────── - -async function processOutgoingFile( - fileName: string, - type: 'credit' | 'debit', - resolvedOutgoingDir: string, -): Promise { - const CreditModel = Form16SapResponse as any; - const DebitModel = Form16DebitNoteSapResponse as any; - - // Idempotency: skip if already fully linked - const existing = - type === 'credit' - ? await CreditModel.findOne({ where: { fileName }, attributes: ['id', 'creditNoteId', 'sapDocumentNumber', 'storageUrl'] }) - : await DebitModel.findOne({ where: { fileName }, attributes: ['id', 'debitNoteId', 'sapDocumentNumber', 'storageUrl'] }); - - if (existing && (existing.creditNoteId ?? existing.debitNoteId) && (existing.storageUrl || existing.sapDocumentNumber)) { - logger.debug(`[Form16 SAP Job] Skipping already-processed ${type} file: ${fileName}`); - return; - } - - // ── Read CSV ── - const rows = await wfmFileService.readForm16OutgoingResponseByPath(path.join(resolvedOutgoingDir, fileName)); - if (!rows || rows.length === 0) { - logger.warn(`[Form16 SAP Job] ${type} file ${fileName}: empty or unreadable CSV`); - const emptyPayload = { rawRow: null, updatedAt: new Date() }; - if (existing) { - type === 'credit' ? await CreditModel.update(emptyPayload, { where: { id: existing.id } }) - : await DebitModel.update(emptyPayload, { where: { id: existing.id } }); - } else { - type === 'credit' ? await CreditModel.create({ type, fileName, ...emptyPayload, createdAt: new Date() }) - : await DebitModel.create({ fileName, ...emptyPayload, createdAt: new Date() }); - } - return; - } - - // ── Pick the best data row ── - // Skip the degenerate "|MSG_TYP|MESSAGE|" lines that some SAP exports include after the header. - type CsvRow = Record; - const normalizedRows = rows as CsvRow[]; - const pick = - normalizedRows.find((row) => { - const trns = (row.TRNS_UNIQ_NO || row.TRNSUNIQNO || row.DMS_UNIQ_NO || '').trim(); - return Boolean(trns); - }) || - normalizedRows.find((row) => { - const tdsId = (row.TDS_TRNS_ID || '').trim(); - const docNo = (row.DOC_NO || row.DOCNO || '').trim(); - const msgTyp = (row.MSG_TYP || '').trim(); - if (!tdsId) return false; - if (!docNo && !msgTyp) return false; - if (['MSG_TYP', 'MESSAGE', 'TDS_TRNS_ID'].includes(tdsId.toUpperCase())) return false; - return true; - }) || - normalizedRows[0]; - - const r = pick as CsvRow; - const { trnsUniqNo, tdsTransId, claimNumber, sapDocNo, msgTyp, message, docDate, tdsAmt, rawRow } = extractCsvFields(r); - - logger.info( - `[Form16 SAP Job] Processing ${type} file ${fileName}: TRNS_UNIQ_NO=${trnsUniqNo ?? '—'}, TDS_TRNS_ID=${tdsTransId ?? '—'}, CLAIM_NUMBER=${claimNumber ?? '—'}, DOC_NO=${sapDocNo ?? '—'}` - ); - - // ── Match to a note in DB ── - let creditNoteId: number | null = null; - let debitNoteId: number | null = null; - let requestId: string | null = null; - let requestNumber: string | null = null; - - if (type === 'credit') { - const res = await findCreditNoteId(trnsUniqNo, tdsTransId, claimNumber, fileName); - creditNoteId = res.creditNoteId; - requestId = res.requestId; - if (creditNoteId && sapDocNo) { + const idUpper = parsed.tdsTransId.toUpperCase(); + if (idUpper.startsWith('CN')) { + totalCreditNotes++; await (Form16CreditNote as any).update( - { sapDocumentNumber: sapDocNo, status: 'completed' }, - { where: { id: creditNoteId } } + { + sapDocumentNumber: parsed.docNo, + status: 'completed', + }, + { where: { creditNoteNumber: parsed.tdsTransId } } ); - } - if (!creditNoteId) { - logger.warn( - `[Form16 SAP Job] Credit file ${fileName}: no matching credit note. TDS_TRNS_ID=${tdsTransId ?? '—'}, TRNS_UNIQ_NO=${trnsUniqNo ?? '—'}.` - ); - } - } else { - debitNoteId = await findDebitNoteId(trnsUniqNo, tdsTransId, claimNumber, fileName); - if (debitNoteId && sapDocNo) { + } else if (idUpper.startsWith('DN')) { + totalDebitNotes++; await (Form16DebitNote as any).update( - { sapDocumentNumber: sapDocNo, status: 'completed' }, - { where: { id: debitNoteId } } - ); - // Fetch requestId from linked credit note → submission - const dn = await (Form16DebitNote as any).findByPk(debitNoteId, { attributes: ['creditNoteId'] }); - if (dn?.creditNoteId) { - const cn = await (Form16CreditNote as any).findByPk(dn.creditNoteId, { attributes: ['submissionId'] }); - if (cn?.submissionId) { - const sub = await (Form16aSubmission as any).findByPk(cn.submissionId, { attributes: ['requestId'] }); - requestId = sub?.requestId ?? null; - } - } - } - if (!debitNoteId) { - logger.warn( - `[Form16 SAP Job] Debit file ${fileName}: no matching debit note. TRNS_UNIQ_NO=${trnsUniqNo ?? '—'}, TDS_TRNS_ID=${tdsTransId ?? '—'}, CLAIM_NUMBER=${claimNumber ?? '—'}.` + { + sapDocumentNumber: parsed.docNo, + status: 'completed', + }, + { where: { debitNoteNumber: parsed.tdsTransId } } ); } } - if (requestId) { - const req = await (WorkflowRequest as any).findOne({ where: { requestId }, attributes: ['requestNumber'] }); - requestNumber = req?.requestNumber ?? null; + return { totalRecords, totalCreditNotes, totalDebitNotes }; +} + +async function processOutgoingFile(fileName: string, resolvedOutgoingDir: string): Promise<{ totalRecords: number; totalCreditNotes: number; totalDebitNotes: number } | null> { + const alreadyRead = await (From16SapReadFile as any).findOne({ + where: { fileName }, + attributes: ['id'], + }); + if (alreadyRead) { + logger.debug(`[Form16 SAP Job] Skipping already-read file: ${fileName}`); + return null; } - // ── Upload raw CSV to storage ── - const absPath = path.join(resolvedOutgoingDir, fileName); - let storageUrl: string | null = null; - try { - if (fs.existsSync(absPath)) { - const buffer = fs.readFileSync(absPath); - const upload = await gcsStorageService.uploadFileWithFallback({ - buffer, - originalName: safeFileName(fileName), - mimeType: 'text/csv', - requestNumber: requestNumber || trnsUniqNo || 'FORM16', - fileType: 'documents', - }); - storageUrl = upload.storageUrl || null; - } - } catch (e) { - logger.error('[Form16 SAP Job] Failed to upload response file:', fileName, e); - } + const rows = (await wfmFileService.readForm16OutgoingResponseByPath(path.join(resolvedOutgoingDir, fileName))) as CsvRow[]; + const counts = await saveRowsAndUpdateNotes(rows || []); - // ── Persist to DB ── - const commonFields = { - trnsUniqNo, - tdsTransId, - claimNumber, - sapDocumentNumber: sapDocNo, - msgTyp, - message, - docDate, - tdsAmt, - rawRow: Object.keys(rawRow).length ? rawRow : null, - storageUrl, + await (From16SapReadFile as any).create({ + fileName, + totalRecords: counts.totalRecords, + totalCreditNotes: counts.totalCreditNotes, + totalDebitNotes: counts.totalDebitNotes, + createdAt: new Date(), updatedAt: new Date(), - }; + }); - if (type === 'credit') { - const payload = { type: 'credit' as const, fileName, creditNoteId, ...commonFields }; - if (existing) await CreditModel.update(payload, { where: { id: existing.id } }); - else await CreditModel.create({ ...payload, createdAt: new Date() }); - } else { - const payload = { fileName, debitNoteId, ...commonFields }; - if (existing) await DebitModel.update(payload, { where: { id: existing.id } }); - else await DebitModel.create({ ...payload, createdAt: new Date() }); - } - - logger.info( - `[Form16 SAP Job] Saved ${type} SAP response for file ${fileName} → ${type === 'credit' ? `credit_note_id=${creditNoteId}` : `debit_note_id=${debitNoteId}`}, storage_url=${storageUrl ? 'yes' : 'no'}` - ); + return counts; } // ─── Public API (called by Pull button controller) ──────────────────────────── @@ -337,40 +112,36 @@ export async function runForm16SapResponseIngestionOnce(): Promise<{ processed: number; creditProcessed: number; debitProcessed: number; + filesProcessed: number; }> { let creditProcessed = 0; let debitProcessed = 0; + let filesProcessed = 0; - const RELATIVE_CREDIT_OUT = path.join('WFM-QRE', 'OUTGOING', 'WFM_SAP_MAIN', 'FORM16_CRDT'); - const RELATIVE_DEBIT_OUT = path.join('WFM-QRE', 'OUTGOING', 'WFM_SAP_MAIN', 'FORM16_DBT'); - - const dirs: Array<{ dir: string; type: 'credit' | 'debit'; relSubdir: string }> = [ - { - dir: path.dirname(wfmFileService.getForm16OutgoingPath('__probe__.csv', 'credit')), - type: 'credit', - relSubdir: RELATIVE_CREDIT_OUT, - }, - { - dir: path.dirname(wfmFileService.getForm16OutgoingPath('__probe__.csv', 'debit')), - type: 'debit', - relSubdir: RELATIVE_DEBIT_OUT, - }, + const RELATIVE_FORM16_OUT = path.join('WFM-QRE', 'OUTGOING', 'WFM_SAP_MAIN', 'FORM16'); + const resolvedDirs = [ + path.dirname(wfmFileService.getForm16OutgoingPath('__probe__.csv', 'credit')), + path.dirname(wfmFileService.getForm16OutgoingPath('__probe__.csv', 'debit')), ]; + const dirs: Array<{ dir: string; relSubdir: string }> = [...new Set(resolvedDirs)].map((d) => ({ + dir: d, + relSubdir: RELATIVE_FORM16_OUT, + })); try { const base = process.env.WFM_BASE_PATH || 'C:\\WFM'; - for (const { dir, type, relSubdir } of dirs) { + for (const { dir, relSubdir } of dirs) { let abs = path.isAbsolute(dir) ? dir : path.join(base, dir); if (!fs.existsSync(abs)) { const cwdFallback = path.join(process.cwd(), relSubdir); if (fs.existsSync(cwdFallback)) { abs = cwdFallback; - logger.info(`[Form16 SAP Job] ${type} OUTGOING dir resolved via cwd: ${abs}`); + logger.info(`[Form16 SAP Job] OUTGOING dir resolved via cwd: ${abs}`); } else { logger.warn( - `[Form16 SAP Job] ${type} OUTGOING dir not found. Tried: ${abs} | ${cwdFallback}. ` + + `[Form16 SAP Job] OUTGOING dir not found. Tried: ${abs} | ${cwdFallback}. ` + `Set WFM_BASE_PATH to the folder containing WFM-QRE.` ); continue; @@ -378,17 +149,17 @@ export async function runForm16SapResponseIngestionOnce(): Promise<{ } const files = fs.readdirSync(abs).filter((f) => f.toLowerCase().endsWith('.csv')); - logger.info( - `[Form16 SAP Job] ${type} OUTGOING dir: ${abs} → ${files.length} CSV file(s)${files.length ? ': ' + files.join(', ') : ''}` - ); + logger.info(`[Form16 SAP Job] OUTGOING dir: ${abs} → ${files.length} CSV file(s)${files.length ? ': ' + files.join(', ') : ''}`); for (const f of files) { try { - await processOutgoingFile(f, type, abs); - if (type === 'credit') creditProcessed++; - else debitProcessed++; + const counts = await processOutgoingFile(f, abs); + if (!counts) continue; + filesProcessed++; + creditProcessed += counts.totalCreditNotes; + debitProcessed += counts.totalDebitNotes; } catch (e) { - logger.error(`[Form16 SAP Job] Error processing ${type} file ${f}:`, e); + logger.error(`[Form16 SAP Job] Error processing file ${f}:`, e); } } } @@ -404,5 +175,6 @@ export async function runForm16SapResponseIngestionOnce(): Promise<{ processed: creditProcessed + debitProcessed, creditProcessed, debitProcessed, + filesProcessed, }; } diff --git a/src/migrations/20260324090001-refactor-form16-sap-response-and-add-read-log.ts b/src/migrations/20260324090001-refactor-form16-sap-response-and-add-read-log.ts new file mode 100644 index 0000000..531e495 --- /dev/null +++ b/src/migrations/20260324090001-refactor-form16-sap-response-and-add-read-log.ts @@ -0,0 +1,87 @@ +import type { QueryInterface } from 'sequelize'; +import { DataTypes } from 'sequelize'; + +module.exports = { + up: async (queryInterface: QueryInterface) => { + // 1) Create read-log table for processed SAP CSV files + await queryInterface.createTable('from16_sap_read_file', { + id: { type: DataTypes.INTEGER, autoIncrement: true, primaryKey: true }, + file_name: { type: DataTypes.STRING(255), allowNull: false, unique: true }, + total_records: { type: DataTypes.INTEGER, allowNull: false, defaultValue: 0 }, + total_credit_notes: { type: DataTypes.INTEGER, allowNull: false, defaultValue: 0 }, + total_debit_notes: { type: DataTypes.INTEGER, allowNull: false, defaultValue: 0 }, + created_at: { type: DataTypes.DATE, allowNull: false, defaultValue: DataTypes.NOW }, + updated_at: { type: DataTypes.DATE, allowNull: false, defaultValue: DataTypes.NOW }, + }).catch(() => {}); + + // 2) Add required new fields to form16_sap_responses + await queryInterface.addColumn('form16_sap_responses', 'doc_no', { + type: DataTypes.STRING(200), + allowNull: true, + }).catch(() => {}); + + // 3) Drop old fields from form16_sap_responses (as requested) + for (const col of [ + 'type', + 'file_name', + 'credit_note_id', + 'debit_note_id', + 'claim_number', + 'sap_document_number', + 'doc_date', + 'tds_amt', + 'raw_row', + 'storage_url', + ]) { + await queryInterface.removeColumn('form16_sap_responses', col).catch(() => {}); + } + + // 4) Ensure required columns exist for new contract + await queryInterface.addColumn('form16_sap_responses', 'trns_uniq_no', { + type: DataTypes.STRING(200), + allowNull: true, + }).catch(() => {}); + + await queryInterface.addColumn('form16_sap_responses', 'tds_trns_id', { + type: DataTypes.STRING(200), + allowNull: true, + }).catch(() => {}); + + await queryInterface.addColumn('form16_sap_responses', 'msg_typ', { + type: DataTypes.STRING(20), + allowNull: true, + }).catch(() => {}); + + await queryInterface.addColumn('form16_sap_responses', 'message', { + type: DataTypes.TEXT, + allowNull: true, + }).catch(() => {}); + + await queryInterface.addIndex('form16_sap_responses', ['tds_trns_id'], { + name: 'idx_form16_sap_responses_tds_trns_id', + }).catch(() => {}); + await queryInterface.addIndex('form16_sap_responses', ['trns_uniq_no'], { + name: 'idx_form16_sap_responses_trns_uniq_no', + }).catch(() => {}); + await queryInterface.addIndex('from16_sap_read_file', ['file_name'], { + name: 'idx_from16_sap_read_file_name', + }).catch(() => {}); + }, + + down: async (queryInterface: QueryInterface) => { + // Recreate old columns in form16_sap_responses + await queryInterface.addColumn('form16_sap_responses', 'type', { type: DataTypes.STRING(10), allowNull: false, defaultValue: 'credit' }).catch(() => {}); + await queryInterface.addColumn('form16_sap_responses', 'file_name', { type: DataTypes.STRING(255), allowNull: true }).catch(() => {}); + await queryInterface.addColumn('form16_sap_responses', 'credit_note_id', { type: DataTypes.INTEGER, allowNull: true }).catch(() => {}); + await queryInterface.addColumn('form16_sap_responses', 'debit_note_id', { type: DataTypes.INTEGER, allowNull: true }).catch(() => {}); + await queryInterface.addColumn('form16_sap_responses', 'claim_number', { type: DataTypes.STRING(100), allowNull: true }).catch(() => {}); + await queryInterface.addColumn('form16_sap_responses', 'sap_document_number', { type: DataTypes.STRING(100), allowNull: true }).catch(() => {}); + await queryInterface.addColumn('form16_sap_responses', 'doc_date', { type: DataTypes.STRING(20), allowNull: true }).catch(() => {}); + await queryInterface.addColumn('form16_sap_responses', 'tds_amt', { type: DataTypes.STRING(50), allowNull: true }).catch(() => {}); + await queryInterface.addColumn('form16_sap_responses', 'raw_row', { type: DataTypes.JSONB, allowNull: true }).catch(() => {}); + await queryInterface.addColumn('form16_sap_responses', 'storage_url', { type: DataTypes.STRING(500), allowNull: true }).catch(() => {}); + + await queryInterface.removeColumn('form16_sap_responses', 'doc_no').catch(() => {}); + await queryInterface.dropTable('from16_sap_read_file').catch(() => {}); + }, +}; diff --git a/src/migrations/20260324110001-add-pan-number-to-26as.ts b/src/migrations/20260324110001-add-pan-number-to-26as.ts new file mode 100644 index 0000000..09e2c89 --- /dev/null +++ b/src/migrations/20260324110001-add-pan-number-to-26as.ts @@ -0,0 +1,20 @@ +import type { QueryInterface } from 'sequelize'; +import { DataTypes } from 'sequelize'; + +module.exports = { + up: async (queryInterface: QueryInterface) => { + await queryInterface.addColumn('tds_26as_entries', 'pan_number', { + type: DataTypes.STRING(20), + allowNull: true, + comment: 'PAN from 26AS header (assessee PAN)', + }).catch(() => {}); + await queryInterface.addIndex('tds_26as_entries', ['pan_number'], { + name: 'idx_tds_26as_pan', + }).catch(() => {}); + }, + + down: async (queryInterface: QueryInterface) => { + await queryInterface.removeIndex('tds_26as_entries', 'idx_tds_26as_pan').catch(() => {}); + await queryInterface.removeColumn('tds_26as_entries', 'pan_number').catch(() => {}); + }, +}; diff --git a/src/models/Form16SapResponse.ts b/src/models/Form16SapResponse.ts index 386037c..4ae706e 100644 --- a/src/models/Form16SapResponse.ts +++ b/src/models/Form16SapResponse.ts @@ -1,23 +1,13 @@ import { DataTypes, Model, Optional } from 'sequelize'; import { sequelize } from '@config/database'; -import { Form16CreditNote } from './Form16CreditNote'; export interface Form16SapResponseAttributes { id: number; - type: 'credit'; - fileName: string; - creditNoteId?: number | null; - // Well-known SAP CSV columns stored as individual fields - trnsUniqNo?: string | null; // TRNS_UNIQ_NO – our unique ID echoed back by SAP - tdsTransId?: string | null; // TDS_TRNS_ID – credit note number echoed back (primary match key) - claimNumber?: string | null; // CLAIM_NUMBER (alias / fallback) - sapDocumentNumber?: string | null;// DOC_NO – SAP-generated document number - msgTyp?: string | null; // MSG_TYP - message?: string | null; // MESSAGE - docDate?: string | null; // DOC_DATE - tdsAmt?: string | null; // TDS_AMT - rawRow?: Record | null; // any extra / unknown columns from the CSV - storageUrl?: string | null; + trnsUniqNo: string | null; + tdsTransId: string | null; + docNo: string | null; + msgTyp: string | null; + message: string | null; createdAt: Date; updatedAt: Date; } @@ -26,17 +16,11 @@ interface Form16SapResponseCreationAttributes extends Optional< Form16SapResponseAttributes, | 'id' - | 'creditNoteId' | 'trnsUniqNo' | 'tdsTransId' - | 'claimNumber' - | 'sapDocumentNumber' + | 'docNo' | 'msgTyp' | 'message' - | 'docDate' - | 'tdsAmt' - | 'rawRow' - | 'storageUrl' | 'createdAt' | 'updatedAt' > {} @@ -46,41 +30,23 @@ class Form16SapResponse implements Form16SapResponseAttributes { public id!: number; - public type!: 'credit'; - public fileName!: string; - public creditNoteId?: number | null; - public trnsUniqNo?: string | null; - public tdsTransId?: string | null; - public claimNumber?: string | null; - public sapDocumentNumber?: string | null; - public msgTyp?: string | null; - public message?: string | null; - public docDate?: string | null; - public tdsAmt?: string | null; - public rawRow?: Record | null; - public storageUrl?: string | null; + public trnsUniqNo!: string | null; + public tdsTransId!: string | null; + public docNo!: string | null; + public msgTyp!: string | null; + public message!: string | null; public createdAt!: Date; public updatedAt!: Date; - - public creditNote?: Form16CreditNote; } Form16SapResponse.init( { id: { type: DataTypes.INTEGER, autoIncrement: true, primaryKey: true }, - type: { type: DataTypes.STRING(10), allowNull: false }, - fileName: { type: DataTypes.STRING(255), allowNull: false, unique: true, field: 'file_name' }, - creditNoteId: { type: DataTypes.INTEGER, allowNull: true, field: 'credit_note_id' }, trnsUniqNo: { type: DataTypes.STRING(200), allowNull: true, field: 'trns_uniq_no' }, tdsTransId: { type: DataTypes.STRING(200), allowNull: true, field: 'tds_trns_id' }, - claimNumber: { type: DataTypes.STRING(200), allowNull: true, field: 'claim_number' }, - sapDocumentNumber:{ type: DataTypes.STRING(100), allowNull: true, field: 'sap_document_number' }, + docNo: { type: DataTypes.STRING(200), allowNull: true, field: 'doc_no' }, msgTyp: { type: DataTypes.STRING(20), allowNull: true, field: 'msg_typ' }, message: { type: DataTypes.TEXT, allowNull: true }, - docDate: { type: DataTypes.STRING(20), allowNull: true, field: 'doc_date' }, - tdsAmt: { type: DataTypes.STRING(50), allowNull: true, field: 'tds_amt' }, - rawRow: { type: DataTypes.JSONB, allowNull: true, field: 'raw_row' }, - storageUrl: { type: DataTypes.STRING(500), allowNull: true, field: 'storage_url' }, createdAt: { type: DataTypes.DATE, allowNull: false, field: 'created_at' }, updatedAt: { type: DataTypes.DATE, allowNull: false, field: 'updated_at' }, }, @@ -94,10 +60,4 @@ Form16SapResponse.init( } ); -Form16SapResponse.belongsTo(Form16CreditNote, { - as: 'creditNote', - foreignKey: 'creditNoteId', - targetKey: 'id', -}); - export { Form16SapResponse }; diff --git a/src/models/From16SapReadFile.ts b/src/models/From16SapReadFile.ts new file mode 100644 index 0000000..e533917 --- /dev/null +++ b/src/models/From16SapReadFile.ts @@ -0,0 +1,50 @@ +import { DataTypes, Model, Optional } from 'sequelize'; +import { sequelize } from '@config/database'; + +export interface From16SapReadFileAttributes { + id: number; + fileName: string; + totalRecords: number; + totalCreditNotes: number; + totalDebitNotes: number; + createdAt: Date; + updatedAt: Date; +} + +interface From16SapReadFileCreationAttributes + extends Optional {} + +class From16SapReadFile + extends Model + implements From16SapReadFileAttributes +{ + public id!: number; + public fileName!: string; + public totalRecords!: number; + public totalCreditNotes!: number; + public totalDebitNotes!: number; + public createdAt!: Date; + public updatedAt!: Date; +} + +From16SapReadFile.init( + { + id: { type: DataTypes.INTEGER, autoIncrement: true, primaryKey: true }, + fileName: { type: DataTypes.STRING(255), allowNull: false, unique: true, field: 'file_name' }, + totalRecords: { type: DataTypes.INTEGER, allowNull: false, defaultValue: 0, field: 'total_records' }, + totalCreditNotes: { type: DataTypes.INTEGER, allowNull: false, defaultValue: 0, field: 'total_credit_notes' }, + totalDebitNotes: { type: DataTypes.INTEGER, allowNull: false, defaultValue: 0, field: 'total_debit_notes' }, + createdAt: { type: DataTypes.DATE, allowNull: false, field: 'created_at' }, + updatedAt: { type: DataTypes.DATE, allowNull: false, field: 'updated_at' }, + }, + { + sequelize, + tableName: 'from16_sap_read_file', + timestamps: true, + underscored: true, + createdAt: 'created_at', + updatedAt: 'updated_at', + } +); + +export { From16SapReadFile }; diff --git a/src/models/Tds26asEntry.ts b/src/models/Tds26asEntry.ts index 8f43c6c..6636c05 100644 --- a/src/models/Tds26asEntry.ts +++ b/src/models/Tds26asEntry.ts @@ -3,6 +3,7 @@ import { sequelize } from '@config/database'; export interface Tds26asEntryAttributes { id: number; + panNumber?: string; tanNumber: string; deductorName?: string; quarter: string; @@ -26,6 +27,7 @@ interface Tds26asEntryCreationAttributes extends Optional< Tds26asEntryAttributes, | 'id' + | 'panNumber' | 'deductorName' | 'assessmentYear' | 'sectionCode' @@ -46,6 +48,7 @@ class Tds26asEntry implements Tds26asEntryAttributes { public id!: number; + public panNumber?: string; public tanNumber!: string; public deductorName?: string; public quarter!: string; @@ -72,6 +75,11 @@ Tds26asEntry.init( autoIncrement: true, primaryKey: true, }, + panNumber: { + type: DataTypes.STRING(20), + allowNull: true, + field: 'pan_number', + }, tanNumber: { type: DataTypes.STRING(20), allowNull: false, diff --git a/src/models/index.ts b/src/models/index.ts index 51d21de..d08046e 100644 --- a/src/models/index.ts +++ b/src/models/index.ts @@ -40,6 +40,7 @@ import { Form16QuarterStatus } from './Form16QuarterStatus'; import { Form16LedgerEntry } from './Form16LedgerEntry'; import { Form16SapResponse } from './Form16SapResponse'; import { Form16DebitNoteSapResponse } from './Form16DebitNoteSapResponse'; +import { From16SapReadFile } from './From16SapReadFile'; // Define associations const defineAssociations = () => { @@ -226,7 +227,8 @@ export { Form16QuarterStatus, Form16LedgerEntry, Form16SapResponse, - Form16DebitNoteSapResponse + Form16DebitNoteSapResponse, + From16SapReadFile }; // Export default sequelize instance diff --git a/src/routes/form16.routes.ts b/src/routes/form16.routes.ts index d87c690..4f4a374 100644 --- a/src/routes/form16.routes.ts +++ b/src/routes/form16.routes.ts @@ -87,11 +87,22 @@ router.get( requireForm16SubmissionAccess, asyncHandler(form16Controller.viewDebitNoteSapResponse.bind(form16Controller)) ); +router.get( + '/debit-notes/:id/sap-response/csv', + requireForm16ReOnly, + requireForm16SubmissionAccess, + asyncHandler(form16Controller.downloadDebitNoteSapResponseCsv.bind(form16Controller)) +); router.get( '/credit-notes/:id/sap-response', requireForm16SubmissionAccess, asyncHandler(form16Controller.viewCreditNoteSapResponse.bind(form16Controller)) ); +router.get( + '/credit-notes/:id/sap-response/csv', + requireForm16SubmissionAccess, + asyncHandler(form16Controller.downloadCreditNoteSapResponseCsv.bind(form16Controller)) +); router.get( '/credit-notes/:id', requireForm16SubmissionAccess, diff --git a/src/scripts/migrate.ts b/src/scripts/migrate.ts index 7fba546..4a02244 100644 --- a/src/scripts/migrate.ts +++ b/src/scripts/migrate.ts @@ -70,6 +70,8 @@ import * as m62 from '../migrations/20260317100001-create-form16-sap-responses'; import * as m63 from '../migrations/20260317120001-add-form16-trns-uniq-no'; import * as m64 from '../migrations/20260318100001-create-form16-debit-note-sap-responses'; import * as m65 from '../migrations/20260318200001-add-sap-response-csv-fields'; +import * as m66 from '../migrations/20260324090001-refactor-form16-sap-response-and-add-read-log'; +import * as m67 from '../migrations/20260324110001-add-pan-number-to-26as'; interface Migration { name: string; @@ -147,6 +149,8 @@ const migrations: Migration[] = [ { name: '20260317120001-add-form16-trns-uniq-no', module: m63 }, { name: '20260318100001-create-form16-debit-note-sap-responses', module: m64 }, { name: '20260318200001-add-sap-response-csv-fields', module: m65 }, + { name: '20260324090001-refactor-form16-sap-response-and-add-read-log', module: m66 }, + { name: '20260324110001-add-pan-number-to-26as', module: m67 }, ]; diff --git a/src/services/configSeed.service.ts b/src/services/configSeed.service.ts index 36f9ddf..7a4dd50 100644 --- a/src/services/configSeed.service.ts +++ b/src/services/configSeed.service.ts @@ -585,7 +585,7 @@ export async function seedDefaultConfigurations(): Promise { gen_random_uuid(), 'FORM16_ADMIN_CONFIG', 'SYSTEM_SETTINGS', - '{"submissionViewerEmails":[],"twentySixAsViewerEmails":[],"reminderEnabled":true,"reminderDays":7,"notification26AsDataAdded":{"enabled":true,"template":"26AS data has been added. Please review."},"notificationForm16SuccessCreditNote":{"enabled":true,"template":"Form 16 submitted successfully. Credit note: [CreditNoteRef]."},"notificationForm16Unsuccessful":{"enabled":true,"template":"Form 16 submission was unsuccessful. Issue: [Issue]."},"alertSubmitForm16Enabled":true,"alertSubmitForm16FrequencyDays":0,"alertSubmitForm16FrequencyHours":24,"alertSubmitForm16Template":"Please submit your Form 16 at your earliest. [Name], due date: [DueDate].","reminderNotificationEnabled":true,"reminderFrequencyDays":0,"reminderFrequencyHours":12,"reminderNotificationTemplate":"Reminder: Form 16 submission is pending. [Name], [Request ID]. Please review.","debitNoteNotification":{"enabled":true,"template":"Debit note issued: [DebitNoteRef]. Please review."}}', + '{"submissionViewerEmails":[],"twentySixAsViewerEmails":[],"reminderEnabled":true,"reminderDays":7,"notification26AsDataAdded":{"enabled":true,"template":"26AS data has been added. Please review."},"notificationForm16SuccessCreditNote":{"enabled":true,"template":"Form 16 submitted successfully. Credit note: [CreditNoteRef]."},"notificationForm16Unsuccessful":{"enabled":true,"template":"Form 16 submission was unsuccessful. Issue: [Issue]."},"alertSubmitForm16Enabled":true,"alertSubmitForm16FrequencyDays":0,"alertSubmitForm16FrequencyHours":24,"alertSubmitForm16Template":"Dear [Name], please submit Form 16A for the pending period. Due: [DueDate].","reminderNotificationEnabled":true,"reminderFrequencyDays":0,"reminderFrequencyHours":12,"reminderNotificationTemplate":"Reminder: Dear [Name], your Form 16A submission is pending for request [Request ID]. Please complete it.","debitNoteNotification":{"enabled":true,"template":"Debit note issued: [DebitNoteRef]. Please review."}}', 'JSON', 'Form 16 Admin Config', 'Form 16 visibility (submission data viewers, 26AS viewers), reminders and notification settings', diff --git a/src/services/form16.service.ts b/src/services/form16.service.ts index 080f0c7..26b22a3 100644 --- a/src/services/form16.service.ts +++ b/src/services/form16.service.ts @@ -19,7 +19,6 @@ import { Form16QuarterStatus, Form16LedgerEntry, Form16SapResponse, - Form16DebitNoteSapResponse, } from '../models'; import { Tds26asEntry } from '../models/Tds26asEntry'; import { Form1626asUploadLog } from '../models/Form1626asUploadLog'; @@ -65,6 +64,23 @@ export async function getDealerCodeForUser(userId: string): Promise { - const normalized = (tanNumber || '').trim().replace(/\s+/g, ' '); + const normalizedTan = normalizeTanNumber(tanNumber); const fy = normalizeFinancialYear(financialYear) || financialYear; const q = normalizeQuarter(quarter) || quarter; const [row] = await sequelize.query<{ sum: string }>( `WITH latest_upload AS ( SELECT MAX(upload_log_id) AS mid FROM tds_26as_entries - WHERE LOWER(REPLACE(TRIM(tan_number), ' ', '')) = LOWER(REPLACE(TRIM(:tan), ' ', '')) + WHERE UPPER(REGEXP_REPLACE(TRIM(COALESCE(tan_number, '')), '[^A-Z0-9]', '', 'g')) = :tan AND financial_year = :fy AND quarter = :qtr - AND section_code = :section - AND (status_oltas = 'F' OR status_oltas = 'O') + AND UPPER(TRIM(COALESCE(section_code, ''))) = :section + AND UPPER(TRIM(COALESCE(status_oltas, ''))) IN ('F', 'O') AND upload_log_id IS NOT NULL ) SELECT COALESCE(SUM(e.tax_deducted), 0)::text AS sum FROM tds_26as_entries e - WHERE LOWER(REPLACE(TRIM(e.tan_number), ' ', '')) = LOWER(REPLACE(TRIM(:tan), ' ', '')) + WHERE UPPER(REGEXP_REPLACE(TRIM(COALESCE(e.tan_number, '')), '[^A-Z0-9]', '', 'g')) = :tan AND e.financial_year = :fy AND e.quarter = :qtr - AND e.section_code = :section - AND (e.status_oltas = 'F' OR e.status_oltas = 'O') + AND UPPER(TRIM(COALESCE(e.section_code, ''))) = :section + AND UPPER(TRIM(COALESCE(e.status_oltas, ''))) IN ('F', 'O') AND ( e.upload_log_id = (SELECT mid FROM latest_upload) OR (SELECT mid FROM latest_upload) IS NULL )`, - { replacements: { tan: normalized, fy, qtr: q, section: SECTION_26AS_194Q }, type: QueryTypes.SELECT } + { replacements: { tan: normalizedTan, fy, qtr: q, section: SECTION_26AS_194Q }, type: QueryTypes.SELECT } ); return parseFloat(row?.sum ?? '0') || 0; } +async function getLatest26asRowsForQuarter( + tanNumber: string, + financialYear: string, + quarter: string +): Promise { + const normalizedTan = normalizeTanNumber(tanNumber); + const fy = normalizeFinancialYear(financialYear) || financialYear; + const q = normalizeQuarter(quarter) || quarter; + + const rows = await sequelize.query<{ + pan_number: string | null; + amount_paid: string | null; + tax_deducted: string; + total_tds_deposited: string | null; + transaction_date: string | null; + date_of_booking: string | null; + }>( + `WITH latest_upload AS ( + SELECT MAX(upload_log_id) AS mid FROM tds_26as_entries + WHERE UPPER(REGEXP_REPLACE(TRIM(COALESCE(tan_number, '')), '[^A-Z0-9]', '', 'g')) = :tan + AND financial_year = :fy AND quarter = :qtr + AND UPPER(TRIM(COALESCE(section_code, ''))) = :section + AND UPPER(TRIM(COALESCE(status_oltas, ''))) IN ('F', 'O') + AND upload_log_id IS NOT NULL + ) + SELECT + e.pan_number, + e.amount_paid, + e.tax_deducted, + e.total_tds_deposited, + e.transaction_date, + e.date_of_booking + FROM tds_26as_entries e + WHERE UPPER(REGEXP_REPLACE(TRIM(COALESCE(e.tan_number, '')), '[^A-Z0-9]', '', 'g')) = :tan + AND e.financial_year = :fy + AND e.quarter = :qtr + AND UPPER(TRIM(COALESCE(e.section_code, ''))) = :section + AND UPPER(TRIM(COALESCE(e.status_oltas, ''))) IN ('F', 'O') + AND ( + e.upload_log_id = (SELECT mid FROM latest_upload) + OR (SELECT mid FROM latest_upload) IS NULL + )`, + { replacements: { tan: normalizedTan, fy, qtr: q, section: SECTION_26AS_194Q }, type: QueryTypes.SELECT } + ); + + return rows.map((r) => ({ + panNumber: r.pan_number ? String(r.pan_number).trim().toUpperCase() : null, + amountPaid: r.amount_paid == null ? null : parseFloat(r.amount_paid), + taxDeducted: parseFloat(r.tax_deducted || '0') || 0, + totalTdsDeposited: r.total_tds_deposited == null ? null : parseFloat(r.total_tds_deposited), + transactionDate: r.transaction_date || null, + dateOfBooking: r.date_of_booking || null, + })); +} + +function normalizeDateOnly(value: unknown): string | null { + if (!value) return null; + const raw = String(value).trim(); + if (!raw) return null; + const d = new Date(raw); + if (!Number.isNaN(d.getTime())) return d.toISOString().slice(0, 10); + const m = raw.match(/^(\d{1,2})[-\/](\d{1,2})[-\/](\d{2,4})$/); + if (!m) return null; + const dd = m[1].padStart(2, '0'); + const mm = m[2].padStart(2, '0'); + const yyyy = m[3].length === 2 ? `20${m[3]}` : m[3]; + return `${yyyy}-${mm}-${dd}`; +} + +function toNumberOrNull(value: unknown): number | null { + if (value == null || value === '') return null; + const n = typeof value === 'number' ? value : parseFloat(String(value).replace(/,/g, '')); + return Number.isFinite(n) ? n : null; +} + /** Get latest 26AS quarter snapshot for (tan, fy, quarter). */ export async function getLatest26asSnapshot( tanNumber: string, @@ -277,16 +368,19 @@ export async function listCreditNotesForDealer(userId: string, filters?: { finan let sapSet = new Set(); if (hasTrnsUniqNoColumn && noteIds.length) { try { + const creditNotes = await Form16CreditNote.findAll({ + where: { id: { [Op.in]: noteIds } }, + attributes: ['id', 'creditNoteNumber'], + raw: true, + }) as any[]; + const creditNumbers = creditNotes.map((n) => n.creditNoteNumber).filter(Boolean); const sapRows = await (Form16SapResponse as any).findAll({ - where: { - type: 'credit', - creditNoteId: { [Op.in]: noteIds }, - [Op.or]: [{ storageUrl: { [Op.ne]: null } }, { sapDocumentNumber: { [Op.ne]: null } }], - }, - attributes: ['creditNoteId'], + where: { tdsTransId: { [Op.in]: creditNumbers } }, + attributes: ['tdsTransId'], raw: true, }); - sapSet = new Set((sapRows as any[]).map((r) => r.creditNoteId)); + const available = new Set((sapRows as any[]).map((r) => String(r.tdsTransId))); + sapSet = new Set(creditNotes.filter((n) => available.has(String(n.creditNoteNumber))).map((n) => Number(n.id))); } catch (e: any) { logger.warn('[Form16] SAP response lookup failed (will treat as unavailable):', e?.message || e); sapSet = new Set(); @@ -530,16 +624,16 @@ export function formatForm16DebitNoteNumber( async function run26asMatchAndCreditNote(submission: Form16aSubmission): Promise<{ validationStatus: string; creditNoteNumber?: string | null; validationNotes?: string }> { const sub = submission as any; const tanNumberRaw = (sub.tanNumber || '').toString().trim(); - const tanNumber = tanNumberRaw.replace(/\s+/g, ' '); + const tanNumber = normalizeTanNumber(tanNumberRaw); const tdsAmount = parseFloat(sub.tdsAmount) || 0; - if (!tanNumber || tdsAmount <= 0) { + if (!tanNumber || tanNumber.length < 10 || tdsAmount <= 0) { logger.warn( `[Form16] 26AS MATCH RESULT: RESUBMISSION_NEEDED – OCR incomplete. Form 16A: TAN=${tanNumber || '(missing)'}, TDS amount=${tdsAmount}, FY=${(sub.financialYear || '').toString().trim() || '(missing)'}, Quarter=${(sub.quarter || '').toString().trim() || '(missing)'}. No 26AS check performed.` ); await submission.update({ validationStatus: 'resubmission_needed', - validationNotes: 'OCR data incomplete (TAN or TDS amount missing). Please resubmit Form 16 or contact RE for manual approval.', + validationNotes: 'OCR data incomplete/invalid (TAN or TDS amount missing). Please resubmit Form 16 or contact RE for manual approval.', }); return { validationStatus: 'resubmission_needed' }; } @@ -560,8 +654,36 @@ async function run26asMatchAndCreditNote(submission: Form16aSubmission): Promise return { validationStatus: 'resubmission_needed' }; } - // Official quarter total from 26AS (Section 194Q, Booking F/O only) - const aggregated26as = await getLatest26asAggregatedForQuarter(tanNumber, financialYear, quarter); + const extracted = (sub.ocrExtractedData || {}) as Record; + const submittedPan = + (extracted.panOfDeductee as string) || + (extracted.deducteePan as string) || + (extracted.panNumber as string) || + null; + const normalizedSubmittedPan = submittedPan ? String(submittedPan).trim().toUpperCase() : null; + const submittedAmountPaid = toNumberOrNull(extracted.totalAmountPaid ?? sub.totalAmount); + const submittedTaxDeducted = toNumberOrNull(extracted.totalTaxDeducted ?? sub.tdsAmount); + const submittedTdsDeposited = toNumberOrNull(extracted.totalTdsDeposited ?? sub.tdsAmount); + const submittedTransactionDate = normalizeDateOnly(extracted.transactionDate); + const submittedBookingDate = normalizeDateOnly(extracted.dateOfBooking); + + // Latest 26AS upload rows for the same TAN + FY + Quarter. + const latestRows = await getLatest26asRowsForQuarter(tanNumber, financialYear, quarter); + const aggregated26as = latestRows.reduce((sum, r) => sum + (r.taxDeducted || 0), 0); + + if (normalizedSubmittedPan) { + const hasPanMatch = latestRows.some((r) => r.panNumber && r.panNumber === normalizedSubmittedPan); + if (!hasPanMatch) { + logger.warn( + `[Form16] 26AS MATCH RESULT: FAILED – PAN mismatch. TAN=${tanNumber}, PAN(Form16A)=${normalizedSubmittedPan}, FY=${financialYear}, Quarter=${quarter}.` + ); + await submission.update({ + validationStatus: 'failed', + validationNotes: `PAN mismatch with latest 26AS for TAN no - ${tanNumber}. Form 16A PAN: ${normalizedSubmittedPan}.`, + }); + return { validationStatus: 'failed', validationNotes: 'PAN mismatch with latest 26AS.' }; + } + } if (aggregated26as <= 0) { logger.warn( @@ -574,8 +696,82 @@ async function run26asMatchAndCreditNote(submission: Form16aSubmission): Promise return { validationStatus: 'failed', validationNotes: `No 26AS record found for this TAN no - ${tanNumber}, financial year and quarter.` }; } - const amountTolerance = 1; // allow 1 rupee rounding - if (Math.abs(tdsAmount - aggregated26as) > amountTolerance) { + // Validate against quarter-level aggregate from latest upload. + // 26AS has many transaction lines; we compare submitted totals against aggregated totals. + const aggregatedAmountPaid = latestRows.reduce((sum, r) => sum + (r.amountPaid || 0), 0); + const aggregatedTaxDeducted = latestRows.reduce((sum, r) => sum + (r.taxDeducted || 0), 0); + const aggregatedTdsDeposited = latestRows.reduce((sum, r) => sum + (r.totalTdsDeposited ?? r.taxDeducted ?? 0), 0); + + if ( + submittedAmountPaid != null && + Math.abs(submittedAmountPaid - aggregatedAmountPaid) > AMOUNT_MATCH_TOLERANCE + ) { + logger.warn( + `[Form16] 26AS MATCH RESULT: FAILED – Amount paid mismatch. TAN=${tanNumber}, PAN=${submittedPan || '(not available)'}, FY=${financialYear}, Quarter=${quarter}. Form16A amountPaid=${submittedAmountPaid}, 26AS latest aggregate amountPaid=${aggregatedAmountPaid}.` + ); + await submission.update({ + validationStatus: 'failed', + validationNotes: + `Amount paid mismatch with latest 26AS for TAN no - ${tanNumber}. Form 16A amount paid: ${submittedAmountPaid}. Latest 26AS aggregated amount paid for this quarter: ${aggregatedAmountPaid}.`, + }); + return { validationStatus: 'failed', validationNotes: 'Amount paid mismatch with latest 26AS.' }; + } + + if ( + submittedTaxDeducted != null && + Math.abs(submittedTaxDeducted - aggregatedTaxDeducted) > AMOUNT_MATCH_TOLERANCE + ) { + logger.warn( + `[Form16] 26AS MATCH RESULT: FAILED – Tax deducted mismatch. TAN=${tanNumber}, PAN=${submittedPan || '(not available)'}, FY=${financialYear}, Quarter=${quarter}. Form16A taxDeducted=${submittedTaxDeducted}, 26AS latest aggregate taxDeducted=${aggregatedTaxDeducted}.` + ); + await submission.update({ + validationStatus: 'failed', + validationNotes: + `Tax deducted mismatch with latest 26AS for TAN no - ${tanNumber}. Form 16A tax deducted: ${submittedTaxDeducted}. Latest 26AS aggregated tax deducted for this quarter: ${aggregatedTaxDeducted}.`, + }); + return { validationStatus: 'failed', validationNotes: 'Tax deducted mismatch with latest 26AS.' }; + } + + if ( + submittedTdsDeposited != null && + Math.abs(submittedTdsDeposited - aggregatedTdsDeposited) > AMOUNT_MATCH_TOLERANCE + ) { + logger.warn( + `[Form16] 26AS MATCH RESULT: FAILED – TDS deposited mismatch. TAN=${tanNumber}, PAN=${submittedPan || '(not available)'}, FY=${financialYear}, Quarter=${quarter}. Form16A tdsDeposited=${submittedTdsDeposited}, 26AS latest aggregate tdsDeposited=${aggregatedTdsDeposited}.` + ); + await submission.update({ + validationStatus: 'failed', + validationNotes: + `TDS deposited mismatch with latest 26AS for TAN no - ${tanNumber}. Form 16A TDS deposited: ${submittedTdsDeposited}. Latest 26AS aggregated TDS deposited for this quarter: ${aggregatedTdsDeposited}.`, + }); + return { validationStatus: 'failed', validationNotes: 'TDS deposited mismatch with latest 26AS.' }; + } + + // Optional date checks: if OCR extracted transaction/booking date, at least one latest-upload row should contain that date. + if (submittedTransactionDate) { + const hasTxDate = latestRows.some((r) => normalizeDateOnly(r.transactionDate) === submittedTransactionDate); + if (!hasTxDate) { + await submission.update({ + validationStatus: 'failed', + validationNotes: + `Transaction date mismatch with latest 26AS for TAN no - ${tanNumber}. No latest 26AS transaction found with date ${submittedTransactionDate}.`, + }); + return { validationStatus: 'failed', validationNotes: 'Transaction date mismatch with latest 26AS.' }; + } + } + if (submittedBookingDate) { + const hasBookingDate = latestRows.some((r) => normalizeDateOnly(r.dateOfBooking) === submittedBookingDate); + if (!hasBookingDate) { + await submission.update({ + validationStatus: 'failed', + validationNotes: + `Booking date mismatch with latest 26AS for TAN no - ${tanNumber}. No latest 26AS record found with booking date ${submittedBookingDate}.`, + }); + return { validationStatus: 'failed', validationNotes: 'Booking date mismatch with latest 26AS.' }; + } + } + + if (Math.abs(tdsAmount - aggregated26as) > AMOUNT_MATCH_TOLERANCE) { logger.warn( `[Form16] 26AS MATCH RESULT: FAILED – Amount mismatch. Form 16A TDS amount=${tdsAmount} | 26AS aggregated amount (quarter)=${aggregated26as} | TAN=${tanNumber}, FY=${financialYear}, Quarter=${quarter}. Difference=${Math.abs(tdsAmount - aggregated26as).toFixed(2)}.` ); @@ -594,7 +790,7 @@ async function run26asMatchAndCreditNote(submission: Form16aSubmission): Promise if (qStatus && qStatus.status === 'SETTLED' && qStatus.lastCreditNoteId) { const lastCn = await Form16CreditNote.findByPk(qStatus.lastCreditNoteId, { attributes: ['amount'] }); const lastAmount = lastCn ? parseFloat((lastCn as any).amount as string) : 0; - if (Math.abs(lastAmount - tdsAmount) <= amountTolerance) { + if (Math.abs(lastAmount - tdsAmount) <= AMOUNT_MATCH_TOLERANCE) { logger.warn( `[Form16] 26AS MATCH RESULT: DUPLICATE – Quarter already settled. Form 16A TDS amount=${tdsAmount} | Existing credit note amount=${lastAmount} | TAN=${tanNumber}, FY=${financialYear}, Quarter=${quarter}. No new credit note issued.` ); @@ -640,7 +836,7 @@ async function run26asMatchAndCreditNote(submission: Form16aSubmission): Promise validationNotes: null, }); - // Push Form 16 credit note incoming CSV to WFM INCOMING/WFM_MAIN/FORM16_CRDT (SAP credit note generation – exact fields only) + // Push Form 16 credit note incoming CSV to WFM INCOMING/WFM_MAIN/FORM16 try { const trnsUniqNo = `F16-CN-${submission.id}-${creditNote.id}-${Date.now()}`; await creditNote.update({ trnsUniqNo }); @@ -651,17 +847,18 @@ async function run26asMatchAndCreditNote(submission: Form16aSubmission): Promise TRNS_UNIQ_NO: trnsUniqNo, TDS_TRNS_ID: cnNumber, DEALER_CODE: padDealerCode(dealerCode), - TDS_TRNS_DOC_TYP: 'ZTDS', + TDS_TRNS_DOC_TYPE: 'ZTDS', DLR_TAN_NO: tanNumber, - 'FIN_YEAR & QUARTER': finYearAndQuarter, + 'FIN_YEAR&QUARTER': finYearAndQuarter, DOC_DATE: docDate, TDS_AMT: Number(tdsAmount).toFixed(2), + TDS_CERTIFICATE_NO: certificateNumber, }; const fileName = `${cnNumber}.csv`; await wfmFileService.generateForm16IncomingCSV([csvRow], fileName, 'credit'); - logger.info(`[Form16] Credit note CSV pushed to WFM FORM16_CRDT: ${cnNumber}`); + logger.info(`[Form16] Credit note CSV pushed to WFM FORM16: ${cnNumber}`); } catch (csvErr: any) { - logger.error('[Form16] Failed to push credit note CSV to WFM FORM16_CRDT:', csvErr?.message || csvErr); + logger.error('[Form16] Failed to push credit note CSV to WFM FORM16:', csvErr?.message || csvErr); // Do not fail the flow; credit note and ledger are already created } @@ -886,16 +1083,19 @@ export async function listAllCreditNotesForRe(filters?: { financialYear?: string let sapSet = new Set(); if (hasTrnsUniqNoColumn && noteIds.length) { try { + const creditNotes = await Form16CreditNote.findAll({ + where: { id: { [Op.in]: noteIds } }, + attributes: ['id', 'creditNoteNumber'], + raw: true, + }) as any[]; + const creditNumbers = creditNotes.map((n) => n.creditNoteNumber).filter(Boolean); const sapRows = await (Form16SapResponse as any).findAll({ - where: { - type: 'credit', - creditNoteId: { [Op.in]: noteIds }, - [Op.or]: [{ storageUrl: { [Op.ne]: null } }, { sapDocumentNumber: { [Op.ne]: null } }], - }, - attributes: ['creditNoteId'], + where: { tdsTransId: { [Op.in]: creditNumbers } }, + attributes: ['tdsTransId'], raw: true, }); - sapSet = new Set((sapRows as any[]).map((r) => r.creditNoteId)); + const available = new Set((sapRows as any[]).map((r) => String(r.tdsTransId))); + sapSet = new Set(creditNotes.filter((n) => available.has(String(n.creditNoteNumber))).map((n) => Number(n.id))); } catch (e: any) { logger.warn('[Form16] SAP response lookup failed (will treat as unavailable):', e?.message || e); sapSet = new Set(); @@ -1002,15 +1202,19 @@ export async function listAllDebitNotesForRe(filters?: { financialYear?: string; let sapSet = new Set(); if (noteIds.length) { try { - const sapRows = await (Form16DebitNoteSapResponse as any).findAll({ - where: { - debitNoteId: { [Op.in]: noteIds }, - [Op.or]: [{ storageUrl: { [Op.ne]: null } }, { sapDocumentNumber: { [Op.ne]: null } }], - }, - attributes: ['debitNoteId'], + const debitNotes = await Form16DebitNote.findAll({ + where: { id: { [Op.in]: noteIds } }, + attributes: ['id', 'debitNoteNumber'], + raw: true, + }) as any[]; + const debitNumbers = debitNotes.map((n) => n.debitNoteNumber).filter(Boolean); + const sapRows = await (Form16SapResponse as any).findAll({ + where: { tdsTransId: { [Op.in]: debitNumbers } }, + attributes: ['tdsTransId'], raw: true, }); - sapSet = new Set((sapRows as any[]).map((r) => r.debitNoteId ?? r.debit_note_id)); + const available = new Set((sapRows as any[]).map((r) => String(r.tdsTransId))); + sapSet = new Set(debitNotes.filter((n) => available.has(String(n.debitNoteNumber))).map((n) => Number(n.id))); } catch (e: any) { logger.warn('[Form16] Debit SAP response lookup failed (will treat as unavailable):', e?.message || e); sapSet = new Set(); @@ -1490,20 +1694,15 @@ export async function getCreditNoteById(creditNoteId: number) { } export async function getCreditNoteSapResponseUrl(creditNoteId: number): Promise { - const row = await (Form16SapResponse as any).findOne({ - where: { type: 'credit', creditNoteId, storageUrl: { [Op.ne]: null } }, - attributes: ['storageUrl', 'createdAt'], - order: [['createdAt', 'DESC']], - }); - const url = row?.storageUrl; - return url && String(url).trim() ? String(url) : null; + // API-backed CSV generation is used now; URL is deterministic when SAP response exists. + const row = await getCreditNoteSapResponse(creditNoteId); + return row ? `/api/v1/form16/credit-notes/${creditNoteId}/sap-response/csv` : null; } export interface Form16SapResponseView { fileName: string | null; trnsUniqNo: string | null; tdsTransId: string | null; - claimNumber: string | null; sapDocumentNumber: string | null; msgTyp: string | null; message: string | null; @@ -1519,61 +1718,48 @@ function mapSapResponseView(row: any): Form16SapResponseView { fileName: row?.fileName ?? null, trnsUniqNo: row?.trnsUniqNo ?? null, tdsTransId: row?.tdsTransId ?? null, - claimNumber: row?.claimNumber ?? null, - sapDocumentNumber: row?.sapDocumentNumber ?? null, + sapDocumentNumber: row?.docNo ?? null, msgTyp: row?.msgTyp ?? null, message: row?.message ?? null, - docDate: row?.docDate ?? null, - tdsAmt: row?.tdsAmt ?? null, - storageUrl: row?.storageUrl ?? null, + docDate: null, + tdsAmt: null, + storageUrl: null, createdAt: row?.createdAt ?? null, updatedAt: row?.updatedAt ?? null, }; } export async function getCreditNoteSapResponse(creditNoteId: number): Promise { + const cn = await Form16CreditNote.findByPk(creditNoteId, { attributes: ['id', 'creditNoteNumber'] }); + const creditNoteNumber = (cn as any)?.creditNoteNumber; + if (!creditNoteNumber) return null; const row = await (Form16SapResponse as any).findOne({ where: { - type: 'credit', - creditNoteId, - [Op.or]: [ - { storageUrl: { [Op.ne]: null } }, - { sapDocumentNumber: { [Op.ne]: null } }, - { trnsUniqNo: { [Op.ne]: null } }, - { tdsTransId: { [Op.ne]: null } }, - ], + tdsTransId: creditNoteNumber, }, - attributes: ['fileName', 'trnsUniqNo', 'tdsTransId', 'claimNumber', 'sapDocumentNumber', 'msgTyp', 'message', 'docDate', 'tdsAmt', 'storageUrl', 'createdAt', 'updatedAt'], + attributes: ['trnsUniqNo', 'tdsTransId', 'docNo', 'msgTyp', 'message', 'createdAt', 'updatedAt'], order: [['createdAt', 'DESC']], }); - return row ? mapSapResponseView(row) : null; + return row ? mapSapResponseView({ ...row.toJSON(), fileName: `${creditNoteNumber}.csv` }) : null; } export async function getDebitNoteSapResponseUrl(debitNoteId: number): Promise { - const row = await (Form16DebitNoteSapResponse as any).findOne({ - where: { debitNoteId, storageUrl: { [Op.ne]: null } }, - attributes: ['storageUrl', 'createdAt'], - order: [['createdAt', 'DESC']], - }); - const url = row?.storageUrl; - return url && String(url).trim() ? String(url) : null; + const row = await getDebitNoteSapResponse(debitNoteId); + return row ? `/api/v1/form16/debit-notes/${debitNoteId}/sap-response/csv` : null; } export async function getDebitNoteSapResponse(debitNoteId: number): Promise { - const row = await (Form16DebitNoteSapResponse as any).findOne({ + const dn = await Form16DebitNote.findByPk(debitNoteId, { attributes: ['id', 'debitNoteNumber'] }); + const debitNoteNumber = (dn as any)?.debitNoteNumber; + if (!debitNoteNumber) return null; + const row = await (Form16SapResponse as any).findOne({ where: { - debitNoteId, - [Op.or]: [ - { storageUrl: { [Op.ne]: null } }, - { sapDocumentNumber: { [Op.ne]: null } }, - { trnsUniqNo: { [Op.ne]: null } }, - { tdsTransId: { [Op.ne]: null } }, - ], + tdsTransId: debitNoteNumber, }, - attributes: ['fileName', 'trnsUniqNo', 'tdsTransId', 'claimNumber', 'sapDocumentNumber', 'msgTyp', 'message', 'docDate', 'tdsAmt', 'storageUrl', 'createdAt', 'updatedAt'], + attributes: ['trnsUniqNo', 'tdsTransId', 'docNo', 'msgTyp', 'message', 'createdAt', 'updatedAt'], order: [['createdAt', 'DESC']], }); - return row ? mapSapResponseView(row) : null; + return row ? mapSapResponseView({ ...row.toJSON(), fileName: `${debitNoteNumber}.csv` }) : null; } /** @@ -1954,6 +2140,7 @@ export async function list26asEntries(filters?: List26asFilters): Promise<{ } export async function create26asEntry(data: { + panNumber?: string; tanNumber: string; deductorName?: string; quarter: string; @@ -1970,6 +2157,7 @@ export async function create26asEntry(data: { remarks?: string; }) { const entry = await Tds26asEntry.create({ + panNumber: data.panNumber, tanNumber: data.tanNumber, deductorName: data.deductorName, quarter: data.quarter, @@ -1991,6 +2179,7 @@ export async function create26asEntry(data: { export async function update26asEntry( id: number, data: Partial<{ + panNumber: string; tanNumber: string; deductorName: string; quarter: string; @@ -2070,6 +2259,7 @@ function getCurrentFinancialYear(): string { function parse26asOfficialFormat(lines: string[]): { rows: any[]; errors: string[] } { const errors: string[] = []; const rows: any[] = []; + let currentPAN = ''; let currentTAN = ''; let currentDeductorName = ''; const transactionDateRe = /^\d{1,2}-\w{3}-\d{4}$/i; @@ -2084,6 +2274,14 @@ function parse26asOfficialFormat(lines: string[]): { rows: any[]; errors: string const c2 = cells[2]; const c3 = cells[3]; + // Header block with PAN row: + // File Creation Date ^ Permanent Account Number (PAN) ^ ... + // 25-10-2024 ^ AAACE3883A ^ ... + if (cells.length >= 2 && /^[A-Z]{5}[0-9]{4}[A-Z]$/i.test(c1 || '')) { + currentPAN = String(c1).trim().toUpperCase(); + continue; + } + // Deductor summary: first cell is numeric (Sr No), third looks like TAN (e.g. AGRA13250G), then empties then amounts const srNoNum = /^\d+$/.test(c0); const looksLikeTan = c2 && c2.length >= 8 && /^[A-Z0-9]+$/i.test(c2); @@ -2104,6 +2302,7 @@ function parse26asOfficialFormat(lines: string[]): { rows: any[]; errors: string const totalTds = parseDecimal(cells[9]); const { financialYear, quarter } = dateToFyAndQuarter(cells[3]); rows.push({ + panNumber: currentPAN || undefined, tanNumber: currentTAN, deductorName: currentDeductorName || undefined, quarter, @@ -2156,6 +2355,7 @@ function buildColumnMap(headerCells: string[]): Record { const norm = (s: string) => s.toLowerCase().replace(/[\s_-]+/g, ''); headerCells.forEach((cell, idx) => { const n = norm(cell); + if (n === 'pan' || n.includes('pannumber') || (n.includes('permanent') && n.includes('account'))) map['panNumber'] = idx; if (n.includes('tan') && !n.includes('amount')) map['tanNumber'] = idx; else if (n.includes('deductor') && (n.includes('name') || n.length < 20)) map['deductorName'] = idx; else if (n.includes('quarter') || n === 'q') map['quarter'] = idx; @@ -2221,6 +2421,7 @@ export function parse26asTxtFile(buffer: Buffer): { rows: any[]; errors: string[ const financialYear = get(cells, 'financialYear') || defaultFY; const taxDeductedNum = parseDecimal(get(cells, 'taxDeducted')) ?? 0; rows.push({ + panNumber: get(cells, 'panNumber') || undefined, tanNumber, deductorName: get(cells, 'deductorName') || undefined, quarter, @@ -2242,7 +2443,7 @@ export function parse26asTxtFile(buffer: Buffer): { rows: any[]; errors: string[ /** Allowed fields for 26AS create – exclude timestamps so Sequelize sets them. */ const TDS_26AS_CREATE_KEYS = [ - 'tanNumber', 'deductorName', 'quarter', 'financialYear', 'sectionCode', + 'panNumber', 'tanNumber', 'deductorName', 'quarter', 'financialYear', 'sectionCode', 'amountPaid', 'taxDeducted', 'totalTdsDeposited', 'natureOfPayment', 'transactionDate', 'dateOfBooking', 'assessmentYear', 'statusOltas', 'remarks', 'uploadLogId', ] as const; @@ -2254,7 +2455,8 @@ function build26asCreatePayload(row: Record, uploadLogId?: numb const v = row[k]; if (v !== undefined && v !== null) payload[k] = v; } - payload.tanNumber = (row.tanNumber != null ? String(row.tanNumber).trim() : '') || ''; + payload.tanNumber = normalizeTanNumber(row.tanNumber); + if (row.panNumber != null) payload.panNumber = String(row.panNumber).trim().toUpperCase(); const rawFy = (row.financialYear != null ? String(row.financialYear).trim() : '') || ''; const rawQ = (row.quarter != null ? String(row.quarter).trim() : '') || 'Q1'; payload.financialYear = normalizeFinancialYear(rawFy) || rawFy; @@ -2359,29 +2561,29 @@ export async function process26asUploadAggregation(uploadLogId: number): Promise await setQuarterStatusDebitIssued(tanNumber, fy, q, debit.id); debitsCreated++; - // Push Form 16 debit note CSV to WFM INCOMING/WFM_MAIN/FORM16_DEBT (same column set as credit note / SAP expectation) + // Push Form 16 debit note CSV to WFM INCOMING/WFM_MAIN/FORM16 try { const trnsUniqNo = `F16-DN-${creditNote.id}-${debit.id}-${Date.now()}`; await debit.update({ trnsUniqNo }); const docDate = now.toISOString().slice(0, 10).replace(/-/g, ''); const fyCompact = form16FyCompact(cnFy) || ''; - const finYearAndQuarter = fyCompact && cnQuarter ? `FY ${fyCompact}_${cnQuarter}` : ''; + const finYearAndQuarter = fyCompact && cnQuarter ? `FY_${fyCompact}_${cnQuarter}` : ''; const csvRow: Record = { TRNS_UNIQ_NO: trnsUniqNo, - TDS_TRNS_ID: creditNoteNumber, + TDS_TRNS_ID: debitNum, DEALER_CODE: padDealerCode(dealerCode), - TDS_TRNS_DOC_TYP: 'ZTDS', - 'Org.Document Number': debit.id, + TDS_TRNS_DOC_TYPE: 'ZTDS', DLR_TAN_NO: tanNumber, - 'FIN_YEAR & QUARTER': finYearAndQuarter, + 'FIN_YEAR&QUARTER': finYearAndQuarter, DOC_DATE: docDate, - TDS_AMT: Number(amount).toFixed(2), + TDS_AMT: `-${Math.abs(Number(amount)).toFixed(2)}`, + TDS_CERTIFICATE_NO: creditNoteCertNumber, }; const fileName = `${debitNum}.csv`; await wfmFileService.generateForm16IncomingCSV([csvRow], fileName, 'debit'); - logger.info(`[Form16] Debit note CSV pushed to WFM FORM16_DEBT: ${debitNum}`); + logger.info(`[Form16] Debit note CSV pushed to WFM FORM16: ${debitNum}`); } catch (csvErr: any) { - logger.error('[Form16] Failed to push debit note CSV to WFM FORM16_DEBT:', csvErr?.message || csvErr); + logger.error('[Form16] Failed to push debit note CSV to WFM FORM16:', csvErr?.message || csvErr); } } } diff --git a/src/services/form16Notification.service.ts b/src/services/form16Notification.service.ts index 8c32a86..584b28c 100644 --- a/src/services/form16Notification.service.ts +++ b/src/services/form16Notification.service.ts @@ -164,9 +164,22 @@ function resolveReminderTemplate(configTemplate: string | undefined): string { if (!t) return fallback; // Guard against swapped template in config. if (/\[duedate\]/i.test(t)) return fallback; + // Guard against legacy awkward template currently seen in UAT screenshots. + if (/form 16 submission is pending\.\s*\[name\],\s*\[request id\]\.\s*please review\.?/i.test(t)) return fallback; + // Enforce required placeholders to keep reminder body meaningful and consistent. + if (!/\[name\]/i.test(t) || !/\[request id\]/i.test(t)) return fallback; return t; } +function sanitizeRequestRef(value: string | undefined): string { + const v = (value || '').trim(); + if (!v) return '—'; + // Avoid sending internal UUIDs in user-facing reminder templates. + const uuidV4Like = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + if (uuidV4Like.test(v)) return 'pending Form 16 request'; + return v; +} + function getQuarterInfoForDate(d: Date): { financialYear: string; quarter: 'Q1' | 'Q2' | 'Q3' | 'Q4' } { const year = d.getFullYear(); const month = d.getMonth() + 1; // 1..12 @@ -342,7 +355,7 @@ export async function triggerForm16Reminder(dealerUserIds: string[], placeholder const template = resolveReminderTemplate(config.reminderNotificationTemplate); const body = replacePlaceholders(template, { Name: placeholders?.name ?? 'Dealer', - 'Request ID': placeholders?.requestId ?? '—', + 'Request ID': sanitizeRequestRef(placeholders?.requestId), }); const { notificationService } = await import('./notification.service'); await notificationService.sendToUsers(dealerUserIds, { diff --git a/src/services/wfmFile.service.ts b/src/services/wfmFile.service.ts index 4fa2194..61a49b8 100644 --- a/src/services/wfmFile.service.ts +++ b/src/services/wfmFile.service.ts @@ -5,17 +5,19 @@ import logger from '../utils/logger'; /** Default WFM folder names (joined with path.sep for current OS). */ const DEFAULT_CLAIMS_INCOMING = path.join('WFM-QRE', 'INCOMING', 'WFM_MAIN', 'DLR_INC_CLAIMS'); const DEFAULT_CLAIMS_OUTGOING = path.join('WFM-QRE', 'OUTGOING', 'WFM_SAP_MAIN', 'DLR_INC_CLAIMS'); -const DEFAULT_FORM16_CREDIT_INCOMING = path.join('WFM-QRE', 'INCOMING', 'WFM_MAIN', 'FORM16_CRDT'); -const DEFAULT_FORM16_DEBIT_INCOMING = path.join('WFM-QRE', 'INCOMING', 'WFM_MAIN', 'FORM16_DBT'); -const DEFAULT_FORM16_CREDIT_OUTGOING = path.join('WFM-QRE', 'OUTGOING', 'WFM_SAP_MAIN', 'FORM16_CRDT'); -const DEFAULT_FORM16_DEBIT_OUTGOING = path.join('WFM-QRE', 'OUTGOING', 'WFM_SAP_MAIN', 'FORM16_DBT'); +const DEFAULT_FORM16_INCOMING_MAIN = path.join('WFM-QRE', 'INCOMING', 'WFM_MAIN', 'FORM16'); +const DEFAULT_FORM16_INCOMING_ARCHIVE = path.join('WFM-QRE', 'INCOMING', 'WFM_ARCHIVE', 'FORM16'); +const DEFAULT_FORM16_OUTGOING_MAIN = path.join('WFM-QRE', 'OUTGOING', 'WFM_SAP_MAIN', 'FORM16'); +const DEFAULT_FORM16_OUTGOING_ARCHIVE = path.join('WFM-QRE', 'OUTGOING', 'WFM_SAP_ARCHIVE', 'FORM16'); /** * WFM File Service * Handles generation and storage of CSV files in the WFM folder structure. - * Dealer claims use DLR_INC_CLAIMS; Form 16 uses: - * - FORM16_CRDT (credit) and FORM16_DEBT (debit) under INCOMING/WFM_MAIN - * - FORM16_CRDT (credit) and FORM16_DBT (debit) under OUTGOING/WFM_SAP_MAIN + * Dealer claims use DLR_INC_CLAIMS; Form 16 uses unified folders: + * - INCOMING/WFM_MAIN/FORM16 + * - INCOMING/WFM_ARCHIVE/FORM16 + * - OUTGOING/WFM_SAP_MAIN/FORM16 + * - OUTGOING/WFM_SAP_ARCHIVE/FORM16 * Paths are cross-platform; set WFM_BASE_PATH (and optional path overrides) in .env for production. */ export class WFMFileService { @@ -34,13 +36,13 @@ export class WFMFileService { private form16IncomingDebitPath: string; private incomingArchiveForm16DebitPath: string; - // --- OUTGOING PATHS (WFM_SAP_MAIN) --- + // --- OUTGOING PATHS (WFM_SAP_MAIN / WFM_SAP_ARCHIVE) --- private outgoingGstClaimsPath: string; private outgoingNonGstClaimsPath: string; - /** Form 16 credit responses: OUTGOING/WFM_SAP_MAIN/FORM16_CRDT */ + /** Form 16 credit responses: OUTGOING/WFM_SAP_MAIN/FORM16 */ private form16OutgoingCreditPath: string; - /** Form 16 debit responses: OUTGOING/WFM_SAP_MAIN/FORM16_DBT */ + /** Form 16 debit responses: OUTGOING/WFM_SAP_MAIN/FORM16 */ private form16OutgoingDebitPath: string; constructor() { @@ -53,27 +55,39 @@ export class WFMFileService { this.incomingNonGstClaimsPath = process.env.WFM_INCOMING_NON_GST_CLAIMS_PATH || DEFAULT_CLAIMS_INCOMING + '_NON_GST'; this.incomingArchiveNonGstClaimsPath = process.env.WFM_ARCHIVE_NON_GST_CLAIMS_PATH || path.join('WFM-QRE', 'INCOMING', 'WFM_ARACHIVE', 'DLR_INC_CLAIMS_NON_GST'); - // Backwards-compatible: support legacy WFM_FORM16_INCOMING_PATH if specific credit/debit paths are not set + // Form16 unified path (credit/debit both use FORM16). Keep legacy vars as fallback. const legacyForm16Incoming = process.env.WFM_FORM16_INCOMING_PATH; + const form16IncomingMain = + process.env.WFM_FORM16_INCOMING_MAIN_PATH || + process.env.WFM_FORM16_CREDIT_INCOMING_PATH || + process.env.WFM_FORM16_DEBIT_INCOMING_PATH || + legacyForm16Incoming || + DEFAULT_FORM16_INCOMING_MAIN; + const form16IncomingArchive = + process.env.WFM_FORM16_INCOMING_ARCHIVE_PATH || + process.env.WFM_FORM16_CREDIT_ARCHIVE_PATH || + process.env.WFM_FORM16_DEBIT_ARCHIVE_PATH || + DEFAULT_FORM16_INCOMING_ARCHIVE; - this.form16IncomingCreditPath = - process.env.WFM_FORM16_CREDIT_INCOMING_PATH || legacyForm16Incoming || DEFAULT_FORM16_CREDIT_INCOMING; - this.incomingArchiveForm16CreditPath = process.env.WFM_FORM16_CREDIT_ARCHIVE_PATH || path.join('WFM-QRE', 'INCOMING', 'WFM_ARACHIVE', 'FORM16_CRDT'); - - this.form16IncomingDebitPath = - process.env.WFM_FORM16_DEBIT_INCOMING_PATH || legacyForm16Incoming || DEFAULT_FORM16_DEBIT_INCOMING; - this.incomingArchiveForm16DebitPath = process.env.WFM_FORM16_DEBIT_ARCHIVE_PATH || path.join('WFM-QRE', 'INCOMING', 'WFM_ARACHIVE', 'FORM16_DBT'); + this.form16IncomingCreditPath = form16IncomingMain; + this.form16IncomingDebitPath = form16IncomingMain; + this.incomingArchiveForm16CreditPath = form16IncomingArchive; + this.incomingArchiveForm16DebitPath = form16IncomingArchive; // Initialize Outgoing Paths from .env or defaults this.outgoingGstClaimsPath = process.env.WFM_OUTGOING_GST_CLAIMS_PATH || DEFAULT_CLAIMS_OUTGOING + '_GST'; this.outgoingNonGstClaimsPath = process.env.WFM_OUTGOING_NON_GST_CLAIMS_PATH || DEFAULT_CLAIMS_OUTGOING + '_NON_GST'; - // Outgoing: allow specific credit/debit overrides; fall back to legacy single path for credit + // Outgoing unified path (credit/debit both use FORM16). Keep legacy vars as fallback. const legacyForm16Outgoing = process.env.WFM_FORM16_OUTGOING_PATH; - this.form16OutgoingCreditPath = - process.env.WFM_FORM16_CREDIT_OUTGOING_PATH || legacyForm16Outgoing || DEFAULT_FORM16_CREDIT_OUTGOING; - this.form16OutgoingDebitPath = - process.env.WFM_FORM16_DEBIT_OUTGOING_PATH || DEFAULT_FORM16_DEBIT_OUTGOING; + const form16OutgoingMain = + process.env.WFM_FORM16_OUTGOING_MAIN_PATH || + process.env.WFM_FORM16_CREDIT_OUTGOING_PATH || + process.env.WFM_FORM16_DEBIT_OUTGOING_PATH || + legacyForm16Outgoing || + DEFAULT_FORM16_OUTGOING_MAIN; + this.form16OutgoingCreditPath = form16OutgoingMain; + this.form16OutgoingDebitPath = form16OutgoingMain; } /** @@ -188,13 +202,11 @@ export class WFMFileService { } /** - * Generate a CSV file for Form 16 (credit/debit note) and store in the appropriate INCOMING/WFM_MAIN folder. - * - Credit: FORM16_CRDT - * - Debit: FORM16_DEBT + * Generate a CSV file for Form 16 (credit/debit note) and store in INCOMING/WFM_MAIN/FORM16. * Format: pipe (|) as column separator, no double quotes around values (SAP/WFM requirement). * @param data Array of one or more row objects (keys become header; use UPPER_SNAKE_CASE for column names) * @param fileName File name (e.g. CN-F-16-6282-24-25-Q1.csv or DN-F-16-6282-24-25-Q1.csv) - * @param type 'credit' (default) or 'debit' – selects FORM16_CRDT vs FORM16_DEBT + * @param type 'credit' (default) or 'debit' – logical type only; folder is unified FORM16 */ async generateForm16IncomingCSV(data: any[], fileName: string, type: 'credit' | 'debit' = 'credit'): Promise { const maxRetries = 3; @@ -257,8 +269,7 @@ export class WFMFileService { /** * Get the absolute path for a Form 16 outgoing (response) file. - * - Credit: WFM/WFM-QRE/OUTGOING/WFM_SAP_MAIN/FORM16_CRDT - * - Debit: WFM/WFM-QRE/OUTGOING/WFM_SAP_MAIN/FORM16_DBT + * Both credit and debit use WFM/WFM-QRE/OUTGOING/WFM_SAP_MAIN/FORM16. */ getForm16OutgoingPath(fileName: string, type: 'credit' | 'debit' = 'credit'): string { const targetPath = type === 'debit' ? this.form16OutgoingDebitPath : this.form16OutgoingCreditPath;