diff --git a/src/controllers/form16.controller.ts b/src/controllers/form16.controller.ts index 42eedd1..79203ff 100644 --- a/src/controllers/form16.controller.ts +++ b/src/controllers/form16.controller.ts @@ -851,7 +851,8 @@ export class Form16Controller { deductorName, version, ocrExtractedData, - } + }, + (req as AuthenticatedRequest).user?.email ); const { triggerForm16SubmissionResultNotification } = await import('../services/form16Notification.service'); diff --git a/src/migrations/20260324110001-add-pan-number-to-26as.ts b/src/migrations/20260324110001-add-pan-number-to-26as.ts index 09e2c89..d039b89 100644 --- a/src/migrations/20260324110001-add-pan-number-to-26as.ts +++ b/src/migrations/20260324110001-add-pan-number-to-26as.ts @@ -7,14 +7,12 @@ module.exports = { type: DataTypes.STRING(20), allowNull: true, comment: 'PAN from 26AS header (assessee PAN)', - }).catch(() => {}); - await queryInterface.addIndex('tds_26as_entries', ['pan_number'], { - name: 'idx_tds_26as_pan', - }).catch(() => {}); + }); + await queryInterface.addIndex('tds_26as_entries', ['pan_number'], { name: 'idx_tds_26as_pan' }); }, down: async (queryInterface: QueryInterface) => { - await queryInterface.removeIndex('tds_26as_entries', 'idx_tds_26as_pan').catch(() => {}); - await queryInterface.removeColumn('tds_26as_entries', 'pan_number').catch(() => {}); + await queryInterface.removeIndex('tds_26as_entries', 'idx_tds_26as_pan'); + await queryInterface.removeColumn('tds_26as_entries', 'pan_number'); }, }; diff --git a/src/migrations/20260325090001-ensure-pan-number-in-26as.ts b/src/migrations/20260325090001-ensure-pan-number-in-26as.ts new file mode 100644 index 0000000..ed55ad1 --- /dev/null +++ b/src/migrations/20260325090001-ensure-pan-number-in-26as.ts @@ -0,0 +1,46 @@ +import type { QueryInterface } from 'sequelize'; +import { DataTypes, QueryTypes } from 'sequelize'; + +module.exports = { + up: async (queryInterface: QueryInterface) => { + // Use information_schema so this migration is safe even if a previous run + // recorded as "executed" but didn't actually alter the schema. + const sequelize = (queryInterface as any).sequelize; + + const [colRow] = await sequelize.query( + `SELECT CASE WHEN COUNT(*) > 0 THEN true ELSE false END AS exists + FROM information_schema.columns + WHERE table_name = 'tds_26as_entries' AND column_name = 'pan_number'`, + { type: QueryTypes.SELECT } + ); + + if (!colRow?.exists) { + await queryInterface.addColumn('tds_26as_entries', 'pan_number', { + type: DataTypes.STRING(20), + allowNull: true, + comment: 'PAN from 26AS header (assessee PAN)', + }); + } + + const [idxRow] = await sequelize.query( + `SELECT CASE WHEN COUNT(*) > 0 THEN true ELSE false END AS exists + FROM pg_indexes + WHERE schemaname = 'public' + AND tablename = 'tds_26as_entries' + AND indexname = 'idx_tds_26as_pan'`, + { type: QueryTypes.SELECT } + ); + + if (!idxRow?.exists) { + await queryInterface.addIndex('tds_26as_entries', ['pan_number'], { name: 'idx_tds_26as_pan' }); + } + }, + + down: async (queryInterface: QueryInterface) => { + // Best-effort rollback. If column/index already absent, these may throw. + // We intentionally keep down strict because rollback isn't required for forward fixes. + await queryInterface.removeIndex('tds_26as_entries', 'idx_tds_26as_pan'); + await queryInterface.removeColumn('tds_26as_entries', 'pan_number'); + }, +}; + diff --git a/src/scripts/migrate.ts b/src/scripts/migrate.ts index 4a02244..44741d0 100644 --- a/src/scripts/migrate.ts +++ b/src/scripts/migrate.ts @@ -72,6 +72,7 @@ import * as m64 from '../migrations/20260318100001-create-form16-debit-note-sap- import * as m65 from '../migrations/20260318200001-add-sap-response-csv-fields'; import * as m66 from '../migrations/20260324090001-refactor-form16-sap-response-and-add-read-log'; import * as m67 from '../migrations/20260324110001-add-pan-number-to-26as'; +import * as m68 from '../migrations/20260325090001-ensure-pan-number-in-26as'; interface Migration { name: string; @@ -151,6 +152,7 @@ const migrations: Migration[] = [ { name: '20260318200001-add-sap-response-csv-fields', module: m65 }, { name: '20260324090001-refactor-form16-sap-response-and-add-read-log', module: m66 }, { name: '20260324110001-add-pan-number-to-26as', module: m67 }, + { name: '20260325090001-ensure-pan-number-in-26as', module: m68 }, ]; @@ -210,7 +212,7 @@ async function markMigrationExecuted(sequelize: any, name: string): Promise process.exit(0)) + .catch(() => process.exit(1)); +} diff --git a/src/server.ts b/src/server.ts index e963e52..f5dc871 100644 --- a/src/server.ts +++ b/src/server.ts @@ -62,6 +62,21 @@ const startServer = async (): Promise => { // Initialize database connection explicitly after secrets are loaded await initializeAppDatabase(); + // Apply pending DB migrations automatically in non-production to prevent schema drift. + // Can be disabled via RUN_MIGRATIONS_ON_STARTUP=false + const autoMigrate = + process.env.RUN_MIGRATIONS_ON_STARTUP === 'true' || + process.env.NODE_ENV !== 'production'; + if (autoMigrate) { + try { + const { runMigrations } = require('./scripts/migrate'); + await runMigrations(); + } catch (e) { + console.error('❌ Auto migration on startup failed:', e); + process.exit(1); + } + } + require('./queues/tatWorker'); // Initialize TAT worker const { logTatConfig } = require('./config/tat.config'); const { logSystemConfig } = require('./config/system.config'); diff --git a/src/services/form16.service.ts b/src/services/form16.service.ts index 26b22a3..d2e1332 100644 --- a/src/services/form16.service.ts +++ b/src/services/form16.service.ts @@ -38,7 +38,7 @@ import logger from '../utils/logger'; * Same dealer code is used for submission, credit note, and debit note generation. * Returns null if no dealer code found. */ -export async function getDealerCodeForUser(userId: string): Promise { +export async function getDealerCodeForUser(userId: string, userEmail?: string | null): Promise { const [row] = await sequelize.query<{ employee_number: string | null }>( `SELECT employee_number FROM users WHERE user_id = :userId LIMIT 1`, { replacements: { userId }, type: QueryTypes.SELECT } @@ -48,10 +48,11 @@ export async function getDealerCodeForUser(userId: string): Promise { + if (_panNumberColumnPresent !== null) return _panNumberColumnPresent; + try { + const [row] = await sequelize.query<{ exists: boolean }>( + `SELECT CASE WHEN COUNT(*) > 0 THEN true ELSE false END AS exists + FROM information_schema.columns + WHERE table_name = 'tds_26as_entries' AND column_name = 'pan_number'`, + { type: QueryTypes.SELECT } + ); + _panNumberColumnPresent = !!row?.exists; + } catch (e) { + logger.warn('[Form16] pan_number column presence check failed; disabling PAN persistence/enforcement.', { + error: e instanceof Error ? e.message : String(e), + }); + _panNumberColumnPresent = false; + } + return _panNumberColumnPresent; +} + /** * Get aggregated TDS amount for (tan, fy, quarter) from the LATEST 26AS upload only (Section 194Q, Booking F/O). * Use case: "Always match Form 16A only with the latest 26AS version." Each upload can be full cumulative; @@ -129,6 +152,9 @@ async function getLatest26asRowsForQuarter( const fy = normalizeFinancialYear(financialYear) || financialYear; const q = normalizeQuarter(quarter) || quarter; + const hasPan = await isPanNumberColumnPresent(); + const panSelect = hasPan ? 'e.pan_number' : 'NULL::text AS pan_number'; + const rows = await sequelize.query<{ pan_number: string | null; amount_paid: string | null; @@ -146,7 +172,7 @@ async function getLatest26asRowsForQuarter( AND upload_log_id IS NOT NULL ) SELECT - e.pan_number, + ${panSelect}, e.amount_paid, e.tax_deducted, e.total_tds_deposited, @@ -175,18 +201,109 @@ async function getLatest26asRowsForQuarter( })); } +async function get26asCoverageDebug(tanNumber: string, financialYear: string, quarter: string) { + const normalizedTan = normalizeTanNumber(tanNumber); + const fy = normalizeFinancialYear(financialYear) || financialYear; + const q = normalizeQuarter(quarter) || quarter; + + // Overall counts + how many rows qualify for our matching rule: + const [counts] = await sequelize.query<{ + total_rows: string; + matching_194q_f_o_rows: string; + }>( + `SELECT + COUNT(*)::text AS total_rows, + SUM( + CASE + WHEN UPPER(TRIM(COALESCE(section_code, ''))) = :section + AND UPPER(TRIM(COALESCE(status_oltas, ''))) IN ('F', 'O') + THEN 1 ELSE 0 + END + )::text AS matching_194q_f_o_rows + FROM tds_26as_entries e + WHERE UPPER(REGEXP_REPLACE(TRIM(COALESCE(e.tan_number, '')), '[^A-Z0-9]', '', 'g')) = :tan + AND e.financial_year = :fy + AND e.quarter = :q`, + { replacements: { tan: normalizedTan, fy, q, section: SECTION_26AS_194Q }, type: QueryTypes.SELECT } + ); + + // Section/status breakdown to make it obvious why latestRows became empty. + const breakdown = (await sequelize.query( + `SELECT + section_code, + status_oltas, + COUNT(*)::text AS cnt + FROM tds_26as_entries e + WHERE UPPER(REGEXP_REPLACE(TRIM(COALESCE(e.tan_number, '')), '[^A-Z0-9]', '', 'g')) = :tan + AND e.financial_year = :fy + AND e.quarter = :q + GROUP BY section_code, status_oltas + ORDER BY cnt DESC + LIMIT 8`, + { replacements: { tan: normalizedTan, fy, q }, type: QueryTypes.SELECT } + )) as Array<{ section_code: string | null; status_oltas: string | null; cnt: string }>; + + const totalRows = parseInt(String(counts?.total_rows ?? '0'), 10) || 0; + const matchingRows = parseInt(String(counts?.matching_194q_f_o_rows ?? '0'), 10) || 0; + + const breakdownLines = (breakdown || []) + .map((b) => { + const sec = (b.section_code ?? '').toString().trim() || '(null)'; + const st = (b.status_oltas ?? '').toString().trim() || '(null)'; + const c = parseInt(String(b.cnt ?? '0'), 10) || 0; + return `${sec}/${st}:${c}`; + }) + .join(', '); + + return { totalRows, matchingRows, breakdownLines }; +} + function normalizeDateOnly(value: unknown): string | null { if (!value) return null; const raw = String(value).trim(); if (!raw) return null; + + // Prefer Indian-style numeric dates from OCR (DD-MM-YYYY or DD/MM/YYYY). + // Do NOT let JS Date parse these first, because it may interpret as MM-DD-YYYY. + const m = raw.match(/^(\d{1,2})[-\/](\d{1,2})[-\/](\d{2,4})$/); + if (m) { + const dd = m[1].padStart(2, '0'); + const mm = m[2].padStart(2, '0'); + const yyyy = m[3].length === 2 ? `20${m[3]}` : m[3]; + return `${yyyy}-${mm}-${dd}`; + } + const d = new Date(raw); if (!Number.isNaN(d.getTime())) return d.toISOString().slice(0, 10); - const m = raw.match(/^(\d{1,2})[-\/](\d{1,2})[-\/](\d{2,4})$/); - if (!m) return null; - const dd = m[1].padStart(2, '0'); - const mm = m[2].padStart(2, '0'); - const yyyy = m[3].length === 2 ? `20${m[3]}` : m[3]; - return `${yyyy}-${mm}-${dd}`; + return null; +} + +/** + * Derive Indian FY and 26AS quarter from an OCR date-only string (YYYY-MM-DD). + * Uses the same quarter boundaries as 26AS dateToFyAndQuarter: + * - Apr-Jun => Q1 (FY end = year+1) + * - Jul-Sep => Q2 + * - Oct-Dec => Q3 + * - Jan-Mar => Q4 (FY end = year) + */ +function deriveFyAndQuarterFromDateOnly(dateOnly: string | null): { financialYear: string; quarter: string } | null { + if (!dateOnly) return null; + const d = new Date(`${dateOnly}T00:00:00.000Z`); + if (Number.isNaN(d.getTime())) return null; + + const month = d.getUTCMonth() + 1; // 1-12 + const year = d.getUTCFullYear(); + + let quarter: string; + if ([4, 5, 6].includes(month)) quarter = 'Q1'; + else if ([7, 8, 9].includes(month)) quarter = 'Q2'; + else if ([10, 11, 12].includes(month)) quarter = 'Q3'; + else quarter = 'Q4'; // Jan-Mar + + const fyEnd = [1, 2, 3].includes(month) ? year : year + 1; + const fyStart = fyEnd - 1; + const next = (fyEnd % 100).toString().padStart(2, '0'); + return { financialYear: `${fyStart}-${next}`, quarter }; } function toNumberOrNull(value: unknown): number | null { @@ -640,8 +757,8 @@ async function run26asMatchAndCreditNote(submission: Form16aSubmission): Promise const financialYearRaw = (sub.financialYear || '').trim(); const quarterRaw = (sub.quarter || '').trim(); - const financialYear = normalizeFinancialYear(financialYearRaw) || financialYearRaw; - const quarter = normalizeQuarter(quarterRaw) || quarterRaw; + let financialYear = normalizeFinancialYear(financialYearRaw) || financialYearRaw; + let quarter = normalizeQuarter(quarterRaw) || quarterRaw; if (!financialYear || !quarter) { logger.warn( @@ -668,10 +785,31 @@ async function run26asMatchAndCreditNote(submission: Form16aSubmission): Promise const submittedBookingDate = normalizeDateOnly(extracted.dateOfBooking); // Latest 26AS upload rows for the same TAN + FY + Quarter. - const latestRows = await getLatest26asRowsForQuarter(tanNumber, financialYear, quarter); + let latestRows = await getLatest26asRowsForQuarter(tanNumber, financialYear, quarter); + + // If OCR extracted FY/Quarter incorrectly, derive FY/Quarter from OCR dates and retry. + if (latestRows.length === 0) { + const derivedFromTx = deriveFyAndQuarterFromDateOnly(submittedTransactionDate); + const derivedFromBooking = deriveFyAndQuarterFromDateOnly(submittedBookingDate); + const derived = derivedFromTx || derivedFromBooking; + if (derived && (derived.financialYear !== financialYear || derived.quarter !== quarter)) { + const altRows = await getLatest26asRowsForQuarter(tanNumber, derived.financialYear, derived.quarter); + if (altRows.length > 0) { + logger.warn( + `[Form16] FY/Quarter retry using OCR date-derived period. TAN=${tanNumber}. OCR FY=${financialYear},Q=${quarter} → derived FY=${derived.financialYear},Q=${derived.quarter}.` + ); + financialYear = derived.financialYear; + quarter = derived.quarter; + latestRows = altRows; + await submission.update({ financialYear, quarter }); + } + } + } + const aggregated26as = latestRows.reduce((sum, r) => sum + (r.taxDeducted || 0), 0); - if (normalizedSubmittedPan) { + const hasPanColumn = await isPanNumberColumnPresent(); + if (normalizedSubmittedPan && hasPanColumn && latestRows.length > 0) { const hasPanMatch = latestRows.some((r) => r.panNumber && r.panNumber === normalizedSubmittedPan); if (!hasPanMatch) { logger.warn( @@ -683,17 +821,34 @@ async function run26asMatchAndCreditNote(submission: Form16aSubmission): Promise }); return { validationStatus: 'failed', validationNotes: 'PAN mismatch with latest 26AS.' }; } + } else if (normalizedSubmittedPan && !hasPanColumn) { + logger.warn( + `[Form16] PAN strict check skipped because DB column tds_26as_entries.pan_number is missing. TAN=${tanNumber}, FY=${financialYear}, Quarter=${quarter}.` + ); } - if (aggregated26as <= 0) { + if (latestRows.length === 0) { + // Provide actionable debug info so we can see why latestRows became empty + // (section_code not 194Q, status not F/O, FY/Quarter mismatch, or TAN mismatch). + let debugNotes = ''; + try { + const dbg = await get26asCoverageDebug(tanNumber, financialYear, quarter); + debugNotes = ` | DEBUG 26AS coverage: total=${dbg.totalRows}, matching(194Q & F/O)=${dbg.matchingRows}. Top breakdown: ${dbg.breakdownLines || '(none)'}`; + } catch (e: any) { + debugNotes = ` | DEBUG 26AS coverage query failed: ${e?.message || String(e)}`; + } + logger.warn( - `[Form16] 26AS MATCH RESULT: FAILED – No 26AS data. Form 16A: TAN=${tanNumber}, FY=${financialYear}, Quarter=${quarter}, TDS amount=${tdsAmount} | 26AS: no records for this TAN/FY/Quarter (Section 194Q, Booking F/O).` + `[Form16] 26AS MATCH RESULT: FAILED – No 26AS data. Form 16A: TAN=${tanNumber}, FY=${financialYear}, Quarter=${quarter}, TDS amount=${tdsAmount} | 26AS: no records for this TAN/FY/Quarter (Section 194Q, Booking F/O).${debugNotes}` ); await submission.update({ validationStatus: 'failed', - validationNotes: `No 26AS data found for TAN no - ${tanNumber}, financial year and quarter. Please ensure 26AS has been uploaded for this period.`, + validationNotes: `No 26AS data found for TAN no - ${tanNumber}, financial year and quarter. Please ensure 26AS has been uploaded for this period.${debugNotes}`, }); - return { validationStatus: 'failed', validationNotes: `No 26AS record found for this TAN no - ${tanNumber}, financial year and quarter.` }; + return { + validationStatus: 'failed', + validationNotes: `No 26AS record found for this TAN no - ${tanNumber}, financial year and quarter.${debugNotes}`, + }; } // Validate against quarter-level aggregate from latest upload. @@ -851,7 +1006,7 @@ async function run26asMatchAndCreditNote(submission: Form16aSubmission): Promise DLR_TAN_NO: tanNumber, 'FIN_YEAR&QUARTER': finYearAndQuarter, DOC_DATE: docDate, - TDS_AMT: Number(tdsAmount).toFixed(2), + TDS_AMT: `+${Number(Math.abs(tdsAmount)).toFixed(2)}`, TDS_CERTIFICATE_NO: certificateNumber, }; const fileName = `${cnNumber}.csv`; @@ -872,17 +1027,23 @@ export async function createSubmission( userId: string, fileBuffer: Buffer, originalName: string, - body: CreateForm16SubmissionBody + body: CreateForm16SubmissionBody, + userEmail?: string | null ): Promise { // Dealer: dealer code from users.employee_number (getDealerCodeForUser) or body.dealerCode for RE/UAT. Used for submission, credit note, and debit note. - const resolvedDealerCode = await getDealerCodeForUser(userId); + const resolvedDealerCode = await getDealerCodeForUser(userId, userEmail); const overrideDealerCode = (body.dealerCode || '').trim() || null; const dealerCode = resolvedDealerCode || overrideDealerCode; + // Frontend may not provide dealerCode (dealer code may be absent in Form16/26AS), + // so we must not hard-fail. Use a deterministic fallback so note generation can continue. + const effectiveDealerCode = dealerCode || '000000'; if (!dealerCode) { - throw new Error('dealerCode is required to submit Form 16.'); + logger.warn( + '[Form16] dealerCode not resolved for submission; using fallback dealerCode="000000". Matching uses TAN/FY/Quarter, but note numbering/CSV dealer code will be for fallback.' + ); } - const version = await getNextVersionForDealerFyQuarter(dealerCode, body.financialYear, body.quarter); + const version = await getNextVersionForDealerFyQuarter(effectiveDealerCode, body.financialYear, body.quarter); const requestNumber = await generateRequestNumber(); const title = version > 1 @@ -937,7 +1098,7 @@ export async function createSubmission( const safeStr = (s: string, max: number) => (s ?? '').slice(0, max); const submission = await Form16aSubmission.create({ requestId, - dealerCode: safeStr(dealerCode, 50), + dealerCode: safeStr(effectiveDealerCode, 50), form16aNumber: safeStr(body.form16aNumber, 50), financialYear: safeStr(body.financialYear, 20), quarter: safeStr(body.quarter, 10), @@ -2156,8 +2317,8 @@ export async function create26asEntry(data: { statusOltas?: string; remarks?: string; }) { - const entry = await Tds26asEntry.create({ - panNumber: data.panNumber, + const includePanNumber = await isPanNumberColumnPresent(); + const payload: any = { tanNumber: data.tanNumber, deductorName: data.deductorName, quarter: data.quarter, @@ -2172,7 +2333,10 @@ export async function create26asEntry(data: { dateOfBooking: data.dateOfBooking, statusOltas: data.statusOltas, remarks: data.remarks, - }); + }; + if (includePanNumber && data.panNumber != null) payload.panNumber = data.panNumber; + + const entry = await Tds26asEntry.create(payload); return entry; } @@ -2198,6 +2362,10 @@ export async function update26asEntry( ) { const entry = await Tds26asEntry.findByPk(id); if (!entry) return null; + if (data.panNumber != null) { + const includePanNumber = await isPanNumberColumnPresent(); + if (!includePanNumber) delete (data as any).panNumber; + } await entry.update(data); return entry; } @@ -2448,15 +2616,20 @@ const TDS_26AS_CREATE_KEYS = [ 'transactionDate', 'dateOfBooking', 'assessmentYear', 'statusOltas', 'remarks', 'uploadLogId', ] as const; -function build26asCreatePayload(row: Record, uploadLogId?: number | null): Record { +function build26asCreatePayload( + row: Record, + uploadLogId: number | null | undefined, + includePanNumber: boolean +): Record { const payload: Record = {}; for (const k of TDS_26AS_CREATE_KEYS) { if (k === 'uploadLogId') continue; + if (k === 'panNumber' && !includePanNumber) continue; const v = row[k]; if (v !== undefined && v !== null) payload[k] = v; } payload.tanNumber = normalizeTanNumber(row.tanNumber); - if (row.panNumber != null) payload.panNumber = String(row.panNumber).trim().toUpperCase(); + if (includePanNumber && row.panNumber != null) payload.panNumber = String(row.panNumber).trim().toUpperCase(); const rawFy = (row.financialYear != null ? String(row.financialYear).trim() : '') || ''; const rawQ = (row.quarter != null ? String(row.quarter).trim() : '') || 'Q1'; payload.financialYear = normalizeFinancialYear(rawFy) || rawFy; @@ -2479,9 +2652,11 @@ export async function upload26asFile(buffer: Buffer, uploadLogId?: number | null } const insertErrors: string[] = []; let imported = 0; + + const includePanNumber = await isPanNumberColumnPresent(); for (let i = 0; i < rows.length; i += TDS_26AS_BATCH_SIZE) { const batch = rows.slice(i, i + TDS_26AS_BATCH_SIZE); - const payloads = batch.map((r) => build26asCreatePayload(r as Record, uploadLogId)); + const payloads = batch.map((r) => build26asCreatePayload(r as Record, uploadLogId, includePanNumber)); try { const created = await Tds26asEntry.bulkCreate(payloads as any[], { validate: true }); imported += created.length;