From 6c0902cd578e5fd67b4216b4a803e5dc0728ec40 Mon Sep 17 00:00:00 2001 From: yashwin-foxy Date: Thu, 25 Sep 2025 17:40:59 +0530 Subject: [PATCH] job creation flow added zoho crm --- .env | 3 +- src/api/controllers/integrationController.js | 116 +++++++++++++++++- src/api/routes/integrationRoutes.js | 5 +- src/data/models/zohoDealsBulk.js | 98 +++++++++++++++ .../repositories/zohoBulkReadRepository.js | 4 +- .../migrations/012_create_zoho_deals_bulk.sql | 33 +++++ src/integrations/zoho/mapper.js | 42 +++++++ src/services/bulkReadService.js | 56 ++++++--- src/services/csvService.js | 23 ++++ 9 files changed, 361 insertions(+), 19 deletions(-) create mode 100644 src/data/models/zohoDealsBulk.js create mode 100644 src/db/migrations/012_create_zoho_deals_bulk.sql diff --git a/.env b/.env index af02f4e..65fa715 100644 --- a/.env +++ b/.env @@ -5,4 +5,5 @@ DB_USER=root DB_PASSWORD=Admin@123 DB_NAME=centralized_reporting DB_HOST=127.0.0.1 -DB_PORT=3306 \ No newline at end of file +DB_PORT=3306 +MY_BASE_URL=http://160.187.167.216 \ No newline at end of file diff --git a/src/api/controllers/integrationController.js b/src/api/controllers/integrationController.js index f619a2f..50557ba 100644 --- a/src/api/controllers/integrationController.js +++ b/src/api/controllers/integrationController.js @@ -952,6 +952,120 @@ const getHolidays = async (req, res) => { } } +// Schedule bulk read jobs for all 5 modules +const scheduleBulkReadJobs = async (req, res) => { + try { + const BulkReadService = require('../../services/bulkReadService'); + const bulkReadService = new BulkReadService(); + + const userId = req.user.uuid; + const accessToken = req.headers.authorization?.replace('Bearer ', '') || null; + console.log(`🚀 Scheduling bulk read jobs for user: ${userId}`); + + // Define the 5 modules with their specific fields + const modules = [ + { + name: 'Accounts', + fields: [ + 'id', 'Account_Name', 'Phone', 'Website', 'Industry', + 'Ownership', 'Annual_Revenue', 'Owner', 'Created_Time' + ] + }, + { + name: 'Deals', + fields: [ + 'id', 'Deal_Name', 'Stage', 'Amount', 'Closing_Date', + 'Account_Name', 'Contact_Name', 'Pipeline', 'Probability', + 'Lead_Source', 'Owner', 'Created_Time', 'Modified_Time' + ] + }, + { + name: 'Contacts', + fields: [ + 'id', 'First_Name', 'Last_Name', 'Email', 'Phone', 'Mobile', + 'Lead_Source', 'Account_Name.Account_Name', 'Owner', 'Created_Time', 'Modified_Time' + ] + }, + { + name: 'Tasks', + fields: [ + 'id', 'Subject', 'Owner', 'Status', 'Priority', + 'Due_Date', 'What_Id', 'Created_Time' + ] + }, + { + name: 'Leads', + fields: [ + 'id', 'First_Name', 'Last_Name', 'Company', 'Lead_Source', + 'Lead_Status', 'Owner', 'Email', 'Phone', 'Created_Time' + ] + } + ]; + + const results = []; + const errors = []; + + // Schedule jobs for each module + for (const module of modules) { + try { + console.log(`📋 Scheduling bulk read for module: ${module.name}`); + + const result = await bulkReadService.initiateBulkRead( + userId, + module.name, + module.fields, + { page: 1, limit: 200000 }, + accessToken + ); + + results.push({ + module: module.name, + jobId: result.jobId, + status: result.status, + message: result.message, + zohoState: result.zohoState, + estimatedTime: result.estimatedTime + }); + + console.log(`✅ Scheduled ${module.name}: ${result.jobId}`); + + } catch (error) { + console.error(`❌ Failed to schedule ${module.name}:`, error.message); + errors.push({ + module: module.name, + error: error.message + }); + } + } + + // Prepare response + const response = { + status: 'success', + message: `Created ${results.length} bulk read jobs successfully. Processing will be handled asynchronously via webhooks.`, + scheduledJobs: results, + errors: errors.length > 0 ? errors : undefined, + summary: { + total: modules.length, + successful: results.length, + failed: errors.length + }, + note: 'Jobs are now being processed by Zoho. You will receive webhook notifications when each job completes.' + }; + + if (errors.length > 0) { + response.message += `, ${errors.length} failed`; + } + + console.log(`🎉 Bulk read scheduling completed: ${results.length}/${modules.length} successful`); + + res.json(response); + + } catch (error) { + console.error('❌ Error scheduling bulk read jobs:', error); + res.status(500).json(failure(error.message, 'BULK_READ_SCHEDULING_ERROR')); + } +} + module.exports = { getData, getServices, getResources, getPortals, getAllProjects, getAllProjectTasks, getAllProjectTaskLists, getAllProjectIssues, getAllProjectPhases, getSalesOrders, @@ -960,5 +1074,5 @@ module.exports = { getExpenses, getBankAccounts, getBankTransactions, getReports, getBooksSalesOrders, getBooksPurchaseOrders, getContacts, getBooksContacts, getBooksInvoices, getEmployeeForms, getEmployeeById, getAttendanceEntries, getShiftConfiguration, getLeaveData, getGoalsData, getPerformanceData, - getUserReport, getLeaveTrackerReport, getHolidays + getUserReport, getLeaveTrackerReport, getHolidays, scheduleBulkReadJobs }; diff --git a/src/api/routes/integrationRoutes.js b/src/api/routes/integrationRoutes.js index d137cb0..5742b43 100644 --- a/src/api/routes/integrationRoutes.js +++ b/src/api/routes/integrationRoutes.js @@ -8,7 +8,7 @@ const { getExpenses, getBankAccounts, getBankTransactions, getReports, getBooksSalesOrders, getBooksPurchaseOrders, getContacts, getBooksContacts, getBooksInvoices, getEmployeeForms, getEmployeeById, getAttendanceEntries, getShiftConfiguration, getLeaveData, getGoalsData, getPerformanceData, - getUserReport, getLeaveTrackerReport, getHolidays + getUserReport, getLeaveTrackerReport, getHolidays, scheduleBulkReadJobs } = require('../controllers/integrationController'); const auth = require('../middlewares/auth'); const ZohoHandler = require('../../integrations/zoho/handler'); @@ -352,6 +352,9 @@ router.get('/zoho/books/reports', auth, validate(reportsSchema), getReports); router.get('/zoho/books/sales-orders', auth, validate(booksSalesOrdersSchema), getBooksSalesOrders); router.get('/zoho/books/purchase-orders', auth, validate(booksPurchaseOrdersSchema), getBooksPurchaseOrders); +// Bulk read job scheduling endpoint +router.post('/zoho/bulk-read/schedule', auth, scheduleBulkReadJobs); + // Webhook endpoints (no auth required - uses signature verification) const zohoHandler = new ZohoHandler(); router.post('/webhooks/zoho/crm', zohoHandler.handleCrmWebhook.bind(zohoHandler)); diff --git a/src/data/models/zohoDealsBulk.js b/src/data/models/zohoDealsBulk.js new file mode 100644 index 0000000..38251f8 --- /dev/null +++ b/src/data/models/zohoDealsBulk.js @@ -0,0 +1,98 @@ +const { DataTypes } = require('sequelize'); +const sequelize = require('../../db/pool'); + +const ZohoDealsBulk = sequelize.define('ZohoDealsBulk', { + internal_id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false + }, + zoho_id: { + type: DataTypes.STRING(255), + allowNull: true + }, + user_uuid: { + type: DataTypes.CHAR(36), + allowNull: false + }, + provider: { + type: DataTypes.STRING(50), + allowNull: false, + defaultValue: 'zoho' + }, + deal_name: { + type: DataTypes.STRING(255), + allowNull: true + }, + account_name: { + type: DataTypes.STRING(255), + allowNull: true + }, + contact_name: { + type: DataTypes.STRING(255), + allowNull: true + }, + amount: { + type: DataTypes.DECIMAL(15, 2), + allowNull: true + }, + stage: { + type: DataTypes.STRING(255), + allowNull: true + }, + closing_date: { + type: DataTypes.DATE, + allowNull: true + }, + lead_source: { + type: DataTypes.STRING(255), + allowNull: true + }, + type: { + type: DataTypes.STRING(255), + allowNull: true + }, + probability: { + type: DataTypes.INTEGER, + allowNull: true + }, + next_step: { + type: DataTypes.TEXT, + allowNull: true + }, + description: { + type: DataTypes.TEXT, + allowNull: true + }, + owner: { + type: DataTypes.STRING(255), + allowNull: true + }, + created_time: { + type: DataTypes.DATE, + allowNull: true + }, + modified_time: { + type: DataTypes.DATE, + allowNull: true + }, + bulk_job_id: { + type: DataTypes.STRING(255), + allowNull: true + } +}, { + tableName: 'zoho_deals_bulk', + timestamps: true, + createdAt: 'created_at', + updatedAt: 'updated_at', + indexes: [ + { fields: ['user_uuid', 'provider'] }, + { fields: ['bulk_job_id'] }, + { fields: ['created_time'] }, + { fields: ['stage'] }, + { fields: ['closing_date'] } + ] +}); + +module.exports = ZohoDealsBulk; diff --git a/src/data/repositories/zohoBulkReadRepository.js b/src/data/repositories/zohoBulkReadRepository.js index de5f0ad..b115f0f 100644 --- a/src/data/repositories/zohoBulkReadRepository.js +++ b/src/data/repositories/zohoBulkReadRepository.js @@ -6,6 +6,7 @@ const ZohoVendorsBulk = require('../models/zohoVendorsBulk'); const ZohoInvoicesBulk = require('../models/zohoInvoicesBulk'); const ZohoSalesOrdersBulk = require('../models/zohoSalesOrdersBulk'); const ZohoPurchaseOrdersBulk = require('../models/zohoPurchaseOrdersBulk'); +const ZohoDealsBulk = require('../models/zohoDealsBulk'); const ZohoBulkReadJobs = require('../models/zohoBulkReadJobs'); class ZohoBulkReadRepository { @@ -18,7 +19,8 @@ class ZohoBulkReadRepository { 'vendors': ZohoVendorsBulk, 'invoices': ZohoInvoicesBulk, 'sales_orders': ZohoSalesOrdersBulk, - 'purchase_orders': ZohoPurchaseOrdersBulk + 'purchase_orders': ZohoPurchaseOrdersBulk, + 'deals': ZohoDealsBulk }; } diff --git a/src/db/migrations/012_create_zoho_deals_bulk.sql b/src/db/migrations/012_create_zoho_deals_bulk.sql new file mode 100644 index 0000000..509f9a4 --- /dev/null +++ b/src/db/migrations/012_create_zoho_deals_bulk.sql @@ -0,0 +1,33 @@ +-- Migration: Create Zoho Deals Bulk Table +-- Description: Creates table for storing bulk read deals data from Zoho CRM + +CREATE TABLE IF NOT EXISTS zoho_deals_bulk ( + internal_id INT AUTO_INCREMENT PRIMARY KEY, + zoho_id VARCHAR(255), + user_uuid CHAR(36) NOT NULL, + provider VARCHAR(50) NOT NULL DEFAULT 'zoho', + deal_name VARCHAR(255), + account_name VARCHAR(255), + contact_name VARCHAR(255), + amount DECIMAL(15, 2), + stage VARCHAR(255), + closing_date DATETIME, + lead_source VARCHAR(255), + type VARCHAR(255), + probability INT, + next_step TEXT, + description TEXT, + owner VARCHAR(255), + created_time DATETIME, + modified_time DATETIME, + bulk_job_id VARCHAR(255), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_user_provider (user_uuid, provider), + CONSTRAINT fk_zoho_deals_bulk_user FOREIGN KEY (user_uuid) REFERENCES users(uuid) ON DELETE CASCADE, + INDEX idx_bulk_job (bulk_job_id), + INDEX idx_created_time (created_time), + INDEX idx_zoho_id (zoho_id), + INDEX idx_stage (stage), + INDEX idx_closing_date (closing_date) +); diff --git a/src/integrations/zoho/mapper.js b/src/integrations/zoho/mapper.js index f070885..4fc0db6 100644 --- a/src/integrations/zoho/mapper.js +++ b/src/integrations/zoho/mapper.js @@ -805,6 +805,27 @@ class ZohoMapper { } break; + case 'accounts': + // Handle both CRM and Books accounts - check response structure + if (zohoResponse.data && zohoResponse.info) { + // Books response structure + records = zohoResponse.data || []; + pageInfo = { + count: zohoResponse.info?.count || records.length, + moreRecords: zohoResponse.info?.more_records || false, + page: zohoResponse.info?.page || 1 + }; + } else { + // CRM response structure + records = zohoResponse.accounts || []; + pageInfo = { + count: zohoResponse.page_context?.count || records.length, + moreRecords: zohoResponse.page_context?.more_records || false, + page: zohoResponse.page_context?.page || 1 + }; + } + break; + case 'contacts': // Books response structure for contacts if (zohoResponse.data && zohoResponse.info) { @@ -825,6 +846,27 @@ class ZohoMapper { } break; + case 'deals': + // Handle both CRM and Books deals - check response structure + if (zohoResponse.data && zohoResponse.info) { + // Books response structure + records = zohoResponse.data || []; + pageInfo = { + count: zohoResponse.info?.count || records.length, + moreRecords: zohoResponse.info?.more_records || false, + page: zohoResponse.info?.page || 1 + }; + } else { + // CRM response structure + records = zohoResponse.deals || []; + pageInfo = { + count: zohoResponse.page_context?.count || records.length, + moreRecords: zohoResponse.page_context?.more_records || false, + page: zohoResponse.page_context?.page || 1 + }; + } + break; + case 'vendors': // Books response structure for vendors (filtered contacts) records = zohoResponse.contacts || []; diff --git a/src/services/bulkReadService.js b/src/services/bulkReadService.js index 84d4b04..add3e95 100644 --- a/src/services/bulkReadService.js +++ b/src/services/bulkReadService.js @@ -15,9 +15,10 @@ class BulkReadService { * @param {string} module - Zoho module name * @param {Array} fields - Fields to fetch * @param {Object} options - Additional options + * @param {string} accessToken - Frontend access token for callback URL * @returns {Promise} Job details */ - async initiateBulkRead(userId, module, fields, options = {}) { + async initiateBulkRead(userId, module, fields, options = {}, accessToken = null) { try { console.log(`🚀 Initiating bulk read for user ${userId}, module ${module}`); @@ -27,12 +28,20 @@ class BulkReadService { throw new Error('No Zoho access token found for user'); } - const accessToken = decrypt(tokenRecord.accessToken); + const zohoAccessToken = decrypt(tokenRecord.accessToken); // Prepare bulk read request + const baseUrl = process.env.API_BASE_URL || process.env.MY_BASE_URL || 'http://localhost:3000'; + let callbackUrl = `${baseUrl}/api/v1/integrations/webhooks/zoho/bulkread`; + + // Add access token to callback URL if provided + if (accessToken) { + callbackUrl += `?access_token=${accessToken}`; + } + const bulkReadData = { callback: { - url: `${process.env.API_BASE_URL || 'http://localhost:3000'}/api/v1/integrations/webhooks/zoho/bulkread`, + url: callbackUrl, method: 'post' }, query: { @@ -51,38 +60,55 @@ class BulkReadService { bulkReadData, { headers: { - 'Authorization': `Zoho-oauthtoken ${accessToken}`, + 'Authorization': `Zoho-oauthtoken ${zohoAccessToken}`, 'Content-Type': 'application/json' } } ); const jobData = response.data; - console.log('✅ Bulk read job initiated:', jobData); + console.log('✅ Bulk read job response from Zoho:', JSON.stringify(jobData, null, 2)); - // Store job in database + // Extract job ID from nested response structure + const jobDetails = jobData.data && jobData.data[0] && jobData.data[0].details; + const jobId = jobDetails?.id; + + if (!jobId) { + console.log('⚠️ No job ID found in response structure'); + console.log('Response structure:', JSON.stringify(jobData, null, 2)); + throw new Error('Invalid response structure from Zoho API - no job ID found'); + } + + console.log('📋 Job ID received from Zoho:', jobId); + console.log('📋 Job details:', jobDetails); + + // Store job in database with real job ID const jobRecord = await ZohoBulkReadRepository.createBulkReadJob({ - id: jobData.id, + id: jobId, user_uuid: userId, module: module, - operation: 'read', - state: 'CREATED', + operation: jobDetails.operation || 'read', + state: jobDetails.state || 'ADDED', file_type: 'csv', records_count: 0, status: 'pending' }); - logger.info('Bulk read job initiated', { + logger.info('Bulk read job created and stored', { userId, module, - jobId: jobData.id, - fields: fields.length + jobId: jobId, + zohoState: jobDetails.state || 'ADDED' }); return { - jobId: jobData.id, - status: 'initiated', - message: `Bulk read job initiated for ${module}`, + jobId: jobId, + status: 'created', + message: `Bulk read job created for ${module}. Processing will be handled asynchronously via webhook.`, + zohoState: jobDetails.state || 'ADDED', + operation: jobDetails.operation || 'read', + createdBy: jobDetails.created_by, + createdTime: jobDetails.created_time, estimatedTime: this.getEstimatedTime(module, options.limit) }; diff --git a/src/services/csvService.js b/src/services/csvService.js index ee8979f..76e5149 100644 --- a/src/services/csvService.js +++ b/src/services/csvService.js @@ -163,6 +163,8 @@ class CsvService { return this.mapSalesOrderFields(record, baseRecord); case 'purchase_orders': return this.mapPurchaseOrderFields(record, baseRecord); + case 'deals': + return this.mapDealFields(record, baseRecord); default: console.warn(`⚠️ Unknown module: ${module}`); return { ...baseRecord, ...record }; @@ -297,6 +299,27 @@ class CsvService { }; } + mapDealFields(record, baseRecord) { + return { + ...baseRecord, + zoho_id: record.id, + deal_name: record.Deal_Name, + account_name: record['Account_Name.Account_Name'] || record.Account_Name, + contact_name: record['Contact_Name.Contact_Name'] || record.Contact_Name, + amount: this.parseDecimal(record.Amount), + stage: record.Stage, + closing_date: this.parseDate(record.Closing_Date), + lead_source: record.Lead_Source, + type: record.Type, + probability: record.Probability ? parseInt(record.Probability) : null, + next_step: record.Next_Step, + description: record.Description, + owner: record.Owner, + created_time: this.parseDate(record.Created_Time), + modified_time: this.parseDate(record.Modified_Time) + }; + } + // Utility methods parseDate(dateString) { if (!dateString || dateString === '') return null;