job creation flow added zoho crm

.env

@@ -6,3 +6,4 @@ DB_PASSWORD=Admin@123
 DB_NAME=centralized_reporting
 DB_HOST=127.0.0.1
 DB_PORT=3306
+MY_BASE_URL=http://160.187.167.216

src/api/controllers/integrationController.js

@@ -952,6 +952,120 @@ const getHolidays = async (req, res) => {
   }
 }
 
+// Schedule bulk read jobs for all 5 modules
+const scheduleBulkReadJobs = async (req, res) => {
+  try {
+    const BulkReadService = require('../../services/bulkReadService');
+    const bulkReadService = new BulkReadService();
+    
+    const userId = req.user.uuid;
+    const accessToken = req.headers.authorization?.replace('Bearer ', '') || null;
+    console.log(`🚀 Scheduling bulk read jobs for user: ${userId}`);
+    
+    // Define the 5 modules with their specific fields
+    const modules = [
+      {
+        name: 'Accounts',
+        fields: [
+          'id', 'Account_Name', 'Phone', 'Website', 'Industry',
+          'Ownership', 'Annual_Revenue', 'Owner', 'Created_Time'
+        ]
+      },
+      {
+        name: 'Deals',
+        fields: [
+          'id', 'Deal_Name', 'Stage', 'Amount', 'Closing_Date',
+          'Account_Name', 'Contact_Name', 'Pipeline', 'Probability',
+          'Lead_Source', 'Owner', 'Created_Time', 'Modified_Time'
+        ]
+      },
+      {
+        name: 'Contacts',
+        fields: [
+          'id', 'First_Name', 'Last_Name', 'Email', 'Phone', 'Mobile',
+          'Lead_Source', 'Account_Name.Account_Name', 'Owner', 'Created_Time', 'Modified_Time'
+        ]
+      },
+      {
+        name: 'Tasks',
+        fields: [
+          'id', 'Subject', 'Owner', 'Status', 'Priority',
+          'Due_Date', 'What_Id', 'Created_Time'
+        ]
+      },
+      {
+        name: 'Leads',
+        fields: [
+          'id', 'First_Name', 'Last_Name', 'Company', 'Lead_Source',
+          'Lead_Status', 'Owner', 'Email', 'Phone', 'Created_Time'
+        ]
+      }
+    ];
+    
+    const results = [];
+    const errors = [];
+    
+    // Schedule jobs for each module
+    for (const module of modules) {
+      try {
+        console.log(`📋 Scheduling bulk read for module: ${module.name}`);
+        
+        const result = await bulkReadService.initiateBulkRead(
+          userId,
+          module.name,
+          module.fields,
+          { page: 1, limit: 200000 },
+          accessToken
+        );
+        
+        results.push({
+          module: module.name,
+          jobId: result.jobId,
+          status: result.status,
+          message: result.message,
+          zohoState: result.zohoState,
+          estimatedTime: result.estimatedTime
+        });
+        
+        console.log(`✅ Scheduled ${module.name}: ${result.jobId}`);
+        
+      } catch (error) {
+        console.error(`❌ Failed to schedule ${module.name}:`, error.message);
+        errors.push({
+          module: module.name,
+          error: error.message
+        });
+      }
+    }
+    
+    // Prepare response
+    const response = {
+      status: 'success',
+      message: `Created ${results.length} bulk read jobs successfully. Processing will be handled asynchronously via webhooks.`,
+      scheduledJobs: results,
+      errors: errors.length > 0 ? errors : undefined,
+      summary: {
+        total: modules.length,
+        successful: results.length,
+        failed: errors.length
+      },
+      note: 'Jobs are now being processed by Zoho. You will receive webhook notifications when each job completes.'
+    };
+    
+    if (errors.length > 0) {
+      response.message += `, ${errors.length} failed`;
+    }
+    
+    console.log(`🎉 Bulk read scheduling completed: ${results.length}/${modules.length} successful`);
+    
+    res.json(response);
+    
+  } catch (error) {
+    console.error('❌ Error scheduling bulk read jobs:', error);
+    res.status(500).json(failure(error.message, 'BULK_READ_SCHEDULING_ERROR'));
+  }
+}
+
 module.exports = { 
   getData, getServices, getResources, getPortals, getAllProjects, getAllProjectTasks, 
   getAllProjectTaskLists, getAllProjectIssues, getAllProjectPhases, getSalesOrders, 
@@ -960,5 +1074,5 @@ module.exports = {
   getExpenses, getBankAccounts, getBankTransactions, getReports, getBooksSalesOrders, 
   getBooksPurchaseOrders, getContacts, getBooksContacts, getBooksInvoices, getEmployeeForms, getEmployeeById, 
   getAttendanceEntries, getShiftConfiguration, getLeaveData, getGoalsData, getPerformanceData,
-  getUserReport, getLeaveTrackerReport, getHolidays
+  getUserReport, getLeaveTrackerReport, getHolidays, scheduleBulkReadJobs
 };

src/api/routes/integrationRoutes.js

@@ -8,7 +8,7 @@ const {
   getExpenses, getBankAccounts, getBankTransactions, getReports, getBooksSalesOrders, 
   getBooksPurchaseOrders, getContacts, getBooksContacts, getBooksInvoices, getEmployeeForms, getEmployeeById, 
   getAttendanceEntries, getShiftConfiguration, getLeaveData, getGoalsData, getPerformanceData,
-  getUserReport, getLeaveTrackerReport, getHolidays 
+  getUserReport, getLeaveTrackerReport, getHolidays, scheduleBulkReadJobs
 } = require('../controllers/integrationController');
 const auth = require('../middlewares/auth');
 const ZohoHandler = require('../../integrations/zoho/handler');
@@ -352,6 +352,9 @@ router.get('/zoho/books/reports', auth, validate(reportsSchema), getReports);
 router.get('/zoho/books/sales-orders', auth, validate(booksSalesOrdersSchema), getBooksSalesOrders);
 router.get('/zoho/books/purchase-orders', auth, validate(booksPurchaseOrdersSchema), getBooksPurchaseOrders);
 
+// Bulk read job scheduling endpoint
+router.post('/zoho/bulk-read/schedule', auth, scheduleBulkReadJobs);
+
 // Webhook endpoints (no auth required - uses signature verification)
 const zohoHandler = new ZohoHandler();
 router.post('/webhooks/zoho/crm', zohoHandler.handleCrmWebhook.bind(zohoHandler));

src/data/models/zohoDealsBulk.js

@@ -0,0 +1,98 @@
+const { DataTypes } = require('sequelize');
+const sequelize = require('../../db/pool');
+
+const ZohoDealsBulk = sequelize.define('ZohoDealsBulk', {
+  internal_id: {
+    type: DataTypes.INTEGER,
+    primaryKey: true,
+    autoIncrement: true,
+    allowNull: false
+  },
+  zoho_id: {
+    type: DataTypes.STRING(255),
+    allowNull: true
+  },
+  user_uuid: {
+    type: DataTypes.CHAR(36),
+    allowNull: false
+  },
+  provider: {
+    type: DataTypes.STRING(50),
+    allowNull: false,
+    defaultValue: 'zoho'
+  },
+  deal_name: {
+    type: DataTypes.STRING(255),
+    allowNull: true
+  },
+  account_name: {
+    type: DataTypes.STRING(255),
+    allowNull: true
+  },
+  contact_name: {
+    type: DataTypes.STRING(255),
+    allowNull: true
+  },
+  amount: {
+    type: DataTypes.DECIMAL(15, 2),
+    allowNull: true
+  },
+  stage: {
+    type: DataTypes.STRING(255),
+    allowNull: true
+  },
+  closing_date: {
+    type: DataTypes.DATE,
+    allowNull: true
+  },
+  lead_source: {
+    type: DataTypes.STRING(255),
+    allowNull: true
+  },
+  type: {
+    type: DataTypes.STRING(255),
+    allowNull: true
+  },
+  probability: {
+    type: DataTypes.INTEGER,
+    allowNull: true
+  },
+  next_step: {
+    type: DataTypes.TEXT,
+    allowNull: true
+  },
+  description: {
+    type: DataTypes.TEXT,
+    allowNull: true
+  },
+  owner: {
+    type: DataTypes.STRING(255),
+    allowNull: true
+  },
+  created_time: {
+    type: DataTypes.DATE,
+    allowNull: true
+  },
+  modified_time: {
+    type: DataTypes.DATE,
+    allowNull: true
+  },
+  bulk_job_id: {
+    type: DataTypes.STRING(255),
+    allowNull: true
+  }
+}, {
+  tableName: 'zoho_deals_bulk',
+  timestamps: true,
+  createdAt: 'created_at',
+  updatedAt: 'updated_at',
+  indexes: [
+    { fields: ['user_uuid', 'provider'] },
+    { fields: ['bulk_job_id'] },
+    { fields: ['created_time'] },
+    { fields: ['stage'] },
+    { fields: ['closing_date'] }
+  ]
+});
+
+module.exports = ZohoDealsBulk;

src/data/repositories/zohoBulkReadRepository.js

@@ -6,6 +6,7 @@ const ZohoVendorsBulk = require('../models/zohoVendorsBulk');
 const ZohoInvoicesBulk = require('../models/zohoInvoicesBulk');
 const ZohoSalesOrdersBulk = require('../models/zohoSalesOrdersBulk');
 const ZohoPurchaseOrdersBulk = require('../models/zohoPurchaseOrdersBulk');
+const ZohoDealsBulk = require('../models/zohoDealsBulk');
 const ZohoBulkReadJobs = require('../models/zohoBulkReadJobs');
 
 class ZohoBulkReadRepository {
@@ -18,7 +19,8 @@ class ZohoBulkReadRepository {
       'vendors': ZohoVendorsBulk,
       'invoices': ZohoInvoicesBulk,
       'sales_orders': ZohoSalesOrdersBulk,
-      'purchase_orders': ZohoPurchaseOrdersBulk
+      'purchase_orders': ZohoPurchaseOrdersBulk,
+      'deals': ZohoDealsBulk
     };
   }
 

src/db/migrations/012_create_zoho_deals_bulk.sql

@@ -0,0 +1,33 @@
+-- Migration: Create Zoho Deals Bulk Table
+-- Description: Creates table for storing bulk read deals data from Zoho CRM
+
+CREATE TABLE IF NOT EXISTS zoho_deals_bulk (
+    internal_id INT AUTO_INCREMENT PRIMARY KEY,
+    zoho_id VARCHAR(255),
+    user_uuid CHAR(36) NOT NULL,
+    provider VARCHAR(50) NOT NULL DEFAULT 'zoho',
+    deal_name VARCHAR(255),
+    account_name VARCHAR(255),
+    contact_name VARCHAR(255),
+    amount DECIMAL(15, 2),
+    stage VARCHAR(255),
+    closing_date DATETIME,
+    lead_source VARCHAR(255),
+    type VARCHAR(255),
+    probability INT,
+    next_step TEXT,
+    description TEXT,
+    owner VARCHAR(255),
+    created_time DATETIME,
+    modified_time DATETIME,
+    bulk_job_id VARCHAR(255),
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+    INDEX idx_user_provider (user_uuid, provider),
+    CONSTRAINT fk_zoho_deals_bulk_user FOREIGN KEY (user_uuid) REFERENCES users(uuid) ON DELETE CASCADE,
+    INDEX idx_bulk_job (bulk_job_id),
+    INDEX idx_created_time (created_time),
+    INDEX idx_zoho_id (zoho_id),
+    INDEX idx_stage (stage),
+    INDEX idx_closing_date (closing_date)
+);

src/integrations/zoho/mapper.js

@@ -805,6 +805,27 @@ class ZohoMapper {
           }
           break;
 
+        case 'accounts':
+          // Handle both CRM and Books accounts - check response structure
+          if (zohoResponse.data && zohoResponse.info) {
+            // Books response structure
+            records = zohoResponse.data || [];
+            pageInfo = {
+              count: zohoResponse.info?.count || records.length,
+              moreRecords: zohoResponse.info?.more_records || false,
+              page: zohoResponse.info?.page || 1
+            };
+          } else {
+            // CRM response structure
+            records = zohoResponse.accounts || [];
+            pageInfo = {
+              count: zohoResponse.page_context?.count || records.length,
+              moreRecords: zohoResponse.page_context?.more_records || false,
+              page: zohoResponse.page_context?.page || 1
+            };
+          }
+          break;
+
         case 'contacts':
           // Books response structure for contacts
           if (zohoResponse.data && zohoResponse.info) {
@@ -825,6 +846,27 @@ class ZohoMapper {
         }
           break;
 
+        case 'deals':
+          // Handle both CRM and Books deals - check response structure
+          if (zohoResponse.data && zohoResponse.info) {
+            // Books response structure
+            records = zohoResponse.data || [];
+            pageInfo = {
+              count: zohoResponse.info?.count || records.length,
+              moreRecords: zohoResponse.info?.more_records || false,
+              page: zohoResponse.info?.page || 1
+            };
+          } else {
+            // CRM response structure
+            records = zohoResponse.deals || [];
+            pageInfo = {
+              count: zohoResponse.page_context?.count || records.length,
+              moreRecords: zohoResponse.page_context?.more_records || false,
+              page: zohoResponse.page_context?.page || 1
+            };
+          }
+          break;
+
         case 'vendors':
           // Books response structure for vendors (filtered contacts)
           records = zohoResponse.contacts || [];

src/services/bulkReadService.js

@@ -15,9 +15,10 @@ class BulkReadService {
    * @param {string} module - Zoho module name
    * @param {Array} fields - Fields to fetch
    * @param {Object} options - Additional options
+   * @param {string} accessToken - Frontend access token for callback URL
    * @returns {Promise<Object>} Job details
    */
-  async initiateBulkRead(userId, module, fields, options = {}) {
+  async initiateBulkRead(userId, module, fields, options = {}, accessToken = null) {
     try {
       console.log(`🚀 Initiating bulk read for user ${userId}, module ${module}`);
       
@@ -27,12 +28,20 @@ class BulkReadService {
         throw new Error('No Zoho access token found for user');
       }
       
-      const accessToken = decrypt(tokenRecord.accessToken);
+      const zohoAccessToken = decrypt(tokenRecord.accessToken);
       
       // Prepare bulk read request
+      const baseUrl = process.env.API_BASE_URL || process.env.MY_BASE_URL || 'http://localhost:3000';
+      let callbackUrl = `${baseUrl}/api/v1/integrations/webhooks/zoho/bulkread`;
+      
+      // Add access token to callback URL if provided
+      if (accessToken) {
+        callbackUrl += `?access_token=${accessToken}`;
+      }
+      
       const bulkReadData = {
         callback: {
-          url: `${process.env.API_BASE_URL || 'http://localhost:3000'}/api/v1/integrations/webhooks/zoho/bulkread`,
+          url: callbackUrl,
           method: 'post'
         },
         query: {
@@ -51,38 +60,55 @@ class BulkReadService {
         bulkReadData,
         {
           headers: {
-            'Authorization': `Zoho-oauthtoken ${accessToken}`,
+            'Authorization': `Zoho-oauthtoken ${zohoAccessToken}`,
             'Content-Type': 'application/json'
           }
         }
       );
       
       const jobData = response.data;
-      console.log('✅ Bulk read job initiated:', jobData);
+      console.log('✅ Bulk read job response from Zoho:', JSON.stringify(jobData, null, 2));
       
-      // Store job in database
+      // Extract job ID from nested response structure
+      const jobDetails = jobData.data && jobData.data[0] && jobData.data[0].details;
+      const jobId = jobDetails?.id;
+      
+      if (!jobId) {
+        console.log('⚠️ No job ID found in response structure');
+        console.log('Response structure:', JSON.stringify(jobData, null, 2));
+        throw new Error('Invalid response structure from Zoho API - no job ID found');
+      }
+      
+      console.log('📋 Job ID received from Zoho:', jobId);
+      console.log('📋 Job details:', jobDetails);
+      
+      // Store job in database with real job ID
       const jobRecord = await ZohoBulkReadRepository.createBulkReadJob({
-        id: jobData.id,
+        id: jobId,
         user_uuid: userId,
         module: module,
-        operation: 'read',
-        state: 'CREATED',
+        operation: jobDetails.operation || 'read',
+        state: jobDetails.state || 'ADDED',
         file_type: 'csv',
         records_count: 0,
         status: 'pending'
       });
       
-      logger.info('Bulk read job initiated', {
+      logger.info('Bulk read job created and stored', {
         userId,
         module,
-        jobId: jobData.id,
-        fields: fields.length
+        jobId: jobId,
+        zohoState: jobDetails.state || 'ADDED'
       });
       
       return {
-        jobId: jobData.id,
-        status: 'initiated',
-        message: `Bulk read job initiated for ${module}`,
+        jobId: jobId,
+        status: 'created',
+        message: `Bulk read job created for ${module}. Processing will be handled asynchronously via webhook.`,
+        zohoState: jobDetails.state || 'ADDED',
+        operation: jobDetails.operation || 'read',
+        createdBy: jobDetails.created_by,
+        createdTime: jobDetails.created_time,
         estimatedTime: this.getEstimatedTime(module, options.limit)
       };
       

src/services/csvService.js

@@ -163,6 +163,8 @@ class CsvService {
           return this.mapSalesOrderFields(record, baseRecord);
         case 'purchase_orders':
           return this.mapPurchaseOrderFields(record, baseRecord);
+        case 'deals':
+          return this.mapDealFields(record, baseRecord);
         default:
           console.warn(`⚠️ Unknown module: ${module}`);
           return { ...baseRecord, ...record };
@@ -297,6 +299,27 @@ class CsvService {
     };
   }
 
+  mapDealFields(record, baseRecord) {
+    return {
+      ...baseRecord,
+      zoho_id: record.id,
+      deal_name: record.Deal_Name,
+      account_name: record['Account_Name.Account_Name'] || record.Account_Name,
+      contact_name: record['Contact_Name.Contact_Name'] || record.Contact_Name,
+      amount: this.parseDecimal(record.Amount),
+      stage: record.Stage,
+      closing_date: this.parseDate(record.Closing_Date),
+      lead_source: record.Lead_Source,
+      type: record.Type,
+      probability: record.Probability ? parseInt(record.Probability) : null,
+      next_step: record.Next_Step,
+      description: record.Description,
+      owner: record.Owner,
+      created_time: this.parseDate(record.Created_Time),
+      modified_time: this.parseDate(record.Modified_Time)
+    };
+  }
+
   // Utility methods
   parseDate(dateString) {
     if (!dateString || dateString === '') return null;