backend changes

docker-compose.yml

@@ -674,6 +674,11 @@ services:
       - GITEA_WEBHOOK_SECRET=mywebhooksecret2025
       - PUBLIC_BASE_URL=https://a1247f5c9f93.ngrok-free.app
       - GITHUB_WEBHOOK_SECRET=mywebhooksecret2025
+      # Additional environment variables for git-integration service
+      - ENABLE_BACKGROUND_DIFF_PROCESSING=true
+      - DIFF_PROCESSING_INTERVAL_MS=30000
+      - DIFF_STORAGE_PATH=/app/git-repos/diffs
+      - MAX_DIFF_SIZE_BYTES=10485760
     volumes:
       - /home/tech4biz/Desktop/Projectsnew/CODENUK1/git-repos:/app/git-repos
     networks:

services/git-integration/Dockerfile

@@ -19,9 +19,17 @@ RUN addgroup -g 1001 -S nodejs
 RUN adduser -S git-integration -u 1001
 
 # Create git-repos directory and set proper permissions
-RUN mkdir -p /app/git-repos
+RUN mkdir -p /app/git-repos /app/git-repos/diffs
 RUN chown -R git-integration:nodejs /app
 RUN chmod -R 755 /app/git-repos
+
+# Create entrypoint script to handle volume permissions
+RUN echo '#!/bin/sh' > /app/entrypoint.sh && \
+    echo 'mkdir -p /app/git-repos/diffs' >> /app/entrypoint.sh && \
+    echo 'chmod -R 755 /app/git-repos' >> /app/entrypoint.sh && \
+    echo 'exec "$@"' >> /app/entrypoint.sh && \
+    chmod +x /app/entrypoint.sh
+
 USER git-integration
 
 # Expose port
@@ -32,4 +40,5 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
   CMD curl -f http://localhost:8012/health || exit 1
 
 # Start the application
+ENTRYPOINT ["/app/entrypoint.sh"]
 CMD ["npm", "start"]

services/git-integration/src/app.js

@@ -1,4 +1,3 @@
-require('dotenv').config();
 const express = require('express');
 const cors = require('cors');
 const helmet = require('helmet');

services/git-integration/src/migrations/001_github_integration.sql

@@ -2,12 +2,13 @@
 -- This migration adds support for GitHub repository integration
 
 -- Create table for GitHub repositories
-CREATE TABLE IF NOT EXISTS "github_repositories@migrations/" (
+CREATE TABLE IF NOT EXISTS all_repositories (
   id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
   template_id UUID REFERENCES templates(id) ON DELETE CASCADE,
   repository_url VARCHAR(500) NOT NULL,
   repository_name VARCHAR(200) NOT NULL,
   owner_name VARCHAR(100) NOT NULL,
+  provider_name VARCHAR(50) DEFAULT 'github' NOT NULL,
   branch_name VARCHAR(100) DEFAULT 'main',
   is_public BOOLEAN DEFAULT true,
   requires_auth BOOLEAN DEFAULT false,
@@ -21,13 +22,14 @@ CREATE TABLE IF NOT EXISTS "github_repositories@migrations/" (
 
 
 -- Create indexes for better performance
-CREATE INDEX IF NOT EXISTS idx_github_repos_template_id ON "github_repositories@migrations/"(template_id);
+CREATE INDEX IF NOT EXISTS idx_github_repos_template_id ON all_repositories(template_id);
-CREATE INDEX IF NOT EXISTS idx_github_repos_owner_name ON "github_repositories@migrations/"(owner_name);
+CREATE INDEX IF NOT EXISTS idx_github_repos_owner_name ON all_repositories(owner_name);
+CREATE INDEX IF NOT EXISTS idx_all_repos_provider_name ON all_repositories(provider_name);
 CREATE INDEX IF NOT EXISTS idx_feature_mappings_feature_id ON feature_codebase_mappings(feature_id);
 CREATE INDEX IF NOT EXISTS idx_feature_mappings_repo_id ON feature_codebase_mappings(repository_id);
 
 -- Add trigger to update timestamp
-CREATE TRIGGER update_github_repos_updated_at BEFORE UPDATE ON "github_repositories@migrations/"
+CREATE TRIGGER update_github_repos_updated_at BEFORE UPDATE ON all_repositories
     FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
 
 -- =============================================

services/git-integration/src/migrations/002_repository_file_storage.sql

@@ -4,7 +4,7 @@
 -- Create table for repository local storage tracking
 CREATE TABLE IF NOT EXISTS repository_storage (
   id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id) ON DELETE CASCADE,
+  repository_id UUID REFERENCES all_repositories(id) ON DELETE CASCADE,
   local_path TEXT NOT NULL,
   storage_status VARCHAR(50) DEFAULT 'pending', -- pending, downloading, completed, error
   total_files_count INTEGER DEFAULT 0,
@@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS repository_storage (
 -- Create table for directory structure
 CREATE TABLE IF NOT EXISTS repository_directories (
   id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id) ON DELETE CASCADE,
+  repository_id UUID REFERENCES all_repositories(id) ON DELETE CASCADE,
   storage_id UUID REFERENCES repository_storage(id) ON DELETE CASCADE,
   parent_directory_id UUID REFERENCES repository_directories(id) ON DELETE CASCADE,
   directory_name VARCHAR(255) NOT NULL,
@@ -38,7 +38,7 @@ CREATE TABLE IF NOT EXISTS repository_directories (
 -- Create table for individual files
 CREATE TABLE IF NOT EXISTS repository_files (
   id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id) ON DELETE CASCADE,
+  repository_id UUID REFERENCES all_repositories(id) ON DELETE CASCADE,
   storage_id UUID REFERENCES repository_storage(id) ON DELETE CASCADE,
   directory_id UUID REFERENCES repository_directories(id) ON DELETE SET NULL,
   filename VARCHAR(255) NOT NULL,

services/git-integration/src/migrations/003_add_user_id_to_template_refs.sql

@@ -2,12 +2,12 @@
 -- This ensures we always track which user owns/initiated records tied to a template
 
 -- Add user_id to github_repositories
-ALTER TABLE IF EXISTS "github_repositories@migrations/"
+ALTER TABLE IF EXISTS all_repositories
   ADD COLUMN IF NOT EXISTS user_id UUID REFERENCES users(id) ON DELETE CASCADE;
 
 -- Indexes for github_repositories
-CREATE INDEX IF NOT EXISTS idx_github_repos_user_id ON "github_repositories@migrations/"(user_id);
+CREATE INDEX IF NOT EXISTS idx_github_repos_user_id ON all_repositories(user_id);
-CREATE INDEX IF NOT EXISTS idx_github_repos_template_user ON "github_repositories@migrations/"(template_id, user_id);
+CREATE INDEX IF NOT EXISTS idx_github_repos_template_user ON all_repositories(template_id, user_id);
 
 -- Add user_id to feature_codebase_mappings
 ALTER TABLE IF EXISTS feature_codebase_mappings

services/git-integration/src/migrations/005_webhook_commits.sql

@@ -8,7 +8,7 @@ CREATE TABLE IF NOT EXISTS github_webhooks (
   action VARCHAR(100),
   owner_name VARCHAR(120),
   repository_name VARCHAR(200),
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id) ON DELETE SET NULL,
+  repository_id UUID REFERENCES all_repositories(id) ON DELETE SET NULL,
   ref VARCHAR(255),
   before_sha VARCHAR(64),
   after_sha VARCHAR(64),
@@ -26,7 +26,7 @@ CREATE INDEX IF NOT EXISTS idx_github_webhooks_event_type ON github_webhooks(eve
 -- Track commit SHA transitions per repository to detect changes over time
 CREATE TABLE IF NOT EXISTS repository_commit_events (
   id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id) ON DELETE CASCADE,
+  repository_id UUID REFERENCES all_repositories(id) ON DELETE CASCADE,
   ref VARCHAR(255),
   before_sha VARCHAR(64),
   after_sha VARCHAR(64),

services/git-integration/src/migrations/006_commit_changes.sql

@@ -3,7 +3,7 @@
 -- Per-commit details linked to an attached repository
 CREATE TABLE IF NOT EXISTS repository_commit_details (
   id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id) ON DELETE CASCADE,
+  repository_id UUID REFERENCES all_repositories(id) ON DELETE CASCADE,
   commit_sha VARCHAR(64) NOT NULL,
   author_name VARCHAR(200),
   author_email VARCHAR(320),

services/git-integration/src/migrations/007_add_last_synced_commit.sql

@@ -1,5 +1,5 @@
 -- 007_add_last_synced_commit.sql
-ALTER TABLE "github_repositories@migrations/"
+ALTER TABLE all_repositories
 ADD COLUMN IF NOT EXISTS last_synced_commit_sha VARCHAR(64),
 ADD COLUMN IF NOT EXISTS last_synced_at TIMESTAMP WITH TIME ZONE;
 

services/git-integration/src/migrations/009_provider_webhook_tables.sql

@@ -8,7 +8,7 @@ CREATE TABLE IF NOT EXISTS gitlab_webhooks (
   action TEXT,
   owner_name TEXT NOT NULL,
   repository_name TEXT NOT NULL,
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id),
+  repository_id UUID REFERENCES all_repositories(id),
   ref TEXT,
   before_sha TEXT,
   after_sha TEXT,
@@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS bitbucket_webhooks (
   action TEXT,
   owner_name TEXT NOT NULL,
   repository_name TEXT NOT NULL,
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id),
+  repository_id UUID REFERENCES all_repositories(id),
   ref TEXT,
   before_sha TEXT,
   after_sha TEXT,
@@ -44,7 +44,7 @@ CREATE TABLE IF NOT EXISTS gitea_webhooks (
   action TEXT,
   owner_name TEXT NOT NULL,
   repository_name TEXT NOT NULL,
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id),
+  repository_id UUID REFERENCES all_repositories(id),
   ref TEXT,
   before_sha TEXT,
   after_sha TEXT,

services/git-integration/src/migrations/010_remove_template_id.sql

@@ -7,7 +7,7 @@ DROP INDEX IF EXISTS idx_github_repos_template_user;
 DROP INDEX IF EXISTS idx_feature_mappings_template_user;
 
 -- Remove template_id column from github_repositories table
-ALTER TABLE IF EXISTS "github_repositories@migrations/" 
+ALTER TABLE IF EXISTS all_repositories 
   DROP COLUMN IF EXISTS template_id;
 
 -- Remove template_id column from feature_codebase_mappings table

services/git-integration/src/migrations/012_add_user_id_to_github_repositories.sql

@@ -1,10 +1,10 @@
 -- Migration 012: Track which user attached/downloaded a repository
 
 -- Add user_id to github_repositories to associate records with the initiating user
-ALTER TABLE "github_repositories@migrations/"
+ALTER TABLE all_repositories
 ADD COLUMN IF NOT EXISTS user_id UUID REFERENCES users(id) ON DELETE SET NULL;
 
 -- Helpful index for filtering user-owned repositories
-CREATE INDEX IF NOT EXISTS idx_github_repositories_user_id ON "github_repositories@migrations/"(user_id);
+CREATE INDEX IF NOT EXISTS idx_github_repositories_user_id ON all_repositories(user_id);
 
 

services/git-integration/src/migrations/013_repository_commit_details.sql

@@ -4,7 +4,7 @@
 -- Per-commit details linked to an attached repository
 CREATE TABLE IF NOT EXISTS repository_commit_details (
   id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id) ON DELETE CASCADE,
+  repository_id UUID REFERENCES all_repositories(id) ON DELETE CASCADE,
   commit_sha VARCHAR(64) NOT NULL,
   author_name VARCHAR(200),
   author_email VARCHAR(320),

services/git-integration/src/migrations/015_diff_storage_system.sql

@@ -34,7 +34,7 @@ CREATE TABLE IF NOT EXISTS diff_contents (
 CREATE TABLE IF NOT EXISTS diff_processing_queue (
   id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
   commit_id UUID REFERENCES repository_commit_details(id) ON DELETE CASCADE,
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id) ON DELETE CASCADE,
+  repository_id UUID REFERENCES all_repositories(id) ON DELETE CASCADE,
   
   -- Processing metadata
   queue_status VARCHAR(20) DEFAULT 'pending', -- 'pending', 'processing', 'completed', 'failed'
@@ -58,7 +58,7 @@ CREATE TABLE IF NOT EXISTS diff_processing_queue (
 -- Diff Statistics (for monitoring and optimization)
 CREATE TABLE IF NOT EXISTS diff_statistics (
   id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id) ON DELETE CASCADE,
+  repository_id UUID REFERENCES all_repositories(id) ON DELETE CASCADE,
   
   -- Statistics period
   period_start TIMESTAMP NOT NULL,

services/git-integration/src/migrations/016_missing_columns_and_indexes.sql

@@ -2,7 +2,7 @@
 -- This migration adds missing columns and indexes from the provided migrations
 
 -- Add missing column to github_repositories if it doesn't exist
-ALTER TABLE "github_repositories@migrations/" 
+ALTER TABLE all_repositories 
 ADD COLUMN IF NOT EXISTS last_synced_commit_sha VARCHAR(64);
 
 -- Add missing ID column to repository_files if it doesn't exist

services/git-integration/src/migrations/017_complete_schema_from_provided_migrations.sql

@@ -6,12 +6,13 @@
 -- =============================================
 
 -- Create table for GitHub repositories (enhanced version from provided migration)
-CREATE TABLE IF NOT EXISTS "github_repositories@migrations/" (
+CREATE TABLE IF NOT EXISTS all_repositories (
   id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
   template_id UUID, -- References templates(id) but table may not exist
   repository_url VARCHAR(500) NOT NULL,
   repository_name VARCHAR(200) NOT NULL,
   owner_name VARCHAR(100) NOT NULL,
+  provider_name VARCHAR(50) DEFAULT 'github' NOT NULL,
   branch_name VARCHAR(100) DEFAULT 'main',
   is_public BOOLEAN DEFAULT true,
   requires_auth BOOLEAN DEFAULT false,
@@ -31,7 +32,7 @@ CREATE TABLE IF NOT EXISTS "github_repositories@migrations/" (
 -- Create table for repository local storage tracking
 CREATE TABLE IF NOT EXISTS repository_storage (
   id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id) ON DELETE CASCADE,
+  repository_id UUID REFERENCES all_repositories(id) ON DELETE CASCADE,
   local_path TEXT NOT NULL,
   storage_status VARCHAR(50) DEFAULT 'pending', -- pending, downloading, completed, error
   total_files_count INTEGER DEFAULT 0,
@@ -48,7 +49,7 @@ CREATE TABLE IF NOT EXISTS repository_storage (
 -- Create table for directory structure
 CREATE TABLE IF NOT EXISTS repository_directories (
   id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id) ON DELETE CASCADE,
+  repository_id UUID REFERENCES all_repositories(id) ON DELETE CASCADE,
   storage_id UUID REFERENCES repository_storage(id) ON DELETE CASCADE,
   parent_directory_id UUID REFERENCES repository_directories(id) ON DELETE CASCADE,
   directory_name VARCHAR(255) NOT NULL,
@@ -65,7 +66,7 @@ CREATE TABLE IF NOT EXISTS repository_directories (
 -- Create table for individual files
 CREATE TABLE IF NOT EXISTS repository_files (
   id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id) ON DELETE CASCADE,
+  repository_id UUID REFERENCES all_repositories(id) ON DELETE CASCADE,
   storage_id UUID REFERENCES repository_storage(id) ON DELETE CASCADE,
   directory_id UUID REFERENCES repository_directories(id) ON DELETE SET NULL,
   filename VARCHAR(255) NOT NULL,
@@ -94,7 +95,7 @@ CREATE TABLE IF NOT EXISTS github_webhooks (
   action VARCHAR(100),
   owner_name VARCHAR(120),
   repository_name VARCHAR(200),
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id) ON DELETE SET NULL,
+  repository_id UUID REFERENCES all_repositories(id) ON DELETE SET NULL,
   ref VARCHAR(255),
   before_sha VARCHAR(64),
   after_sha VARCHAR(64),
@@ -113,7 +114,7 @@ CREATE TABLE IF NOT EXISTS gitlab_webhooks (
   action TEXT,
   owner_name TEXT NOT NULL,
   repository_name TEXT NOT NULL,
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id),
+  repository_id UUID REFERENCES all_repositories(id),
   ref TEXT,
   before_sha TEXT,
   after_sha TEXT,
@@ -131,7 +132,7 @@ CREATE TABLE IF NOT EXISTS bitbucket_webhooks (
   action TEXT,
   owner_name TEXT NOT NULL,
   repository_name TEXT NOT NULL,
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id),
+  repository_id UUID REFERENCES all_repositories(id),
   ref TEXT,
   before_sha TEXT,
   after_sha TEXT,
@@ -149,7 +150,7 @@ CREATE TABLE IF NOT EXISTS gitea_webhooks (
   action TEXT,
   owner_name TEXT NOT NULL,
   repository_name TEXT NOT NULL,
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id),
+  repository_id UUID REFERENCES all_repositories(id),
   ref TEXT,
   before_sha TEXT,
   after_sha TEXT,
@@ -166,7 +167,7 @@ CREATE TABLE IF NOT EXISTS gitea_webhooks (
 -- Per-commit details linked to an attached repository
 CREATE TABLE IF NOT EXISTS repository_commit_details (
   id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id) ON DELETE CASCADE,
+  repository_id UUID REFERENCES all_repositories(id) ON DELETE CASCADE,
   commit_sha VARCHAR(64) NOT NULL,
   author_name VARCHAR(200),
   author_email VARCHAR(320),
@@ -274,7 +275,7 @@ CREATE TABLE IF NOT EXISTS diff_contents (
 CREATE TABLE IF NOT EXISTS diff_processing_queue (
   id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
   commit_id UUID REFERENCES repository_commit_details(id) ON DELETE CASCADE,
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id) ON DELETE CASCADE,
+  repository_id UUID REFERENCES all_repositories(id) ON DELETE CASCADE,
   
   -- Processing metadata
   queue_status VARCHAR(20) DEFAULT 'pending', -- 'pending', 'processing', 'completed', 'failed'
@@ -298,7 +299,7 @@ CREATE TABLE IF NOT EXISTS diff_processing_queue (
 -- Diff Statistics (for monitoring and optimization)
 CREATE TABLE IF NOT EXISTS diff_statistics (
   id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
-  repository_id UUID REFERENCES "github_repositories@migrations/"(id) ON DELETE CASCADE,
+  repository_id UUID REFERENCES all_repositories(id) ON DELETE CASCADE,
   
   -- Statistics period
   period_start TIMESTAMP NOT NULL,
@@ -329,8 +330,9 @@ CREATE TABLE IF NOT EXISTS diff_statistics (
 -- =============================================
 
 -- GitHub repositories indexes
-CREATE INDEX IF NOT EXISTS idx_github_repos_template_id ON "github_repositories@migrations/"(template_id);
+CREATE INDEX IF NOT EXISTS idx_github_repos_template_id ON all_repositories(template_id);
-CREATE INDEX IF NOT EXISTS idx_github_repos_owner_name ON "github_repositories@migrations/"(owner_name);
+CREATE INDEX IF NOT EXISTS idx_github_repos_owner_name ON all_repositories(owner_name);
+CREATE INDEX IF NOT EXISTS idx_all_repos_provider_name ON all_repositories(provider_name);
 
 -- Repository storage indexes
 CREATE INDEX IF NOT EXISTS idx_repository_storage_repo_id ON repository_storage(repository_id);
@@ -421,7 +423,7 @@ BEGIN
   IF NOT EXISTS (
     SELECT 1 FROM pg_trigger WHERE tgname = 'update_github_repos_updated_at'
   ) THEN
-    CREATE TRIGGER update_github_repos_updated_at BEFORE UPDATE ON "github_repositories@migrations/"
+    CREATE TRIGGER update_github_repos_updated_at BEFORE UPDATE ON all_repositories
       FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
   END IF;
 END $$;

services/git-integration/src/migrations/018_add_provider_name_column.sql

@@ -0,0 +1,17 @@
+-- Migration 018: Add provider_name column to all_repositories table
+-- This migration adds support for multiple repository providers (GitHub, GitLab, Bitbucket, etc.)
+
+-- Add provider_name column to all_repositories table
+ALTER TABLE all_repositories 
+ADD COLUMN IF NOT EXISTS provider_name VARCHAR(50) DEFAULT 'github' NOT NULL;
+
+-- Create index for provider_name for better query performance
+CREATE INDEX IF NOT EXISTS idx_all_repos_provider_name ON all_repositories(provider_name);
+
+-- Add comment to document the column purpose
+COMMENT ON COLUMN all_repositories.provider_name IS 'Repository provider (github, gitlab, bitbucket, etc.)';
+
+-- Update existing records to have 'github' as provider_name (if any exist without it)
+UPDATE all_repositories 
+SET provider_name = 'github' 
+WHERE provider_name IS NULL OR provider_name = '';