prakash/codenuk_backend_mine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

backend changes

Browse Source
This commit is contained in:
Chandini 2025-09-04 15:29:50 +05:30
parent 64ef1718fb
commit 56571c445e
16 changed files with 3993 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
docker-compose.yml
View File

@ -594,6 +594,7 @@ services:
# Service URLs # Service URLs
- USER_AUTH_URL=http://user-auth:8011 - USER_AUTH_URL=http://user-auth:8011
- TEMPLATE_MANAGER_URL=http://template-manager:8009 - TEMPLATE_MANAGER_URL=http://template-manager:8009
- GIT_INTEGRATION_URL=http://git-integration:8012
- REQUIREMENT_PROCESSOR_URL=http://requirement-processor:8001 - REQUIREMENT_PROCESSOR_URL=http://requirement-processor:8001
- TECH_STACK_SELECTOR_URL=http://tech-stack-selector:8002 - TECH_STACK_SELECTOR_URL=http://tech-stack-selector:8002
- ARCHITECTURE_DESIGNER_URL=http://architecture-designer:8003 - ARCHITECTURE_DESIGNER_URL=http://architecture-designer:8003
@ -905,7 +906,51 @@ services:
interval: 30s interval: 30s
timeout: 10s timeout: 10s
retries: 3 retries: 3
start_period: 40s start_period: 40s
restart: unless-stopped
git-integration:
build: ./services/git-integration
container_name: pipeline_git_integration
ports:
- "8012:8012"
env_file:
- ./services/git-integration/.env
environment:
- PORT=8012
- HOST=0.0.0.0
- FRONTEND_URL=http://localhost:3000
- POSTGRES_HOST=postgres
- POSTGRES_PORT=5432
- POSTGRES_DB=dev_pipeline
- POSTGRES_USER=pipeline_admin
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- REDIS_HOST=redis
- REDIS_PORT=6379
- REDIS_PASSWORD=${REDIS_PASSWORD}
- NODE_ENV=development
- GITHUB_REDIRECT_URI=http://localhost:8012/api/github/auth/github/callback
- ATTACHED_REPOS_DIR=/tmp/attached-repos
- SESSION_SECRET=git-integration-secret-key-2024
volumes:
- git_repos_data:/tmp/attached-repos
networks:
- pipeline_network
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
migrations:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8012/health"]
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
restart: unless-stopped
self-improving-generator: self-improving-generator:
build: ./self-improving-generator build: ./self-improving-generator
container_name: pipeline_self_improving_generator container_name: pipeline_self_improving_generator
@ -1041,6 +1086,8 @@ volumes:
driver: local driver: local
rabbitmq_logs: rabbitmq_logs:
driver: local driver: local
git_repos_data:
driver: local
n8n_data: n8n_data:
driver: local driver: local
neo4j_data: neo4j_data:

87
services/api-gateway/src/server.js
View File

@ -42,6 +42,7 @@ global.io = io;
const serviceTargets = { const serviceTargets = {
USER_AUTH_URL: process.env.USER_AUTH_URL || 'http://localhost:8011', USER_AUTH_URL: process.env.USER_AUTH_URL || 'http://localhost:8011',
TEMPLATE_MANAGER_URL: process.env.TEMPLATE_MANAGER_URL || 'http://localhost:8009', TEMPLATE_MANAGER_URL: process.env.TEMPLATE_MANAGER_URL || 'http://localhost:8009',
GIT_INTEGRATION_URL: process.env.GIT_INTEGRATION_URL || 'http://localhost:8012',
REQUIREMENT_PROCESSOR_URL: process.env.REQUIREMENT_PROCESSOR_URL || 'http://localhost:8001', REQUIREMENT_PROCESSOR_URL: process.env.REQUIREMENT_PROCESSOR_URL || 'http://localhost:8001',
TECH_STACK_SELECTOR_URL: process.env.TECH_STACK_SELECTOR_URL || 'http://localhost:8002', TECH_STACK_SELECTOR_URL: process.env.TECH_STACK_SELECTOR_URL || 'http://localhost:8002',
ARCHITECTURE_DESIGNER_URL: process.env.ARCHITECTURE_DESIGNER_URL || 'http://localhost:8003', ARCHITECTURE_DESIGNER_URL: process.env.ARCHITECTURE_DESIGNER_URL || 'http://localhost:8003',
@ -93,6 +94,7 @@ app.use('/api/gateway', express.json({ limit: '10mb' }));
app.use('/api/auth', express.json({ limit: '10mb' })); app.use('/api/auth', express.json({ limit: '10mb' }));
app.use('/api/templates', express.json({ limit: '10mb' })); app.use('/api/templates', express.json({ limit: '10mb' }));
app.use('/api/features', express.json({ limit: '10mb' })); app.use('/api/features', express.json({ limit: '10mb' }));
app.use('/api/github', express.json({ limit: '10mb' }));
app.use('/health', express.json({ limit: '10mb' })); app.use('/health', express.json({ limit: '10mb' }));
// Trust proxy for accurate IP addresses // Trust proxy for accurate IP addresses
@ -136,6 +138,7 @@ app.get('/health', (req, res) => {
services: { services: {
user_auth: process.env.USER_AUTH_URL ? 'configured' : 'not configured', user_auth: process.env.USER_AUTH_URL ? 'configured' : 'not configured',
template_manager: process.env.TEMPLATE_MANAGER_URL ? 'configured' : 'not configured', template_manager: process.env.TEMPLATE_MANAGER_URL ? 'configured' : 'not configured',
git_integration: process.env.GIT_INTEGRATION_URL ? 'configured' : 'not configured',
requirement_processor: process.env.REQUIREMENT_PROCESSOR_URL ? 'configured' : 'not configured', requirement_processor: process.env.REQUIREMENT_PROCESSOR_URL ? 'configured' : 'not configured',
tech_stack_selector: process.env.TECH_STACK_SELECTOR_URL ? 'configured' : 'not configured', tech_stack_selector: process.env.TECH_STACK_SELECTOR_URL ? 'configured' : 'not configured',
architecture_designer: process.env.ARCHITECTURE_DESIGNER_URL ? 'configured' : 'not configured', architecture_designer: process.env.ARCHITECTURE_DESIGNER_URL ? 'configured' : 'not configured',
@ -416,6 +419,88 @@ app.use('/api/features',
} }
); );
// Git Integration Service - Direct HTTP forwarding
console.log('🔧 Registering /api/github proxy route...');
app.use('/api/github',
createServiceLimiter(200),
// Conditionally require auth: allow public GETs, require token for write ops
(req, res, next) => {
// Allow unauthenticated access for read-only requests and specific public endpoints
if (req.method === 'GET') {
return next();
}
// Allowlist certain POST endpoints that must be public to initiate flows
const url = req.originalUrl || '';
const isPublicGithubEndpoint = (
url.startsWith('/api/github/test-access') ||
url.startsWith('/api/github/auth/github') ||
url.startsWith('/api/github/auth/github/callback') ||
url.startsWith('/api/github/auth/github/status')
);
if (isPublicGithubEndpoint) {
return next();
}
return authMiddleware.verifyToken(req, res, () => authMiddleware.forwardUserContext(req, res, next));
},
(req, res, next) => {
const gitServiceUrl = serviceTargets.GIT_INTEGRATION_URL;
console.log(`🔥 [GIT PROXY] ${req.method} ${req.originalUrl} → ${gitServiceUrl}${req.originalUrl}`);
// Set response timeout to prevent hanging
res.setTimeout(15000, () => {
console.error('❌ [GIT PROXY] Response timeout');
if (!res.headersSent) {
res.status(504).json({ error: 'Gateway timeout', service: 'git-integration' });
}
});
const options = {
method: req.method,
url: `${gitServiceUrl}${req.originalUrl}`,
headers: {
'Content-Type': 'application/json',
'User-Agent': 'API-Gateway/1.0',
'Connection': 'keep-alive',
// Forward user context from auth middleware
'X-User-ID': req.user?.id || req.user?.userId,
'X-User-Role': req.user?.role,
'Authorization': req.headers.authorization
},
timeout: 8000,
validateStatus: () => true,
maxRedirects: 0
};
// Always include request body for POST/PUT/PATCH requests
if (req.method === 'POST' || req.method === 'PUT' || req.method === 'PATCH') {
options.data = req.body || {};
console.log(`📦 [GIT PROXY] Request body:`, JSON.stringify(req.body));
}
axios(options)
.then(response => {
console.log(`✅ [GIT PROXY] Response: ${response.status} for ${req.method} ${req.originalUrl}`);
if (!res.headersSent) {
res.status(response.status).json(response.data);
}
})
.catch(error => {
console.error(`❌ [GIT PROXY ERROR]:`, error.message);
if (!res.headersSent) {
if (error.response) {
res.status(error.response.status).json(error.response.data);
} else {
res.status(502).json({
error: 'Git integration service unavailable',
message: error.code || error.message,
service: 'git-integration'
});
}
}
});
}
);
// Gateway management endpoints // Gateway management endpoints
app.get('/api/gateway/info', authMiddleware.verifyToken, (req, res) => { app.get('/api/gateway/info', authMiddleware.verifyToken, (req, res) => {
res.json({ res.json({
@ -464,6 +549,7 @@ app.get('/', (req, res) => {
services: { services: {
auth: '/api/auth', auth: '/api/auth',
templates: '/api/templates', templates: '/api/templates',
github: '/api/github',
requirements: '/api/requirements', requirements: '/api/requirements',
tech_stack: '/api/tech-stack', tech_stack: '/api/tech-stack',
architecture: '/api/architecture', architecture: '/api/architecture',
@ -488,6 +574,7 @@ app.use('*', (req, res) => {
available_services: { available_services: {
auth: '/api/auth', auth: '/api/auth',
templates: '/api/templates', templates: '/api/templates',
github: '/api/github',
requirements: '/api/requirements', requirements: '/api/requirements',
tech_stack: '/api/tech-stack', tech_stack: '/api/tech-stack',
architecture: '/api/architecture', architecture: '/api/architecture',

30
services/git-integration/Dockerfile Normal file
View File

@ -0,0 +1,30 @@
FROM node:18-alpine
WORKDIR /app
# Copy package files
COPY package*.json ./
# Install dependencies
RUN npm install
# Copy source code
COPY . .
# Create non-root user
RUN addgroup -g 1001 -S nodejs
RUN adduser -S git-integration -u 1001
# Change ownership
RUN chown -R git-integration:nodejs /app
USER git-integration
# Expose port
EXPOSE 8012
# Health check
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
CMD curl -f http://localhost:8012/health || exit 1
# Start the application
CMD ["npm", "start"]

1716
services/git-integration/package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

25
services/git-integration/package.json Normal file
View File

@ -0,0 +1,25 @@
{
"name": "git-integration",
"version": "1.0.0",
"description": "Git Integration Service with GitHub Integration",
"main": "src/app.js",
"scripts": {
"start": "node src/app.js",
"dev": "nodemon src/app.js",
"migrate": "node src/migrations/migrate.js"
},
"dependencies": {
"@octokit/rest": "^20.0.2",
"cors": "^2.8.5",
"dotenv": "^16.3.1",
"express": "^4.18.2",
"express-session": "^1.18.2",
"helmet": "^7.1.0",
"morgan": "^1.10.0",
"pg": "^8.11.3",
"uuid": "^9.0.1"
},
"devDependencies": {
"nodemon": "^3.0.2"
}
}

103
services/git-integration/src/app.js Normal file
View File

@ -0,0 +1,103 @@
require('dotenv').config();
const express = require('express');
const cors = require('cors');
const helmet = require('helmet');
const session = require('express-session');
const morgan = require('morgan');
// Import database
const database = require('./config/database');
// Import routes
const githubRoutes = require('./routes/github-integration.routes');
const githubOAuthRoutes = require('./routes/github-oauth');
const app = express();
const PORT = process.env.PORT || 8012;
// Middleware
app.use(helmet());
app.use(cors());
app.use(morgan('combined'));
app.use(express.json({ limit: '10mb' }));
app.use(express.urlencoded({ extended: true }));
// Session middleware
app.use(session({
secret: process.env.SESSION_SECRET || 'git-integration-secret-key-2024',
resave: false,
saveUninitialized: false,
cookie: {
secure: false, // Set to true if using HTTPS
httpOnly: true,
maxAge: 24 * 60 * 60 * 1000 // 24 hours
}
}));
// Routes
app.use('/api/github', githubRoutes);
app.use('/api/github', githubOAuthRoutes);
// Health check endpoint
app.get('/health', (req, res) => {
res.status(200).json({
status: 'healthy',
service: 'git-integration',
timestamp: new Date().toISOString(),
uptime: process.uptime(),
version: '1.0.0'
});
});
// Root endpoint
app.get('/', (req, res) => {
res.json({
message: 'Git Integration Service',
version: '1.0.0',
endpoints: {
health: '/health',
github: '/api/github',
oauth: '/api/github/auth'
}
});
});
// Error handling middleware
app.use((err, req, res, next) => {
console.error('Error:', err);
res.status(500).json({
success: false,
message: 'Internal server error',
error: process.env.NODE_ENV === 'development' ? err.message : 'Something went wrong'
});
});
// 404 handler
app.use('*', (req, res) => {
res.status(404).json({
success: false,
message: 'Endpoint not found'
});
});
// Graceful shutdown
process.on('SIGTERM', async () => {
console.log('SIGTERM received, shutting down gracefully');
await database.close();
process.exit(0);
});
process.on('SIGINT', async () => {
console.log('SIGINT received, shutting down gracefully');
await database.close();
process.exit(0);
});
// Start server
app.listen(PORT, '0.0.0.0', () => {
console.log(`🚀 Git Integration Service running on port ${PORT}`);
console.log(`📊 Health check: http://localhost:${PORT}/health`);
console.log(`🔗 GitHub API: http://localhost:${PORT}/api/github`);
});
module.exports = app;

54
services/git-integration/src/config/database.js Normal file
View File

@ -0,0 +1,54 @@
const { Pool } = require('pg');
class Database {
constructor() {
this.pool = new Pool({
host: process.env.POSTGRES_HOST || 'localhost',
port: process.env.POSTGRES_PORT || 5432,
database: process.env.POSTGRES_DB || 'dev_pipeline',
user: process.env.POSTGRES_USER || 'pipeline_admin',
password: process.env.POSTGRES_PASSWORD || 'secure_pipeline_2024',
max: 20,
idleTimeoutMillis: 30000,
connectionTimeoutMillis: 2000,
});
// Test connection on startup
this.testConnection();
}
async testConnection() {
try {
const client = await this.pool.connect();
console.log('✅ Database connected successfully');
client.release();
} catch (err) {
console.error('❌ Database connection failed:', err.message);
process.exit(1);
}
}
async query(text, params) {
const start = Date.now();
try {
const res = await this.pool.query(text, params);
const duration = Date.now() - start;
console.log('📊 Query executed:', { text: text.substring(0, 50), duration, rows: res.rowCount });
return res;
} catch (err) {
console.error('❌ Query error:', err.message);
throw err;
}
}
async getClient() {
return await this.pool.connect();
}
async close() {
await this.pool.end();
console.log('🔌 Database connection closed');
}
}
module.exports = new Database();

61
services/git-integration/src/migrations/001_github_integration.sql Normal file
View File

@ -0,0 +1,61 @@
-- Migration 001: Add GitHub Integration Tables (PostgreSQL Only)
-- This migration adds support for GitHub repository integration
-- Create table for GitHub repositories
CREATE TABLE IF NOT EXISTS github_repositories (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
template_id UUID REFERENCES templates(id) ON DELETE CASCADE,
repository_url VARCHAR(500) NOT NULL,
repository_name VARCHAR(200) NOT NULL,
owner_name VARCHAR(100) NOT NULL,
branch_name VARCHAR(100) DEFAULT 'main',
is_public BOOLEAN DEFAULT true,
requires_auth BOOLEAN DEFAULT false,
last_synced_at TIMESTAMP,
sync_status VARCHAR(50) DEFAULT 'pending',
metadata JSONB,
codebase_analysis JSONB,
created_at TIMESTAMP DEFAULT NOW(),
updated_at TIMESTAMP DEFAULT NOW()
);
-- Create table for feature-codebase mappings
CREATE TABLE IF NOT EXISTS feature_codebase_mappings (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
template_id UUID REFERENCES templates(id) ON DELETE CASCADE,
feature_id UUID REFERENCES template_features(id) ON DELETE CASCADE,
repository_id UUID REFERENCES github_repositories(id) ON DELETE CASCADE,
file_paths TEXT[],
code_snippets JSONB,
implementation_notes TEXT,
created_at TIMESTAMP DEFAULT NOW()
);
-- Create indexes for better performance
CREATE INDEX IF NOT EXISTS idx_github_repos_template_id ON github_repositories(template_id);
CREATE INDEX IF NOT EXISTS idx_github_repos_owner_name ON github_repositories(owner_name);
CREATE INDEX IF NOT EXISTS idx_feature_mappings_feature_id ON feature_codebase_mappings(feature_id);
CREATE INDEX IF NOT EXISTS idx_feature_mappings_repo_id ON feature_codebase_mappings(repository_id);
-- Add trigger to update timestamp
CREATE TRIGGER update_github_repos_updated_at BEFORE UPDATE ON github_repositories
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
-- =============================================
-- GitHub OAuth Tables
-- =============================================
-- Create table to store GitHub OAuth tokens
CREATE TABLE IF NOT EXISTS github_user_tokens (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
access_token TEXT NOT NULL,
github_username VARCHAR(100) NOT NULL,
github_user_id INTEGER NOT NULL,
scopes JSONB,
expires_at TIMESTAMP,
created_at TIMESTAMP DEFAULT NOW(),
updated_at TIMESTAMP DEFAULT NOW()
);
-- Create index for faster lookups
CREATE INDEX IF NOT EXISTS idx_github_user_tokens_github_username ON github_user_tokens(github_username);

109
services/git-integration/src/migrations/002_repository_file_storage.sql Normal file
View File

@ -0,0 +1,109 @@
-- Migration 002: Add Repository File Storage Tables
-- This migration adds comprehensive file system storage in PostgreSQL
-- Create table for repository local storage tracking
CREATE TABLE IF NOT EXISTS repository_storage (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
repository_id UUID REFERENCES github_repositories(id) ON DELETE CASCADE,
local_path TEXT NOT NULL,
storage_status VARCHAR(50) DEFAULT 'pending', -- pending, downloading, completed, error
total_files_count INTEGER DEFAULT 0,
total_directories_count INTEGER DEFAULT 0,
total_size_bytes BIGINT DEFAULT 0,
download_started_at TIMESTAMP,
download_completed_at TIMESTAMP,
last_verified_at TIMESTAMP,
created_at TIMESTAMP DEFAULT NOW(),
updated_at TIMESTAMP DEFAULT NOW(),
UNIQUE(repository_id)
);
-- Create table for directory structure
CREATE TABLE IF NOT EXISTS repository_directories (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
repository_id UUID REFERENCES github_repositories(id) ON DELETE CASCADE,
storage_id UUID REFERENCES repository_storage(id) ON DELETE CASCADE,
parent_directory_id UUID REFERENCES repository_directories(id) ON DELETE CASCADE,
directory_name VARCHAR(255) NOT NULL,
relative_path TEXT NOT NULL, -- path from repository root
absolute_path TEXT NOT NULL, -- full local filesystem path
level INTEGER DEFAULT 0, -- depth in hierarchy (0 = root)
files_count INTEGER DEFAULT 0,
subdirectories_count INTEGER DEFAULT 0,
total_size_bytes BIGINT DEFAULT 0,
created_at TIMESTAMP DEFAULT NOW(),
updated_at TIMESTAMP DEFAULT NOW()
);
-- Create table for individual files
CREATE TABLE IF NOT EXISTS repository_files (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
repository_id UUID REFERENCES github_repositories(id) ON DELETE CASCADE,
storage_id UUID REFERENCES repository_storage(id) ON DELETE CASCADE,
directory_id UUID REFERENCES repository_directories(id) ON DELETE SET NULL,
filename VARCHAR(255) NOT NULL,
file_extension VARCHAR(50),
relative_path TEXT NOT NULL, -- path from repository root
absolute_path TEXT NOT NULL, -- full local filesystem path
file_size_bytes BIGINT DEFAULT 0,
file_hash VARCHAR(64), -- SHA-256 hash for integrity
mime_type VARCHAR(100),
is_binary BOOLEAN DEFAULT false,
encoding VARCHAR(50) DEFAULT 'utf-8',
github_sha VARCHAR(40), -- GitHub blob SHA for tracking changes
created_at TIMESTAMP DEFAULT NOW(),
updated_at TIMESTAMP DEFAULT NOW()
);
-- Create table for file contents (for text files and searchability)
CREATE TABLE IF NOT EXISTS repository_file_contents (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
file_id UUID REFERENCES repository_files(id) ON DELETE CASCADE,
content_text TEXT, -- full content for text files
content_preview TEXT, -- first 1000 characters for quick preview
language_detected VARCHAR(50), -- programming language detected
line_count INTEGER DEFAULT 0,
char_count INTEGER DEFAULT 0,
is_indexed BOOLEAN DEFAULT false, -- for search indexing status
created_at TIMESTAMP DEFAULT NOW(),
updated_at TIMESTAMP DEFAULT NOW(),
UNIQUE(file_id)
);
-- Create indexes for performance
CREATE INDEX IF NOT EXISTS idx_repository_storage_repo_id ON repository_storage(repository_id);
CREATE INDEX IF NOT EXISTS idx_repository_storage_status ON repository_storage(storage_status);
CREATE INDEX IF NOT EXISTS idx_repo_directories_repo_id ON repository_directories(repository_id);
CREATE INDEX IF NOT EXISTS idx_repo_directories_parent_id ON repository_directories(parent_directory_id);
CREATE INDEX IF NOT EXISTS idx_repo_directories_storage_id ON repository_directories(storage_id);
CREATE INDEX IF NOT EXISTS idx_repo_directories_level ON repository_directories(level);
CREATE INDEX IF NOT EXISTS idx_repo_directories_relative_path ON repository_directories(relative_path);
CREATE INDEX IF NOT EXISTS idx_repo_files_repo_id ON repository_files(repository_id);
CREATE INDEX IF NOT EXISTS idx_repo_files_directory_id ON repository_files(directory_id);
CREATE INDEX IF NOT EXISTS idx_repo_files_storage_id ON repository_files(storage_id);
CREATE INDEX IF NOT EXISTS idx_repo_files_extension ON repository_files(file_extension);
CREATE INDEX IF NOT EXISTS idx_repo_files_filename ON repository_files(filename);
CREATE INDEX IF NOT EXISTS idx_repo_files_relative_path ON repository_files(relative_path);
CREATE INDEX IF NOT EXISTS idx_repo_files_is_binary ON repository_files(is_binary);
CREATE INDEX IF NOT EXISTS idx_file_contents_file_id ON repository_file_contents(file_id);
CREATE INDEX IF NOT EXISTS idx_file_contents_language ON repository_file_contents(language_detected);
CREATE INDEX IF NOT EXISTS idx_file_contents_is_indexed ON repository_file_contents(is_indexed);
-- Full text search index for file contents
CREATE INDEX IF NOT EXISTS idx_file_contents_text_search ON repository_file_contents USING gin(to_tsvector('english', content_text));
-- Add update triggers
CREATE TRIGGER update_repository_storage_updated_at BEFORE UPDATE ON repository_storage
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
CREATE TRIGGER update_repository_directories_updated_at BEFORE UPDATE ON repository_directories
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
CREATE TRIGGER update_repository_files_updated_at BEFORE UPDATE ON repository_files
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
CREATE TRIGGER update_repository_file_contents_updated_at BEFORE UPDATE ON repository_file_contents
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();

21
services/git-integration/src/migrations/003_add_user_id_to_template_refs.sql Normal file
View File

@ -0,0 +1,21 @@
-- Migration 003: Add user_id to tables that reference template_id
-- This ensures we always track which user owns/initiated records tied to a template
-- Add user_id to github_repositories
ALTER TABLE IF EXISTS github_repositories
ADD COLUMN IF NOT EXISTS user_id UUID REFERENCES users(id) ON DELETE CASCADE;
-- Indexes for github_repositories
CREATE INDEX IF NOT EXISTS idx_github_repos_user_id ON github_repositories(user_id);
CREATE INDEX IF NOT EXISTS idx_github_repos_template_user ON github_repositories(template_id, user_id);
-- Add user_id to feature_codebase_mappings
ALTER TABLE IF EXISTS feature_codebase_mappings
ADD COLUMN IF NOT EXISTS user_id UUID REFERENCES users(id) ON DELETE CASCADE;
-- Indexes for feature_codebase_mappings
CREATE INDEX IF NOT EXISTS idx_feature_mappings_user_id ON feature_codebase_mappings(user_id);
CREATE INDEX IF NOT EXISTS idx_feature_mappings_template_user ON feature_codebase_mappings(template_id, user_id);
-- Note: Columns are nullable to allow backfill before enforcing NOT NULL if desired

81
services/git-integration/src/migrations/migrate.js Normal file
View File

@ -0,0 +1,81 @@
const fs = require('fs');
const path = require('path');
const database = require('../config/database');
const migrationsDir = path.join(__dirname);
async function runMigrations() {
console.log('🚀 Starting Git Integration database migration...');
try {
// Connect to database
await database.testConnection();
console.log('✅ Database connected successfully');
// Get list of migration files
const migrationFiles = fs.readdirSync(migrationsDir)
.filter(file => file.endsWith('.sql'))
.sort();
console.log(`📄 Found ${migrationFiles.length} migration files:`, migrationFiles);
for (const migrationFile of migrationFiles) {
console.log(`🚀 Running migration: ${migrationFile}`);
const migrationPath = path.join(migrationsDir, migrationFile);
const migrationSQL = fs.readFileSync(migrationPath, 'utf8');
try {
await database.query(migrationSQL);
console.log(`✅ Migration ${migrationFile} completed successfully!`);
} catch (err) {
const message = (err && err.message) ? err.message.toLowerCase() : '';
const code = err && err.code ? err.code : '';
// Continue on idempotency-safe errors (objects already exist)
const isIdempotentError =
message.includes('already exists') ||
code === '42710' /* duplicate_object */ ||
code === '42P07' /* duplicate_table */ ||
code === '42701' /* duplicate_column */ ||
code === '42P06' /* duplicate_schema */ ||
code === '42723' /* duplicate_function */;
if (isIdempotentError) {
console.warn(`⚠️ Skipping idempotent error for ${migrationFile}:`, err.message);
continue;
}
throw err; // rethrow non-idempotent errors
}
}
// Verify tables were created
const tablesQuery = `
SELECT table_name
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name NOT LIKE 'pg_%'
ORDER BY table_name
`;
const tablesResult = await database.query(tablesQuery);
const tableNames = tablesResult.rows.map(row => row.table_name);
console.log('🔍 Verified tables:', tableNames);
console.log('🎉 All migrations completed successfully!');
} catch (error) {
console.error('❌ Migration failed:', error);
process.exit(1);
} finally {
await database.close();
console.log('🔌 Database connection closed');
}
}
// Run migrations if this file is executed directly
if (require.main === module) {
runMigrations();
}
module.exports = { runMigrations };

561
services/git-integration/src/routes/github-integration.routes.js Normal file
View File

@ -0,0 +1,561 @@
// Updated routes/github-integration.js
const express = require('express');
const router = express.Router();
const GitHubIntegrationService = require('../services/github-integration.service');
const GitHubOAuthService = require('../services/github-oauth');
const FileStorageService = require('../services/file-storage.service');
const database = require('../config/database');
const fs = require('fs');
const path = require('path');
const githubService = new GitHubIntegrationService();
const oauthService = new GitHubOAuthService();
const fileStorageService = new FileStorageService();
// Attach GitHub repository to template
router.post('/attach-repository', async (req, res) => {
try {
const { template_id, repository_url, branch_name } = req.body;
// Validate input
if (!template_id || !repository_url) {
return res.status(400).json({
success: false,
message: 'Template ID and repository URL are required'
});
}
// Check if template exists
const templateQuery = 'SELECT * FROM templates WHERE id = $1 AND is_active = true';
const templateResult = await database.query(templateQuery, [template_id]);
if (templateResult.rows.length === 0) {
return res.status(404).json({
success: false,
message: 'Template not found'
});
}
// Parse GitHub URL
const { owner, repo, branch } = githubService.parseGitHubUrl(repository_url);
// Check repository access
const accessCheck = await githubService.checkRepositoryAccess(owner, repo);
if (!accessCheck.hasAccess) {
if (accessCheck.requiresAuth) {
// Check if we have OAuth token
const tokenRecord = await oauthService.getToken();
if (!tokenRecord) {
return res.status(401).json({
success: false,
message: 'GitHub authentication required for this repository',
requires_auth: true,
auth_url: `/api/github/auth/github`
});
}
}
return res.status(404).json({
success: false,
message: accessCheck.error || 'Repository not accessible'
});
}
// Get repository information from GitHub
const repositoryData = await githubService.fetchRepositoryMetadata(owner, repo);
// Analyze the codebase
const codebaseAnalysis = await githubService.analyzeCodebase(owner, repo, branch || branch_name);
// Store everything in PostgreSQL
const insertQuery = `
INSERT INTO github_repositories (
template_id, repository_url, repository_name, owner_name,
branch_name, is_public, metadata, codebase_analysis, sync_status,
requires_auth
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
RETURNING *
`;
const insertValues = [
template_id,
repository_url,
repo,
owner,
branch || branch_name || 'main',
repositoryData.visibility === 'public',
JSON.stringify(repositoryData),
JSON.stringify(codebaseAnalysis),
'synced',
accessCheck.requiresAuth
];
const insertResult = await database.query(insertQuery, insertValues);
const repositoryRecord = insertResult.rows[0];
// Download repository with file storage
console.log('Downloading repository with storage...');
const downloadResult = await githubService.downloadRepositoryWithStorage(
owner, repo, branch || branch_name || 'main', repositoryRecord.id
);
if (!downloadResult.success) {
// If download failed, still return the repository record but mark the storage issue
console.warn('Repository download failed:', downloadResult.error);
}
// Create feature-codebase mappings
const featureQuery = 'SELECT id FROM template_features WHERE template_id = $1';
const featureResult = await database.query(featureQuery, [template_id]);
if (featureResult.rows.length > 0) {
const mappingValues = [];
const mappingParams = [];
let paramIndex = 1;
for (const feature of featureResult.rows) {
mappingValues.push(`(uuid_generate_v4(), $${paramIndex++}, $${paramIndex++}, $${paramIndex++}, $${paramIndex++})`);
mappingParams.push(
feature.id,
repositoryRecord.id,
'[]', // Empty paths for now
'{}' // Empty snippets for now
);
}
const mappingQuery = `
INSERT INTO feature_codebase_mappings (id, feature_id, repository_id, code_paths, code_snippets)
VALUES ${mappingValues.join(', ')}
`;
await database.query(mappingQuery, mappingParams);
}
// Get storage information
const storageInfo = await githubService.getRepositoryStorage(repositoryRecord.id);
res.status(201).json({
success: true,
message: 'Repository attached successfully',
data: {
repository_id: repositoryRecord.id,
template_id: repositoryRecord.template_id,
repository_name: repositoryRecord.repository_name,
owner_name: repositoryRecord.owner_name,
branch_name: repositoryRecord.branch_name,
is_public: repositoryRecord.is_public,
requires_auth: repositoryRecord.requires_auth,
metadata: repositoryData,
codebase_analysis: codebaseAnalysis,
storage_info: storageInfo,
download_result: downloadResult
}
});
} catch (error) {
console.error('Error attaching repository:', error);
res.status(500).json({
success: false,
message: error.message || 'Failed to attach repository'
});
}
});
// Get repository information for a template
router.get('/template/:id/repository', async (req, res) => {
try {
const { id } = req.params;
const query = `
SELECT gr.*, rs.local_path, rs.storage_status, rs.total_files_count,
rs.total_directories_count, rs.total_size_bytes, rs.download_completed_at
FROM github_repositories gr
LEFT JOIN repository_storage rs ON gr.id = rs.repository_id
WHERE gr.template_id = $1
ORDER BY gr.created_at DESC
LIMIT 1
`;
const result = await database.query(query, [id]);
if (result.rows.length === 0) {
return res.status(404).json({
success: false,
message: 'No repository found for this template'
});
}
const repository = result.rows[0];
const parseMaybe = (v) => {
if (v == null) return {};
if (typeof v === 'string') {
try { return JSON.parse(v); } catch { return {}; }
}
return v; // already an object from jsonb
};
res.json({
success: true,
data: {
...repository,
metadata: parseMaybe(repository.metadata),
codebase_analysis: parseMaybe(repository.codebase_analysis)
}
});
} catch (error) {
console.error('Error fetching repository:', error);
res.status(500).json({
success: false,
message: error.message || 'Failed to fetch repository'
});
}
});
// Get repository file structure
router.get('/repository/:id/structure', async (req, res) => {
try {
const { id } = req.params;
const { path: directoryPath } = req.query;
// Get repository info
const repoQuery = 'SELECT * FROM github_repositories WHERE id = $1';
const repoResult = await database.query(repoQuery, [id]);
if (repoResult.rows.length === 0) {
return res.status(404).json({
success: false,
message: 'Repository not found'
});
}
const structure = await fileStorageService.getRepositoryStructure(id, directoryPath);
res.json({
success: true,
data: {
repository_id: id,
directory_path: directoryPath || '',
structure: structure
}
});
} catch (error) {
console.error('Error fetching repository structure:', error);
res.status(500).json({
success: false,
message: error.message || 'Failed to fetch repository structure'
});
}
});
// Get files in a directory
router.get('/repository/:id/files', async (req, res) => {
try {
const { id } = req.params;
const { directory_path = '' } = req.query;
// Get repository info
const repoQuery = 'SELECT * FROM github_repositories WHERE id = $1';
const repoResult = await database.query(repoQuery, [id]);
if (repoResult.rows.length === 0) {
return res.status(404).json({
success: false,
message: 'Repository not found'
});
}
const files = await fileStorageService.getDirectoryFiles(id, directory_path);
res.json({
success: true,
data: {
repository_id: id,
directory_path: directory_path,
files: files
}
});
} catch (error) {
console.error('Error fetching directory files:', error);
res.status(500).json({
success: false,
message: error.message || 'Failed to fetch directory files'
});
}
});
// Get file content
router.get('/repository/:id/file-content', async (req, res) => {
try {
const { id } = req.params;
const { file_path } = req.query;
if (!file_path) {
return res.status(400).json({
success: false,
message: 'File path is required'
});
}
const query = `
SELECT rf.*, rfc.content_text, rfc.content_preview, rfc.language_detected,
rfc.line_count, rfc.char_count
FROM repository_files rf
LEFT JOIN repository_file_contents rfc ON rf.id = rfc.file_id
WHERE rf.repository_id = $1 AND rf.relative_path = $2
`;
const result = await database.query(query, [id, file_path]);
if (result.rows.length === 0) {
return res.status(404).json({
success: false,
message: 'File not found'
});
}
const file = result.rows[0];
res.json({
success: true,
data: {
file_info: {
id: file.id,
filename: file.filename,
file_extension: file.file_extension,
relative_path: file.relative_path,
file_size_bytes: file.file_size_bytes,
mime_type: file.mime_type,
is_binary: file.is_binary,
language_detected: file.language_detected,
line_count: file.line_count,
char_count: file.char_count
},
content: file.is_binary ? null : file.content_text,
preview: file.content_preview
}
});
} catch (error) {
console.error('Error fetching file content:', error);
res.status(500).json({
success: false,
message: error.message || 'Failed to fetch file content'
});
}
});
// Search repository files
router.get('/repository/:id/search', async (req, res) => {
try {
const { id } = req.params;
const { q: query } = req.query;
if (!query) {
return res.status(400).json({
success: false,
message: 'Search query is required'
});
}
const results = await fileStorageService.searchFileContent(id, query);
res.json({
success: true,
data: {
repository_id: id,
search_query: query,
results: results,
total_results: results.length
}
});
} catch (error) {
console.error('Error searching repository:', error);
res.status(500).json({
success: false,
message: error.message || 'Failed to search repository'
});
}
});
// List all repositories for a template
router.get('/template/:id/repositories', async (req, res) => {
try {
const { id } = req.params;
const query = `
SELECT gr.*, rs.local_path, rs.storage_status, rs.total_files_count,
rs.total_directories_count, rs.total_size_bytes, rs.download_completed_at
FROM github_repositories gr
LEFT JOIN repository_storage rs ON gr.id = rs.repository_id
WHERE gr.template_id = $1
ORDER BY gr.created_at DESC
`;
const result = await database.query(query, [id]);
const repositories = result.rows.map(repo => ({
...repo,
metadata: JSON.parse(repo.metadata || '{}'),
codebase_analysis: JSON.parse(repo.codebase_analysis || '{}')
}));
res.json({
success: true,
data: repositories
});
} catch (error) {
console.error('Error fetching repositories:', error);
res.status(500).json({
success: false,
message: error.message || 'Failed to fetch repositories'
});
}
});
// Download repository files (legacy endpoint for backward compatibility)
router.post('/download', async (req, res) => {
try {
const { repository_url, branch_name } = req.body;
if (!repository_url) {
return res.status(400).json({
success: false,
message: 'Repository URL is required'
});
}
const { owner, repo, branch } = githubService.parseGitHubUrl(repository_url);
const targetBranch = branch || branch_name || 'main';
const result = await githubService.downloadRepository(owner, repo, targetBranch);
if (result.success) {
res.json({
success: true,
message: 'Repository downloaded successfully',
data: result
});
} else {
res.status(500).json({
success: false,
message: 'Failed to download repository',
error: result.error
});
}
} catch (error) {
console.error('Error downloading repository:', error);
res.status(500).json({
success: false,
message: error.message || 'Failed to download repository'
});
}
});
// Re-sync repository (re-download and update database)
router.post('/repository/:id/sync', async (req, res) => {
try {
const { id } = req.params;
// Get repository info
const repoQuery = 'SELECT * FROM github_repositories WHERE id = $1';
const repoResult = await database.query(repoQuery, [id]);
if (repoResult.rows.length === 0) {
return res.status(404).json({
success: false,
message: 'Repository not found'
});
}
const repository = repoResult.rows[0];
const { owner, repo, branch } = githubService.parseGitHubUrl(repository.repository_url);
// Clean up existing storage
await githubService.cleanupRepositoryStorage(id);
// Re-download with storage
const downloadResult = await githubService.downloadRepositoryWithStorage(
owner, repo, branch || repository.branch_name, id
);
// Update sync status
await database.query(
'UPDATE github_repositories SET sync_status = $1, updated_at = NOW() WHERE id = $2',
[downloadResult.success ? 'synced' : 'error', id]
);
res.json({
success: downloadResult.success,
message: downloadResult.success ? 'Repository synced successfully' : 'Failed to sync repository',
data: downloadResult
});
} catch (error) {
console.error('Error syncing repository:', error);
res.status(500).json({
success: false,
message: error.message || 'Failed to sync repository'
});
}
});
// Remove repository from template
router.delete('/repository/:id', async (req, res) => {
try {
const { id } = req.params;
// Get repository info before deletion
const getQuery = 'SELECT * FROM github_repositories WHERE id = $1';
const getResult = await database.query(getQuery, [id]);
if (getResult.rows.length === 0) {
return res.status(404).json({
success: false,
message: 'Repository not found'
});
}
const repository = getResult.rows[0];
// Clean up file storage
await githubService.cleanupRepositoryStorage(id);
// Delete feature mappings first
await database.query(
'DELETE FROM feature_codebase_mappings WHERE repository_id = $1',
[id]
);
// Delete repository record
await database.query(
'DELETE FROM github_repositories WHERE id = $1',
[id]
);
res.json({
success: true,
message: 'Repository removed successfully',
data: {
removed_repository: repository.repository_name,
template_id: repository.template_id
}
});
} catch (error) {
console.error('Error removing repository:', error);
res.status(500).json({
success: false,
message: error.message || 'Failed to remove repository'
});
}
});
module.exports = router;

169
services/git-integration/src/routes/github-oauth.js Normal file
View File

@ -0,0 +1,169 @@
// routes/github-oauth.js
const express = require('express');
const router = express.Router();
const GitHubOAuthService = require('../services/github-oauth');
const oauthService = new GitHubOAuthService();
// Initiate GitHub OAuth flow
router.get('/auth/github', async (req, res) => {
try {
const state = Math.random().toString(36).substring(7);
const authUrl = oauthService.getAuthUrl(state);
res.json({
success: true,
data: {
auth_url: authUrl,
state: state
}
});
} catch (error) {
console.error('Error initiating GitHub OAuth:', error);
res.status(500).json({
success: false,
message: error.message || 'Failed to initiate GitHub authentication'
});
}
});
// Handle GitHub OAuth callback
router.get('/auth/github/callback', async (req, res) => {
try {
const { code, state } = req.query;
if (!code) {
return res.status(400).json({
success: false,
message: 'Authorization code missing'
});
}
// Exchange code for token
const accessToken = await oauthService.exchangeCodeForToken(code);
// Get user info from GitHub
const githubUser = await oauthService.getUserInfo(accessToken);
// Store token
const tokenRecord = await oauthService.storeToken(accessToken, githubUser);
// Redirect back to frontend if configured
const frontendUrl = process.env.FRONTEND_URL || 'http://localhost:3000';
try {
const redirectUrl = `${frontendUrl}/project-builder?github_connected=1&user=${encodeURIComponent(githubUser.login)}`;
return res.redirect(302, redirectUrl);
} catch (e) {
// Fallback to JSON if redirect fails
return res.json({
success: true,
message: 'GitHub account connected successfully',
data: {
github_username: githubUser.login,
github_user_id: githubUser.id,
connected_at: tokenRecord.created_at
}
});
}
} catch (error) {
console.error('Error handling GitHub callback:', error);
res.status(500).json({
success: false,
message: error.message || 'Failed to connect GitHub account'
});
}
});
// Get GitHub connection status
router.get('/auth/github/status', async (req, res) => {
try {
const tokenRecord = await oauthService.getToken();
if (tokenRecord) {
res.json({
success: true,
data: {
connected: true,
github_username: tokenRecord.github_username,
github_user_id: tokenRecord.github_user_id,
connected_at: tokenRecord.created_at,
scopes: tokenRecord.scopes
}
});
} else {
res.json({
success: true,
data: {
connected: false
}
});
}
} catch (error) {
console.error('Error checking GitHub status:', error);
res.status(500).json({
success: false,
message: error.message || 'Failed to check GitHub connection status'
});
}
});
// Disconnect GitHub account
router.delete('/auth/github', async (req, res) => {
try {
await oauthService.revokeToken();
res.json({
success: true,
message: 'GitHub account disconnected successfully'
});
} catch (error) {
console.error('Error disconnecting GitHub:', error);
res.status(500).json({
success: false,
message: error.message || 'Failed to disconnect GitHub account'
});
}
});
// Test repository access
router.post('/test-access', async (req, res) => {
try {
const { repository_url } = req.body;
if (!repository_url) {
return res.status(400).json({
success: false,
message: 'Repository URL is required'
});
}
const GitHubIntegrationService = require('../services/github-integration.service');
const githubService = new GitHubIntegrationService();
const { owner, repo } = githubService.parseGitHubUrl(repository_url);
const canAccess = await oauthService.canAccessRepository(owner, repo);
res.json({
success: true,
data: {
repository_url,
owner,
repo,
can_access: canAccess
}
});
} catch (error) {
console.error('Error testing repository access:', error);
res.status(500).json({
success: false,
message: error.message || 'Failed to test repository access'
});
}
});
module.exports = router;

373
services/git-integration/src/services/file-storage.service.js Normal file
View File

@ -0,0 +1,373 @@
// services/file-storage.service.js
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const database = require('../config/database');
class FileStorageService {
constructor() {
this.supportedTextExtensions = new Set([
'.js', '.ts', '.jsx', '.tsx', '.vue', '.py', '.java', '.cpp', '.c', '.cs',
'.php', '.rb', '.go', '.rs', '.kt', '.swift', '.scala', '.clj', '.hs',
'.elm', '.ml', '.fs', '.vb', '.pas', '.asm', '.sql', '.sh', '.bash',
'.ps1', '.bat', '.cmd', '.html', '.htm', '.xml', '.css', '.scss', '.sass',
'.less', '.json', '.yaml', '.yml', '.toml', '.ini', '.cfg', '.conf',
'.env', '.md', '.txt', '.rst', '.adoc', '.tex', '.r', '.m', '.pl',
'.lua', '.dart', '.jl', '.nim', '.zig', '.v', '.d', '.cr', '.ex', '.exs'
]);
this.languageMap = {
'.js': 'javascript', '.ts': 'typescript', '.jsx': 'javascript', '.tsx': 'typescript',
'.vue': 'vue', '.py': 'python', '.java': 'java', '.cpp': 'cpp', '.c': 'c',
'.cs': 'csharp', '.php': 'php', '.rb': 'ruby', '.go': 'go', '.rs': 'rust',
'.kt': 'kotlin', '.swift': 'swift', '.scala': 'scala', '.clj': 'clojure',
'.hs': 'haskell', '.elm': 'elm', '.ml': 'ocaml', '.fs': 'fsharp',
'.vb': 'vbnet', '.html': 'html', '.css': 'css', '.scss': 'scss',
'.json': 'json', '.yaml': 'yaml', '.yml': 'yaml', '.xml': 'xml',
'.sql': 'sql', '.sh': 'bash', '.md': 'markdown'
};
}
// Initialize storage record for a repository
async initializeRepositoryStorage(repositoryId, localPath) {
const query = `
INSERT INTO repository_storage (
repository_id, local_path, storage_status, download_started_at
) VALUES ($1, $2, $3, NOW())
ON CONFLICT (repository_id)
DO UPDATE SET
local_path = $2,
storage_status = $3,
download_started_at = NOW(),
updated_at = NOW()
RETURNING *
`;
const result = await database.query(query, [repositoryId, localPath, 'downloading']);
return result.rows[0];
}
// Process and store directory structure
async processDirectoryStructure(storageId, repositoryId, basePath, currentPath = '', parentDirId = null, level = 0) {
const fullPath = path.join(basePath, currentPath);
if (!fs.existsSync(fullPath)) {
return null;
}
const stats = fs.statSync(fullPath);
if (!stats.isDirectory()) {
return null;
}
// Insert directory record
const dirName = currentPath === '' ? '.' : path.basename(currentPath);
const relativePath = currentPath === '' ? '' : currentPath;
const dirQuery = `
INSERT INTO repository_directories (
repository_id, storage_id, parent_directory_id, directory_name,
relative_path, absolute_path, level
) VALUES ($1, $2, $3, $4, $5, $6, $7)
RETURNING *
`;
const dirResult = await database.query(dirQuery, [
repositoryId, storageId, parentDirId, dirName,
relativePath, fullPath, level
]);
const directoryRecord = dirResult.rows[0];
let totalFiles = 0;
let totalSubDirs = 0;
let totalSize = 0;
try {
const items = fs.readdirSync(fullPath);
for (const item of items) {
const itemPath = path.join(fullPath, item);
const itemRelativePath = currentPath ? path.join(currentPath, item) : item;
const itemStats = fs.statSync(itemPath);
if (itemStats.isDirectory()) {
// Recursively process subdirectory
const subDir = await this.processDirectoryStructure(
storageId, repositoryId, basePath, itemRelativePath,
directoryRecord.id, level + 1
);
if (subDir) {
totalSubDirs++;
}
} else if (itemStats.isFile()) {
// Process file
const fileRecord = await this.processFile(
storageId, repositoryId, directoryRecord.id, itemPath, itemRelativePath
);
if (fileRecord) {
totalFiles++;
// IMPORTANT: node-postgres returns BIGINT columns as strings.
// Avoid string concatenation causing huge numeric strings by summing the
// actual filesystem-reported size (a number) for aggregation.
totalSize += Number(itemStats.size) || 0;
}
}
}
} catch (error) {
console.warn(`Error processing directory ${fullPath}:`, error.message);
}
// Update directory stats
await database.query(`
UPDATE repository_directories
SET files_count = $1, subdirectories_count = $2, total_size_bytes = $3, updated_at = NOW()
WHERE id = $4
`, [totalFiles, totalSubDirs, totalSize, directoryRecord.id]);
return directoryRecord;
}
// Process and store individual file
async processFile(storageId, repositoryId, directoryId, absolutePath, relativePath) {
try {
const stats = fs.statSync(absolutePath);
const filename = path.basename(absolutePath);
const extension = path.extname(filename).toLowerCase();
// Calculate file hash
const fileBuffer = fs.readFileSync(absolutePath);
const hash = crypto.createHash('sha256').update(fileBuffer).digest('hex');
// Determine if file is binary
const isBinary = !this.supportedTextExtensions.has(extension) || this.isBinaryContent(fileBuffer);
// Get MIME type (simplified)
const mimeType = this.getMimeType(extension, isBinary);
// Insert file record
const fileQuery = `
INSERT INTO repository_files (
repository_id, storage_id, directory_id, filename, file_extension,
relative_path, absolute_path, file_size_bytes, file_hash,
mime_type, is_binary, encoding
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
RETURNING *
`;
const fileResult = await database.query(fileQuery, [
repositoryId, storageId, directoryId, filename, extension,
relativePath, absolutePath, stats.size, hash,
mimeType, isBinary, isBinary ? null : 'utf-8'
]);
const fileRecord = fileResult.rows[0];
// Process file content if it's a text file and not too large (< 10MB)
if (!isBinary && stats.size < 10 * 1024 * 1024) {
await this.processFileContent(fileRecord.id, absolutePath, extension);
}
return fileRecord;
} catch (error) {
console.warn(`Error processing file ${absolutePath}:`, error.message);
return null;
}
}
// Process and store file content
async processFileContent(fileId, absolutePath, extension) {
try {
const content = fs.readFileSync(absolutePath, 'utf-8');
const lines = content.split('\n');
const preview = content.substring(0, 1000);
const language = this.languageMap[extension] || 'text';
const contentQuery = `
INSERT INTO repository_file_contents (
file_id, content_text, content_preview, language_detected,
line_count, char_count
) VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (file_id) DO UPDATE SET
content_text = $2,
content_preview = $3,
language_detected = $4,
line_count = $5,
char_count = $6,
updated_at = NOW()
`;
await database.query(contentQuery, [
fileId, content, preview, language, lines.length, content.length
]);
} catch (error) {
console.warn(`Error processing file content for file ID ${fileId}:`, error.message);
}
}
// Complete storage process for a repository
async completeRepositoryStorage(storageId) {
// Calculate totals
const statsQuery = `
SELECT
COUNT(DISTINCT rd.id) as total_directories,
COUNT(rf.id) as total_files,
COALESCE(SUM(rf.file_size_bytes), 0) as total_size
FROM repository_storage rs
LEFT JOIN repository_directories rd ON rs.id = rd.storage_id
LEFT JOIN repository_files rf ON rs.id = rf.storage_id
WHERE rs.id = $1
`;
const statsResult = await database.query(statsQuery, [storageId]);
const stats = statsResult.rows[0];
// Update storage record
const updateQuery = `
UPDATE repository_storage
SET
storage_status = 'completed',
total_files_count = $1,
total_directories_count = $2,
total_size_bytes = $3,
download_completed_at = NOW(),
updated_at = NOW()
WHERE id = $4
RETURNING *
`;
const result = await database.query(updateQuery, [
parseInt(stats.total_files),
parseInt(stats.total_directories),
parseInt(stats.total_size),
storageId
]);
return result.rows[0];
}
// Mark storage as failed
async markStorageFailed(storageId, error) {
const query = `
UPDATE repository_storage
SET
storage_status = 'error',
updated_at = NOW()
WHERE id = $1
RETURNING *
`;
const result = await database.query(query, [storageId]);
return result.rows[0];
}
// Get repository file structure
async getRepositoryStructure(repositoryId, directoryPath = null) {
let query = `
SELECT
rd.*,
COUNT(DISTINCT rdf.id) as files_count,
COUNT(DISTINCT rds.id) as subdirs_count
FROM repository_directories rd
LEFT JOIN repository_files rdf ON rd.id = rdf.directory_id
LEFT JOIN repository_directories rds ON rd.id = rds.parent_directory_id
WHERE rd.repository_id = $1
`;
const params = [repositoryId];
if (directoryPath !== null) {
query += ` AND rd.relative_path = $2`;
params.push(directoryPath);
}
query += ` GROUP BY rd.id ORDER BY rd.level, rd.directory_name`;
const result = await database.query(query, params);
return result.rows;
}
// Get files in a directory
async getDirectoryFiles(repositoryId, directoryPath = '') {
const query = `
SELECT rf.*, rfc.language_detected, rfc.line_count
FROM repository_files rf
LEFT JOIN repository_directories rd ON rf.directory_id = rd.id
LEFT JOIN repository_file_contents rfc ON rf.id = rfc.file_id
WHERE rf.repository_id = $1 AND rd.relative_path = $2
ORDER BY rf.filename
`;
const result = await database.query(query, [repositoryId, directoryPath]);
return result.rows;
}
// Search files by content
async searchFileContent(repositoryId, searchQuery) {
const query = `
SELECT rf.filename, rf.relative_path, rfc.language_detected,
ts_rank_cd(to_tsvector('english', rfc.content_text), plainto_tsquery('english', $2)) as rank
FROM repository_files rf
JOIN repository_file_contents rfc ON rf.id = rfc.file_id
WHERE rf.repository_id = $1
AND to_tsvector('english', rfc.content_text) @@ plainto_tsquery('english', $2)
ORDER BY rank DESC, rf.filename
LIMIT 50
`;
const result = await database.query(query, [repositoryId, searchQuery]);
return result.rows;
}
// Utility methods
isBinaryContent(buffer) {
// Simple binary detection - check for null bytes in first 1024 bytes
const sample = buffer.slice(0, Math.min(1024, buffer.length));
return sample.includes(0);
}
getMimeType(extension, isBinary) {
const mimeTypes = {
'.js': 'application/javascript',
'.ts': 'application/typescript',
'.json': 'application/json',
'.html': 'text/html',
'.css': 'text/css',
'.md': 'text/markdown',
'.txt': 'text/plain',
'.xml': 'application/xml',
'.yml': 'application/x-yaml',
'.yaml': 'application/x-yaml',
'.pdf': 'application/pdf',
'.png': 'image/png',
'.jpg': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.gif': 'image/gif',
'.svg': 'image/svg+xml'
};
if (mimeTypes[extension]) {
return mimeTypes[extension];
}
return isBinary ? 'application/octet-stream' : 'text/plain';
}
// Clean up storage for a repository
async cleanupRepositoryStorage(repositoryId) {
const queries = [
'DELETE FROM repository_file_contents WHERE file_id IN (SELECT id FROM repository_files WHERE repository_id = $1)',
'DELETE FROM repository_files WHERE repository_id = $1',
'DELETE FROM repository_directories WHERE repository_id = $1',
'DELETE FROM repository_storage WHERE repository_id = $1'
];
for (const query of queries) {
await database.query(query, [repositoryId]);
}
}
}
module.exports = FileStorageService;

402
services/git-integration/src/services/github-integration.service.js Normal file
View File

@ -0,0 +1,402 @@
// Updated github-integration.js service
const { Octokit } = require('@octokit/rest');
const fs = require('fs');
const path = require('path');
const { exec } = require('child_process');
const GitHubOAuthService = require('./github-oauth');
const FileStorageService = require('./file-storage.service');
class GitHubIntegrationService {
constructor() {
this.oauthService = new GitHubOAuthService();
this.fileStorageService = new FileStorageService();
// Default unauthenticated instance
this.octokit = new Octokit({
userAgent: 'CodeNuk-GitIntegration/1.0.0',
});
}
// Get authenticated Octokit instance
async getAuthenticatedOctokit() {
return await this.oauthService.getAuthenticatedOctokit();
}
// Extract owner, repo, and branch from GitHub URL
parseGitHubUrl(url) {
const regex = /github\.com\/([^\/]+)\/([^\/]+)(?:\/tree\/([^\/]+))?/;
const match = url.match(regex);
if (!match) {
throw new Error('Invalid GitHub repository URL');
}
return {
owner: match[1],
repo: match[2].replace('.git', ''),
branch: match[3] || 'main'
};
}
// Check repository access and type
async checkRepositoryAccess(owner, repo) {
try {
const octokit = await this.getAuthenticatedOctokit();
const { data } = await octokit.repos.get({ owner, repo });
return {
exists: true,
isPrivate: data.private,
hasAccess: true,
requiresAuth: data.private
};
} catch (error) {
if (error.status === 404) {
return {
exists: false,
isPrivate: null,
hasAccess: false,
requiresAuth: true,
error: 'Repository not found or requires authentication'
};
}
throw error;
}
}
// Get repository information from GitHub
async fetchRepositoryMetadata(owner, repo) {
const octokit = await this.getAuthenticatedOctokit();
const safe = async (fn, fallback) => {
try {
return await fn();
} catch (error) {
console.warn(`API call failed: ${error.message}`);
return fallback;
}
};
const repoData = await safe(
async () => (await octokit.repos.get({ owner, repo })).data,
{}
);
const languages = await safe(
async () => (await octokit.repos.listLanguages({ owner, repo })).data,
{}
);
const topics = await safe(
async () => (await octokit.repos.getAllTopics({ owner, repo })).data?.names || [],
[]
);
return {
full_name: repoData.full_name || `${owner}/${repo}`,
description: repoData.description || null,
language: repoData.language || null,
topics,
languages,
visibility: repoData.private ? 'private' : 'public',
stargazers_count: repoData.stargazers_count || 0,
forks_count: repoData.forks_count || 0,
default_branch: repoData.default_branch || 'main',
size: repoData.size || 0,
updated_at: repoData.updated_at || new Date().toISOString()
};
}
// Analyze codebase structure
async analyzeCodebase(owner, repo, branch) {
try {
const octokit = await this.getAuthenticatedOctokit();
// Get the commit SHA for the branch
const { data: ref } = await octokit.git.getRef({
owner,
repo,
ref: `heads/${branch}`
});
const commitSha = ref.object.sha;
// Get the tree recursively
const { data: tree } = await octokit.git.getTree({
owner,
repo,
tree_sha: commitSha,
recursive: 'true'
});
const analysis = {
total_files: 0,
total_size: 0,
languages: {},
file_types: {},
directories: [],
last_commit: commitSha,
branch: branch
};
tree.tree.forEach(item => {
if (item.type === 'blob') {
analysis.total_files++;
analysis.total_size += item.size || 0;
const ext = path.extname(item.path).toLowerCase();
analysis.file_types[ext] = (analysis.file_types[ext] || 0) + 1;
} else if (item.type === 'tree') {
analysis.directories.push(item.path);
}
});
return analysis;
} catch (error) {
console.error('Error analyzing codebase:', error);
return {
error: error.message,
total_files: 0,
total_size: 0
};
}
}
// Update GitHub SHAs for files after processing
async updateFileGitHubSHAs(repositoryId, fileMap) {
try {
const database = require('../config/database');
for (const [relativePath, githubSha] of fileMap.entries()) {
await database.query(
'UPDATE repository_files SET github_sha = $1 WHERE repository_id = $2 AND relative_path = $3',
[githubSha, repositoryId, relativePath]
);
}
} catch (error) {
console.warn('Error updating GitHub SHAs:', error.message);
}
}
// Get repository storage information
async getRepositoryStorage(repositoryId) {
const database = require('../config/database');
const query = `
SELECT rs.*,
COUNT(DISTINCT rd.id) as directories_count,
COUNT(rf.id) as files_count
FROM repository_storage rs
LEFT JOIN repository_directories rd ON rs.id = rd.storage_id
LEFT JOIN repository_files rf ON rs.id = rf.storage_id
WHERE rs.repository_id = $1
GROUP BY rs.id
`;
const result = await database.query(query, [repositoryId]);
return result.rows[0] || null;
}
// Clean up repository storage
async cleanupRepositoryStorage(repositoryId) {
return await this.fileStorageService.cleanupRepositoryStorage(repositoryId);
}
// Download repository files locally and store in database
async downloadRepositoryWithStorage(owner, repo, branch, repositoryId) {
const targetDir = path.join(
process.env.ATTACHED_REPOS_DIR || '/tmp/attached-repos',
`${owner}__${repo}__${branch}`
);
// Create target directory
if (!fs.existsSync(targetDir)) {
fs.mkdirSync(targetDir, { recursive: true });
}
let storageRecord = null;
try {
// Initialize storage record
storageRecord = await this.fileStorageService.initializeRepositoryStorage(
repositoryId,
targetDir
);
const octokit = await this.getAuthenticatedOctokit();
// Get the commit SHA for the branch
const { data: ref } = await octokit.git.getRef({
owner,
repo,
ref: `heads/${branch}`
});
const commitSha = ref.object.sha;
// Get the tree recursively
const { data: tree } = await octokit.git.getTree({
owner,
repo,
tree_sha: commitSha,
recursive: 'true'
});
let filesWritten = 0;
let totalBytes = 0;
const fileMap = new Map(); // Map to store GitHub SHA for files
// Process each file
for (const item of tree.tree) {
if (item.type === 'blob') {
try {
const { data: blob } = await octokit.git.getBlob({
owner,
repo,
file_sha: item.sha
});
const filePath = path.join(targetDir, item.path);
const fileDir = path.dirname(filePath);
// Create directory if it doesn't exist
if (!fs.existsSync(fileDir)) {
fs.mkdirSync(fileDir, { recursive: true });
}
// Write file content
const content = Buffer.from(blob.content, 'base64');
fs.writeFileSync(filePath, content);
// Store GitHub SHA for later use
fileMap.set(item.path, item.sha);
filesWritten++;
totalBytes += content.length;
} catch (error) {
console.warn(`Failed to download file ${item.path}:`, error.message);
}
}
}
// Process directory structure and store in database
console.log('Processing directory structure...');
await this.fileStorageService.processDirectoryStructure(
storageRecord.id,
repositoryId,
targetDir
);
// Update GitHub SHAs for files
await this.updateFileGitHubSHAs(repositoryId, fileMap);
// Complete storage process
const finalStorage = await this.fileStorageService.completeRepositoryStorage(
storageRecord.id
);
console.log(`Repository storage completed: ${finalStorage.total_files_count} files, ${finalStorage.total_directories_count} directories`);
return {
success: true,
targetDir,
files: filesWritten,
bytes: totalBytes,
storage: finalStorage
};
} catch (error) {
console.error('Error downloading repository with storage:', error);
if (storageRecord) {
await this.fileStorageService.markStorageFailed(storageRecord.id, error.message);
}
return {
success: false,
error: error.message
};
}
}
// Legacy method - download repository files locally (backwards compatibility)
async downloadRepository(owner, repo, branch) {
const targetDir = path.join(
process.env.ATTACHED_REPOS_DIR || '/tmp/attached-repos',
`${owner}__${repo}__${branch}`
);
// Create target directory
if (!fs.existsSync(targetDir)) {
fs.mkdirSync(targetDir, { recursive: true });
}
try {
const octokit = await this.getAuthenticatedOctokit();
// Get the commit SHA for the branch
const { data: ref } = await octokit.git.getRef({
owner,
repo,
ref: `heads/${branch}`
});
const commitSha = ref.object.sha;
// Get the tree recursively
const { data: tree } = await octokit.git.getTree({
owner,
repo,
tree_sha: commitSha,
recursive: 'true'
});
let filesWritten = 0;
let totalBytes = 0;
// Process each file
for (const item of tree.tree) {
if (item.type === 'blob') {
try {
const { data: blob } = await octokit.git.getBlob({
owner,
repo,
file_sha: item.sha
});
const filePath = path.join(targetDir, item.path);
const fileDir = path.dirname(filePath);
// Create directory if it doesn't exist
if (!fs.existsSync(fileDir)) {
fs.mkdirSync(fileDir, { recursive: true });
}
// Write file content
const content = Buffer.from(blob.content, 'base64');
fs.writeFileSync(filePath, content);
filesWritten++;
totalBytes += content.length;
} catch (error) {
console.warn(`Failed to download file ${item.path}:`, error.message);
}
}
}
return {
success: true,
targetDir,
files: filesWritten,
bytes: totalBytes
};
} catch (error) {
console.error('Error downloading repository:', error);
return {
success: false,
error: error.message
};
}
}
}
module.exports = GitHubIntegrationService;

153
services/git-integration/src/services/github-oauth.js Normal file
View File

@ -0,0 +1,153 @@
// github-oauth.js
const { Octokit } = require('@octokit/rest');
const database = require('../config/database');
class GitHubOAuthService {
constructor() {
this.clientId = process.env.GITHUB_CLIENT_ID;
this.clientSecret = process.env.GITHUB_CLIENT_SECRET;
this.redirectUri = process.env.GITHUB_REDIRECT_URI || 'http://localhost:8010/api/github/auth/github/callback';
if (!this.clientId || !this.clientSecret) {
console.warn('GitHub OAuth not configured. Only public repositories will be accessible.');
}
}
// Generate GitHub OAuth URL
getAuthUrl(state) {
if (!this.clientId) {
throw new Error('GitHub OAuth not configured');
}
const params = new URLSearchParams({
client_id: this.clientId,
redirect_uri: this.redirectUri,
scope: 'repo,user:email',
state: state,
allow_signup: 'false'
});
return `https://github.com/login/oauth/authorize?${params.toString()}`;
}
// Exchange authorization code for access token
async exchangeCodeForToken(code) {
const response = await fetch('https://github.com/login/oauth/access_token', {
method: 'POST',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json',
},
body: JSON.stringify({
client_id: this.clientId,
client_secret: this.clientSecret,
code: code,
}),
});
const data = await response.json();
if (data.error) {
throw new Error(`OAuth error: ${data.error_description}`);
}
return data.access_token;
}
// Get user info from GitHub
async getUserInfo(accessToken) {
const octokit = new Octokit({ auth: accessToken });
const { data: user } = await octokit.users.getAuthenticated();
return user;
}
// Store GitHub token (no user ID)
async storeToken(accessToken, githubUser) {
const query = `
INSERT INTO github_user_tokens (access_token, github_username, github_user_id, scopes, expires_at)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (id)
DO UPDATE SET
access_token = $1,
github_username = $2,
github_user_id = $3,
scopes = $4,
expires_at = $5,
updated_at = NOW()
RETURNING *
`;
const result = await database.query(query, [
accessToken,
githubUser.login,
githubUser.id,
JSON.stringify(['repo', 'user:email']),
null
]);
return result.rows[0];
}
// Get stored token
async getToken() {
const query = 'SELECT * FROM github_user_tokens ORDER BY created_at DESC LIMIT 1';
const result = await database.query(query);
return result.rows[0];
}
// Create authenticated Octokit instance
async getAuthenticatedOctokit() {
const tokenRecord = await this.getToken();
if (!tokenRecord) {
return new Octokit({
userAgent: 'CodeNuk-GitIntegration/1.0.0',
});
}
return new Octokit({
auth: tokenRecord.access_token,
userAgent: 'CodeNuk-GitIntegration/1.0.0',
});
}
// Check repository access
async canAccessRepository(owner, repo) {
try {
const octokit = await this.getAuthenticatedOctokit();
await octokit.repos.get({ owner, repo });
return true;
} catch (error) {
if (error.status === 404) {
return false;
}
throw error;
}
}
// Revoke token
async revokeToken() {
const tokenRecord = await this.getToken();
if (tokenRecord) {
try {
await fetch(`https://api.github.com/applications/${this.clientId}/grant`, {
method: 'DELETE',
headers: {
'Authorization': `Basic ${Buffer.from(`${this.clientId}:${this.clientSecret}`).toString('base64')}`,
'Accept': 'application/vnd.github.v3+json',
},
body: JSON.stringify({
access_token: tokenRecord.access_token
})
});
} catch (error) {
console.error('Error revoking token on GitHub:', error);
}
await database.query('DELETE FROM github_user_tokens');
}
}
}
module.exports = GitHubOAuthService;