From c58b7d2687f3c50bc6f97813811dba2439e2b4da Mon Sep 17 00:00:00 2001 From: Chandini Date: Thu, 4 Sep 2025 15:59:15 +0530 Subject: [PATCH] backend changes --- Jenkinsfile | 462 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 462 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..b6de3a9 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,462 @@ +pipeline { + agent any + + environment { + SSH_CREDENTIALS = 'cloudtopiaa' + REMOTE_SERVER = 'ubuntu@160.187.166.39' + REMOTE_WORKSPACE = '/home/ubuntu' + PROJECT_NAME = 'codenuk-backend-live' + DEPLOY_PATH = '/home/ubuntu/codenuk-backend-live' + GIT_CREDENTIALS = 'git-cred' + REPO_URL = 'https://git.tech4biz.wiki/Tech4Biz-Services/codenuk-backend-live.git' + EMAIL_RECIPIENT = 'jassim.mohammed@tech4biz.io, chandini.pachigunta@tech4biz.org' + COMPOSE_FILE = 'docker-compose.yml' + ENV_FILE = '.env' + } + + options { + timeout(time: 45, unit: 'MINUTES') + retry(2) + timestamps() + buildDiscarder(logRotator(numToKeepStr: '10')) + } + + stages { + stage('Preparation') { + steps { + script { + echo "Starting ${PROJECT_NAME} microservices deployment pipeline" + echo "Server: ${REMOTE_SERVER}" + echo "Deploy Path: ${DEPLOY_PATH}" + echo "Docker Compose deployment with persistent data" + } + } + } + + stage('Git Operations on Remote Server') { + steps { + script { + sshagent(credentials: [SSH_CREDENTIALS]) { + withCredentials([usernamePassword(credentialsId: GIT_CREDENTIALS, usernameVariable: 'GIT_USER', passwordVariable: 'GIT_PASS')]) { + sh """ + ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} ' + set -e + + echo "Checking Git repo..." + if [ -d "${DEPLOY_PATH}/.git" ]; then + echo "Pulling latest code..." + cd ${DEPLOY_PATH} + + # Fix ownership issues + sudo chown -R ubuntu:ubuntu ${DEPLOY_PATH} + git config --global --add safe.directory ${DEPLOY_PATH} + + git reset --hard + git clean -fd + git config pull.rebase false + git pull https://${GIT_USER}:${GIT_PASS}@git.tech4biz.wiki/Tech4Biz-Services/codenuk-backend-live.git main + else + echo "Cloning fresh repo..." + sudo rm -rf ${DEPLOY_PATH} + sudo mkdir -p ${DEPLOY_PATH} + sudo git clone https://${GIT_USER}:${GIT_PASS}@git.tech4biz.wiki/Tech4Biz-Services/codenuk-backend-live.git ${DEPLOY_PATH} + sudo chown -R ubuntu:ubuntu ${DEPLOY_PATH} + git config --global --add safe.directory ${DEPLOY_PATH} + fi + + cd ${DEPLOY_PATH} + echo "Current commit: \$(git rev-parse HEAD)" + echo "Branch: \$(git branch --show-current)" + ' + """ + } + } + } + } + } + + stage('Environment Setup') { + steps { + sshagent(credentials: [SSH_CREDENTIALS]) { + sh """ + ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} ' + set -e + cd ${DEPLOY_PATH} + + # Verify Docker and Docker Compose are available + echo "Checking Docker installation..." + docker --version + docker compose version + + # Create persistent data directories + echo "Creating persistent data directories..." + mkdir -p data/postgres + mkdir -p data/redis + mkdir -p data/mongodb + mkdir -p data/rabbitmq/data + mkdir -p data/rabbitmq/logs + mkdir -p data/neo4j/data + mkdir -p data/neo4j/logs + mkdir -p data/chromadb + mkdir -p data/n8n + mkdir -p generated-projects + mkdir -p generation-logs + mkdir -p dashboard-exports + mkdir -p logs/api-gateway + + echo "Setting proper permissions..." + sudo chown -R ubuntu:ubuntu ${DEPLOY_PATH} + chmod +x ${DEPLOY_PATH}/scripts/* || echo "No scripts directory found" + ' + """ + } + } + } + + stage('Pre-deployment Backup') { + steps { + sshagent(credentials: [SSH_CREDENTIALS]) { + sh """ + ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} ' + set -e + cd ${DEPLOY_PATH} + + # Create backup directory with timestamp + BACKUP_DIR="backups/\$(date +%Y%m%d_%H%M%S)" + mkdir -p \$BACKUP_DIR + + # Backup database volumes if they exist + if [ -d "data" ]; then + echo "Creating backup of persistent data..." + sudo tar -czf "\$BACKUP_DIR/data_backup.tar.gz" data/ || echo "Backup failed, continuing..." + fi + + # Keep only last 5 backups + ls -t backups/ | tail -n +6 | xargs -r rm -rf + echo "Backup completed" + ' + """ + } + } + } + + stage('Stop Services Gracefully') { + steps { + sshagent(credentials: [SSH_CREDENTIALS]) { + sh """ + ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} ' + set -e + cd ${DEPLOY_PATH} + + echo "Stopping services gracefully..." + docker compose down --timeout 30 || echo "No running services found" + + # Clean up orphaned containers + docker container prune -f || true + + echo "Services stopped" + ' + """ + } + } + } + + stage('Build Services') { + steps { + sshagent(credentials: [SSH_CREDENTIALS]) { + sh """ + ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} ' + set -e + cd ${DEPLOY_PATH} + + echo "Building all services..." + docker compose build --no-cache --parallel + + echo "Listing built images..." + docker images | grep codenuk || echo "No codenuk images found" + ' + """ + } + } + } + + stage('Start Infrastructure Services') { + steps { + sshagent(credentials: [SSH_CREDENTIALS]) { + sh """ + ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} ' + set -e + cd ${DEPLOY_PATH} + + echo "Starting infrastructure services..." + # Start databases and infrastructure first + docker compose up -d postgres redis mongodb rabbitmq neo4j chromadb + + echo "Waiting for infrastructure services to be healthy..." + timeout 300 sh -c "until docker compose ps | grep -E \\"(postgres|redis)\\" | grep -q \\"healthy\\"; do echo \\"Waiting for infrastructure...\\"; sleep 10; done" + + echo "Infrastructure services status:" + docker compose ps + ' + """ + } + } + } + + + stage('Deploy Application Services') { + steps { + sshagent(credentials: [SSH_CREDENTIALS]) { + sh """ + ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} ' + set -e + cd ${DEPLOY_PATH} + + echo "Starting application services..." + docker compose up -d + + echo "Waiting for application services to be ready..." + sleep 60 + + echo "All services status:" + docker compose ps + + echo "Checking service health..." + docker compose ps --format "table {{.Name}}\\t{{.Status}}\\t{{.Ports}}" + ' + """ + } + } + } + + stage('Health Check') { + steps { + sshagent(credentials: [SSH_CREDENTIALS]) { + sh """ + ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} ' + set -e + cd ${DEPLOY_PATH} + + echo "Performing comprehensive health check..." + + # Check if all services are running + FAILED_SERVICES=\$(docker compose ps --services --filter "status=exited") + if [ -n "\$FAILED_SERVICES" ]; then + echo "Failed services: \$FAILED_SERVICES" + docker compose logs \$FAILED_SERVICES + exit 1 + fi + + # Test database connectivity + echo "Testing database connectivity..." + docker compose exec -T postgres pg_isready -U pipeline_admin -d dev_pipeline || exit 1 + + # Test Redis connectivity + echo "Testing Redis connectivity..." + docker compose exec -T redis redis-cli ping || exit 1 + + # Test API Gateway endpoint (if available) + echo "Testing API Gateway health..." + timeout 30 bash -c "until curl -f http://localhost:8000/health 2>/dev/null; do echo \\"Waiting for API Gateway...\\"; sleep 5; done" || echo "API Gateway health check timeout" + + echo "Container resource usage:" + docker stats --no-stream --format "table {{.Container}}\\t{{.CPUPerc}}\\t{{.MemUsage}}" + + echo "Volume usage:" + docker volume ls | grep -E "(postgres|redis|mongodb|rabbitmq|neo4j|chromadb|n8n)_data" + + echo "Network connectivity:" + docker network ls | grep pipeline_network + + echo "Deployment verification completed successfully" + ' + """ + } + } + } + + stage('Service Logs Check') { + steps { + sshagent(credentials: [SSH_CREDENTIALS]) { + sh """ + ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} ' + set -e + cd ${DEPLOY_PATH} + + echo "Checking recent logs for critical errors..." + + # Check for critical errors in all services + SERVICES=\$(docker compose ps --services --filter "status=running") + for service in \$SERVICES; do + echo "=== \$service logs ===" + docker compose logs --tail=10 \$service || echo "No logs for \$service" + done + + echo "Log check completed" + ' + """ + } + } + } + + stage('Performance Verification') { + steps { + sshagent(credentials: [SSH_CREDENTIALS]) { + sh """ + ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} ' + set -e + cd ${DEPLOY_PATH} + + echo "Performance and resource verification..." + + # Check system resources + echo "System resources:" + free -h + df -h + + # Check Docker system usage + echo "Docker system usage:" + docker system df + + # Verify persistent volumes + echo "Persistent volume verification:" + docker volume inspect \${PROJECT_NAME}_postgres_data > /dev/null 2>&1 && echo "PostgreSQL data volume: OK" || echo "PostgreSQL data volume: MISSING" + docker volume inspect \${PROJECT_NAME}_redis_data > /dev/null 2>&1 && echo "Redis data volume: OK" || echo "Redis data volume: MISSING" + docker volume inspect \${PROJECT_NAME}_mongodb_data > /dev/null 2>&1 && echo "MongoDB data volume: OK" || echo "MongoDB data volume: MISSING" + + echo "Performance verification completed" + ' + """ + } + } + } + } + + post { + always { + script { + // Collect logs from remote server for debugging + sshagent(credentials: [SSH_CREDENTIALS]) { + sh """ + ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} ' + cd ${DEPLOY_PATH} + echo "=== Final Service Status ===" + docker compose ps || echo "Could not get service status" + + echo "=== Docker System Info ===" + docker system df || echo "Could not get system info" + ' || echo "Failed to collect final status" + """ + } + } + cleanWs() + } + + success { + script { + sshagent(credentials: [SSH_CREDENTIALS]) { + def serviceStatus = sh( + script: """ + ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} ' + cd ${DEPLOY_PATH} + docker compose ps --format "table {{.Name}}\\t{{.Status}}\\t{{.Ports}}" + ' + """, + returnStdout: true + ).trim() + + mail to: "${EMAIL_RECIPIENT}", + subject: "✅ Jenkins - ${PROJECT_NAME} Microservices Deployment Successful", + body: """The deployment of '${PROJECT_NAME}' microservices to ${REMOTE_SERVER} was successful. + +Build Number: ${BUILD_NUMBER} +Build URL: ${BUILD_URL} +Deployment Time: ${new Date()} +Commit: ${env.GIT_COMMIT ?: 'Unknown'} + +All microservices have been deployed using Docker Compose with persistent data volumes. + +Service Status: +${serviceStatus} + +Key Features: +- Database persistence maintained across deployments +- All services deployed and running +- Health checks passed +- Graceful service restart completed + +Access URLs: +- API Gateway: http://${REMOTE_SERVER.split('@')[1]}:8000 +- Dashboard: http://${REMOTE_SERVER.split('@')[1]}:8008 +- N8N Workflow: http://${REMOTE_SERVER.split('@')[1]}:5678 + +For detailed logs, visit: ${BUILD_URL}console +""" + } + } + + failure { + script { + def errorLogs = "" + try { + sshagent(credentials: [SSH_CREDENTIALS]) { + errorLogs = sh( + script: """ + ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} ' + cd ${DEPLOY_PATH} + echo "=== Failed Services ===" + docker compose ps --filter "status=exited" + echo "=== Recent Error Logs ===" + docker compose logs --tail=50 2>&1 || echo "Could not fetch logs" + ' + """, + returnStdout: true + ).trim() + } + } catch (Exception e) { + errorLogs = "Could not fetch error logs: ${e.message}" + } + + mail to: "${EMAIL_RECIPIENT}", + subject: "❌ Jenkins - ${PROJECT_NAME} Microservices Deployment Failed", + body: """Microservices deployment failed for '${PROJECT_NAME}' on ${REMOTE_SERVER}. + +Build Number: ${BUILD_NUMBER} +Build URL: ${BUILD_URL}console +Failure Time: ${new Date()} + +Error Details: +${errorLogs} + +Please review the full logs at: ${BUILD_URL}console + +Common troubleshooting steps: +1. Check Docker service status on the server +2. Verify .env file configuration +3. Check available disk space and memory +4. Review individual service logs +5. Ensure all required ports are available + +To manually investigate: +ssh ${REMOTE_SERVER} +cd ${DEPLOY_PATH} +docker compose logs [service-name] +docker compose ps +""" + } + } + + unstable { + mail to: "${EMAIL_RECIPIENT}", + subject: "⚠️ Jenkins - ${PROJECT_NAME} Deployment Unstable", + body: """The deployment of '${PROJECT_NAME}' completed but some issues were detected. + +Build Number: ${BUILD_NUMBER} +Build URL: ${BUILD_URL}console +Time: ${new Date()} + +Please review the logs and verify all services are functioning correctly. +""" + } + } +} \ No newline at end of file