backend changes
This commit is contained in:
parent
56571c445e
commit
c58b7d2687
462
Jenkinsfile
vendored
Normal file
462
Jenkinsfile
vendored
Normal file
@ -0,0 +1,462 @@
|
||||
pipeline {
|
||||
agent any
|
||||
|
||||
environment {
|
||||
SSH_CREDENTIALS = 'cloudtopiaa'
|
||||
REMOTE_SERVER = 'ubuntu@160.187.166.39'
|
||||
REMOTE_WORKSPACE = '/home/ubuntu'
|
||||
PROJECT_NAME = 'codenuk-backend-live'
|
||||
DEPLOY_PATH = '/home/ubuntu/codenuk-backend-live'
|
||||
GIT_CREDENTIALS = 'git-cred'
|
||||
REPO_URL = 'https://git.tech4biz.wiki/Tech4Biz-Services/codenuk-backend-live.git'
|
||||
EMAIL_RECIPIENT = 'jassim.mohammed@tech4biz.io, chandini.pachigunta@tech4biz.org'
|
||||
COMPOSE_FILE = 'docker-compose.yml'
|
||||
ENV_FILE = '.env'
|
||||
}
|
||||
|
||||
options {
|
||||
timeout(time: 45, unit: 'MINUTES')
|
||||
retry(2)
|
||||
timestamps()
|
||||
buildDiscarder(logRotator(numToKeepStr: '10'))
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Preparation') {
|
||||
steps {
|
||||
script {
|
||||
echo "Starting ${PROJECT_NAME} microservices deployment pipeline"
|
||||
echo "Server: ${REMOTE_SERVER}"
|
||||
echo "Deploy Path: ${DEPLOY_PATH}"
|
||||
echo "Docker Compose deployment with persistent data"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Git Operations on Remote Server') {
|
||||
steps {
|
||||
script {
|
||||
sshagent(credentials: [SSH_CREDENTIALS]) {
|
||||
withCredentials([usernamePassword(credentialsId: GIT_CREDENTIALS, usernameVariable: 'GIT_USER', passwordVariable: 'GIT_PASS')]) {
|
||||
sh """
|
||||
ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
|
||||
set -e
|
||||
|
||||
echo "Checking Git repo..."
|
||||
if [ -d "${DEPLOY_PATH}/.git" ]; then
|
||||
echo "Pulling latest code..."
|
||||
cd ${DEPLOY_PATH}
|
||||
|
||||
# Fix ownership issues
|
||||
sudo chown -R ubuntu:ubuntu ${DEPLOY_PATH}
|
||||
git config --global --add safe.directory ${DEPLOY_PATH}
|
||||
|
||||
git reset --hard
|
||||
git clean -fd
|
||||
git config pull.rebase false
|
||||
git pull https://${GIT_USER}:${GIT_PASS}@git.tech4biz.wiki/Tech4Biz-Services/codenuk-backend-live.git main
|
||||
else
|
||||
echo "Cloning fresh repo..."
|
||||
sudo rm -rf ${DEPLOY_PATH}
|
||||
sudo mkdir -p ${DEPLOY_PATH}
|
||||
sudo git clone https://${GIT_USER}:${GIT_PASS}@git.tech4biz.wiki/Tech4Biz-Services/codenuk-backend-live.git ${DEPLOY_PATH}
|
||||
sudo chown -R ubuntu:ubuntu ${DEPLOY_PATH}
|
||||
git config --global --add safe.directory ${DEPLOY_PATH}
|
||||
fi
|
||||
|
||||
cd ${DEPLOY_PATH}
|
||||
echo "Current commit: \$(git rev-parse HEAD)"
|
||||
echo "Branch: \$(git branch --show-current)"
|
||||
'
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Environment Setup') {
|
||||
steps {
|
||||
sshagent(credentials: [SSH_CREDENTIALS]) {
|
||||
sh """
|
||||
ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
|
||||
set -e
|
||||
cd ${DEPLOY_PATH}
|
||||
|
||||
# Verify Docker and Docker Compose are available
|
||||
echo "Checking Docker installation..."
|
||||
docker --version
|
||||
docker compose version
|
||||
|
||||
# Create persistent data directories
|
||||
echo "Creating persistent data directories..."
|
||||
mkdir -p data/postgres
|
||||
mkdir -p data/redis
|
||||
mkdir -p data/mongodb
|
||||
mkdir -p data/rabbitmq/data
|
||||
mkdir -p data/rabbitmq/logs
|
||||
mkdir -p data/neo4j/data
|
||||
mkdir -p data/neo4j/logs
|
||||
mkdir -p data/chromadb
|
||||
mkdir -p data/n8n
|
||||
mkdir -p generated-projects
|
||||
mkdir -p generation-logs
|
||||
mkdir -p dashboard-exports
|
||||
mkdir -p logs/api-gateway
|
||||
|
||||
echo "Setting proper permissions..."
|
||||
sudo chown -R ubuntu:ubuntu ${DEPLOY_PATH}
|
||||
chmod +x ${DEPLOY_PATH}/scripts/* || echo "No scripts directory found"
|
||||
'
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Pre-deployment Backup') {
|
||||
steps {
|
||||
sshagent(credentials: [SSH_CREDENTIALS]) {
|
||||
sh """
|
||||
ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
|
||||
set -e
|
||||
cd ${DEPLOY_PATH}
|
||||
|
||||
# Create backup directory with timestamp
|
||||
BACKUP_DIR="backups/\$(date +%Y%m%d_%H%M%S)"
|
||||
mkdir -p \$BACKUP_DIR
|
||||
|
||||
# Backup database volumes if they exist
|
||||
if [ -d "data" ]; then
|
||||
echo "Creating backup of persistent data..."
|
||||
sudo tar -czf "\$BACKUP_DIR/data_backup.tar.gz" data/ || echo "Backup failed, continuing..."
|
||||
fi
|
||||
|
||||
# Keep only last 5 backups
|
||||
ls -t backups/ | tail -n +6 | xargs -r rm -rf
|
||||
echo "Backup completed"
|
||||
'
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Stop Services Gracefully') {
|
||||
steps {
|
||||
sshagent(credentials: [SSH_CREDENTIALS]) {
|
||||
sh """
|
||||
ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
|
||||
set -e
|
||||
cd ${DEPLOY_PATH}
|
||||
|
||||
echo "Stopping services gracefully..."
|
||||
docker compose down --timeout 30 || echo "No running services found"
|
||||
|
||||
# Clean up orphaned containers
|
||||
docker container prune -f || true
|
||||
|
||||
echo "Services stopped"
|
||||
'
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Build Services') {
|
||||
steps {
|
||||
sshagent(credentials: [SSH_CREDENTIALS]) {
|
||||
sh """
|
||||
ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
|
||||
set -e
|
||||
cd ${DEPLOY_PATH}
|
||||
|
||||
echo "Building all services..."
|
||||
docker compose build --no-cache --parallel
|
||||
|
||||
echo "Listing built images..."
|
||||
docker images | grep codenuk || echo "No codenuk images found"
|
||||
'
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Start Infrastructure Services') {
|
||||
steps {
|
||||
sshagent(credentials: [SSH_CREDENTIALS]) {
|
||||
sh """
|
||||
ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
|
||||
set -e
|
||||
cd ${DEPLOY_PATH}
|
||||
|
||||
echo "Starting infrastructure services..."
|
||||
# Start databases and infrastructure first
|
||||
docker compose up -d postgres redis mongodb rabbitmq neo4j chromadb
|
||||
|
||||
echo "Waiting for infrastructure services to be healthy..."
|
||||
timeout 300 sh -c "until docker compose ps | grep -E \\"(postgres|redis)\\" | grep -q \\"healthy\\"; do echo \\"Waiting for infrastructure...\\"; sleep 10; done"
|
||||
|
||||
echo "Infrastructure services status:"
|
||||
docker compose ps
|
||||
'
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
stage('Deploy Application Services') {
|
||||
steps {
|
||||
sshagent(credentials: [SSH_CREDENTIALS]) {
|
||||
sh """
|
||||
ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
|
||||
set -e
|
||||
cd ${DEPLOY_PATH}
|
||||
|
||||
echo "Starting application services..."
|
||||
docker compose up -d
|
||||
|
||||
echo "Waiting for application services to be ready..."
|
||||
sleep 60
|
||||
|
||||
echo "All services status:"
|
||||
docker compose ps
|
||||
|
||||
echo "Checking service health..."
|
||||
docker compose ps --format "table {{.Name}}\\t{{.Status}}\\t{{.Ports}}"
|
||||
'
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Health Check') {
|
||||
steps {
|
||||
sshagent(credentials: [SSH_CREDENTIALS]) {
|
||||
sh """
|
||||
ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
|
||||
set -e
|
||||
cd ${DEPLOY_PATH}
|
||||
|
||||
echo "Performing comprehensive health check..."
|
||||
|
||||
# Check if all services are running
|
||||
FAILED_SERVICES=\$(docker compose ps --services --filter "status=exited")
|
||||
if [ -n "\$FAILED_SERVICES" ]; then
|
||||
echo "Failed services: \$FAILED_SERVICES"
|
||||
docker compose logs \$FAILED_SERVICES
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Test database connectivity
|
||||
echo "Testing database connectivity..."
|
||||
docker compose exec -T postgres pg_isready -U pipeline_admin -d dev_pipeline || exit 1
|
||||
|
||||
# Test Redis connectivity
|
||||
echo "Testing Redis connectivity..."
|
||||
docker compose exec -T redis redis-cli ping || exit 1
|
||||
|
||||
# Test API Gateway endpoint (if available)
|
||||
echo "Testing API Gateway health..."
|
||||
timeout 30 bash -c "until curl -f http://localhost:8000/health 2>/dev/null; do echo \\"Waiting for API Gateway...\\"; sleep 5; done" || echo "API Gateway health check timeout"
|
||||
|
||||
echo "Container resource usage:"
|
||||
docker stats --no-stream --format "table {{.Container}}\\t{{.CPUPerc}}\\t{{.MemUsage}}"
|
||||
|
||||
echo "Volume usage:"
|
||||
docker volume ls | grep -E "(postgres|redis|mongodb|rabbitmq|neo4j|chromadb|n8n)_data"
|
||||
|
||||
echo "Network connectivity:"
|
||||
docker network ls | grep pipeline_network
|
||||
|
||||
echo "Deployment verification completed successfully"
|
||||
'
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Service Logs Check') {
|
||||
steps {
|
||||
sshagent(credentials: [SSH_CREDENTIALS]) {
|
||||
sh """
|
||||
ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
|
||||
set -e
|
||||
cd ${DEPLOY_PATH}
|
||||
|
||||
echo "Checking recent logs for critical errors..."
|
||||
|
||||
# Check for critical errors in all services
|
||||
SERVICES=\$(docker compose ps --services --filter "status=running")
|
||||
for service in \$SERVICES; do
|
||||
echo "=== \$service logs ==="
|
||||
docker compose logs --tail=10 \$service || echo "No logs for \$service"
|
||||
done
|
||||
|
||||
echo "Log check completed"
|
||||
'
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Performance Verification') {
|
||||
steps {
|
||||
sshagent(credentials: [SSH_CREDENTIALS]) {
|
||||
sh """
|
||||
ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
|
||||
set -e
|
||||
cd ${DEPLOY_PATH}
|
||||
|
||||
echo "Performance and resource verification..."
|
||||
|
||||
# Check system resources
|
||||
echo "System resources:"
|
||||
free -h
|
||||
df -h
|
||||
|
||||
# Check Docker system usage
|
||||
echo "Docker system usage:"
|
||||
docker system df
|
||||
|
||||
# Verify persistent volumes
|
||||
echo "Persistent volume verification:"
|
||||
docker volume inspect \${PROJECT_NAME}_postgres_data > /dev/null 2>&1 && echo "PostgreSQL data volume: OK" || echo "PostgreSQL data volume: MISSING"
|
||||
docker volume inspect \${PROJECT_NAME}_redis_data > /dev/null 2>&1 && echo "Redis data volume: OK" || echo "Redis data volume: MISSING"
|
||||
docker volume inspect \${PROJECT_NAME}_mongodb_data > /dev/null 2>&1 && echo "MongoDB data volume: OK" || echo "MongoDB data volume: MISSING"
|
||||
|
||||
echo "Performance verification completed"
|
||||
'
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
post {
|
||||
always {
|
||||
script {
|
||||
// Collect logs from remote server for debugging
|
||||
sshagent(credentials: [SSH_CREDENTIALS]) {
|
||||
sh """
|
||||
ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
|
||||
cd ${DEPLOY_PATH}
|
||||
echo "=== Final Service Status ==="
|
||||
docker compose ps || echo "Could not get service status"
|
||||
|
||||
echo "=== Docker System Info ==="
|
||||
docker system df || echo "Could not get system info"
|
||||
' || echo "Failed to collect final status"
|
||||
"""
|
||||
}
|
||||
}
|
||||
cleanWs()
|
||||
}
|
||||
|
||||
success {
|
||||
script {
|
||||
sshagent(credentials: [SSH_CREDENTIALS]) {
|
||||
def serviceStatus = sh(
|
||||
script: """
|
||||
ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
|
||||
cd ${DEPLOY_PATH}
|
||||
docker compose ps --format "table {{.Name}}\\t{{.Status}}\\t{{.Ports}}"
|
||||
'
|
||||
""",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
mail to: "${EMAIL_RECIPIENT}",
|
||||
subject: "✅ Jenkins - ${PROJECT_NAME} Microservices Deployment Successful",
|
||||
body: """The deployment of '${PROJECT_NAME}' microservices to ${REMOTE_SERVER} was successful.
|
||||
|
||||
Build Number: ${BUILD_NUMBER}
|
||||
Build URL: ${BUILD_URL}
|
||||
Deployment Time: ${new Date()}
|
||||
Commit: ${env.GIT_COMMIT ?: 'Unknown'}
|
||||
|
||||
All microservices have been deployed using Docker Compose with persistent data volumes.
|
||||
|
||||
Service Status:
|
||||
${serviceStatus}
|
||||
|
||||
Key Features:
|
||||
- Database persistence maintained across deployments
|
||||
- All services deployed and running
|
||||
- Health checks passed
|
||||
- Graceful service restart completed
|
||||
|
||||
Access URLs:
|
||||
- API Gateway: http://${REMOTE_SERVER.split('@')[1]}:8000
|
||||
- Dashboard: http://${REMOTE_SERVER.split('@')[1]}:8008
|
||||
- N8N Workflow: http://${REMOTE_SERVER.split('@')[1]}:5678
|
||||
|
||||
For detailed logs, visit: ${BUILD_URL}console
|
||||
"""
|
||||
}
|
||||
}
|
||||
|
||||
failure {
|
||||
script {
|
||||
def errorLogs = ""
|
||||
try {
|
||||
sshagent(credentials: [SSH_CREDENTIALS]) {
|
||||
errorLogs = sh(
|
||||
script: """
|
||||
ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
|
||||
cd ${DEPLOY_PATH}
|
||||
echo "=== Failed Services ==="
|
||||
docker compose ps --filter "status=exited"
|
||||
echo "=== Recent Error Logs ==="
|
||||
docker compose logs --tail=50 2>&1 || echo "Could not fetch logs"
|
||||
'
|
||||
""",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
}
|
||||
} catch (Exception e) {
|
||||
errorLogs = "Could not fetch error logs: ${e.message}"
|
||||
}
|
||||
|
||||
mail to: "${EMAIL_RECIPIENT}",
|
||||
subject: "❌ Jenkins - ${PROJECT_NAME} Microservices Deployment Failed",
|
||||
body: """Microservices deployment failed for '${PROJECT_NAME}' on ${REMOTE_SERVER}.
|
||||
|
||||
Build Number: ${BUILD_NUMBER}
|
||||
Build URL: ${BUILD_URL}console
|
||||
Failure Time: ${new Date()}
|
||||
|
||||
Error Details:
|
||||
${errorLogs}
|
||||
|
||||
Please review the full logs at: ${BUILD_URL}console
|
||||
|
||||
Common troubleshooting steps:
|
||||
1. Check Docker service status on the server
|
||||
2. Verify .env file configuration
|
||||
3. Check available disk space and memory
|
||||
4. Review individual service logs
|
||||
5. Ensure all required ports are available
|
||||
|
||||
To manually investigate:
|
||||
ssh ${REMOTE_SERVER}
|
||||
cd ${DEPLOY_PATH}
|
||||
docker compose logs [service-name]
|
||||
docker compose ps
|
||||
"""
|
||||
}
|
||||
}
|
||||
|
||||
unstable {
|
||||
mail to: "${EMAIL_RECIPIENT}",
|
||||
subject: "⚠️ Jenkins - ${PROJECT_NAME} Deployment Unstable",
|
||||
body: """The deployment of '${PROJECT_NAME}' completed but some issues were detected.
|
||||
|
||||
Build Number: ${BUILD_NUMBER}
|
||||
Build URL: ${BUILD_URL}console
|
||||
Time: ${new Date()}
|
||||
|
||||
Please review the logs and verify all services are functioning correctly.
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user