pipeline {
    agent any

    environment {
        SSH_CREDENTIALS = 'cloudtopiaa'
        REMOTE_SERVER = 'ubuntu@160.187.166.39'
        REMOTE_WORKSPACE = '/home/ubuntu'
        PROJECT_NAME = 'codenuk-backend-live'
        DEPLOY_PATH = '/home/ubuntu/codenuk-backend-live'
        GIT_CREDENTIALS = 'git-cred'
        REPO_URL = 'https://git.tech4biz.wiki/Tech4Biz-Services/codenuk-backend-live.git'
        EMAIL_RECIPIENT = 'jassim.mohammed@tech4biz.io, chandini.pachigunta@tech4biz.org'
        COMPOSE_FILE = 'docker-compose.yml'
        ENV_FILE = '.env'
    }

    options {
        timeout(time: 45, unit: 'MINUTES')
        retry(2)
        timestamps()
        buildDiscarder(logRotator(numToKeepStr: '10'))
    }

    stages {
        stage('Preparation') {
            steps {
                script {
                    echo "Starting ${PROJECT_NAME} microservices deployment pipeline"
                    echo "Server: ${REMOTE_SERVER}"
                    echo "Deploy Path: ${DEPLOY_PATH}"
                    echo "Docker Compose deployment with persistent data"
                }
            }
        }

        stage('Git Operations on Remote Server') {
            steps {
                script {
                    sshagent(credentials: [SSH_CREDENTIALS]) {
                        withCredentials([usernamePassword(credentialsId: GIT_CREDENTIALS, usernameVariable: 'GIT_USER', passwordVariable: 'GIT_PASS')]) {
                            sh """
                            ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
                                set -e

                                echo "Checking Git repo..."
                                if [ -d "${DEPLOY_PATH}/.git" ]; then
                                    echo "Pulling latest code..."
                                    cd ${DEPLOY_PATH}
                                    
                                    # Fix ownership issues
                                    sudo chown -R ubuntu:ubuntu ${DEPLOY_PATH}
                                    git config --global --add safe.directory ${DEPLOY_PATH}
                                    
                                    git reset --hard
                                    git clean -fd
                                    git config pull.rebase false
                                    git pull https://${GIT_USER}:${GIT_PASS}@git.tech4biz.wiki/Tech4Biz-Services/codenuk-backend-live.git main
                                else
                                    echo "Cloning fresh repo..."
                                    sudo rm -rf ${DEPLOY_PATH}
                                    sudo mkdir -p ${DEPLOY_PATH}
                                    sudo git clone https://${GIT_USER}:${GIT_PASS}@git.tech4biz.wiki/Tech4Biz-Services/codenuk-backend-live.git ${DEPLOY_PATH}
                                    sudo chown -R ubuntu:ubuntu ${DEPLOY_PATH}
                                    git config --global --add safe.directory ${DEPLOY_PATH}
                                fi

                                cd ${DEPLOY_PATH}
                                echo "Current commit: \$(git rev-parse HEAD)"
                                echo "Branch: \$(git branch --show-current)"
                            '
                            """
                        }
                    }
                }
            }
        }

        stage('Environment Setup') {
            steps {
                sshagent(credentials: [SSH_CREDENTIALS]) {
                    sh """
                    ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
                        set -e
                        cd ${DEPLOY_PATH}

                        # Verify Docker and Docker Compose are available
                        echo "Checking Docker installation..."
                        docker --version
                        docker compose version

                        # Create persistent data directories
                        echo "Creating persistent data directories..."
                        mkdir -p data/postgres
                        mkdir -p data/redis
                        mkdir -p data/mongodb
                        mkdir -p data/rabbitmq/data
                        mkdir -p data/rabbitmq/logs
                        mkdir -p data/neo4j/data
                        mkdir -p data/neo4j/logs
                        mkdir -p data/chromadb
                        mkdir -p data/n8n
                        mkdir -p generated-projects
                        mkdir -p generation-logs
                        mkdir -p dashboard-exports
                        mkdir -p logs/api-gateway

                        echo "Setting proper permissions..."
                        sudo chown -R ubuntu:ubuntu ${DEPLOY_PATH}
                        chmod +x ${DEPLOY_PATH}/scripts/* || echo "No scripts directory found"
                    '
                    """
                }
            }
        }

        stage('Pre-deployment Backup') {
            steps {
                sshagent(credentials: [SSH_CREDENTIALS]) {
                    sh """
                    ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
                        set -e
                        cd ${DEPLOY_PATH}

                        # Create backup directory with timestamp
                        BACKUP_DIR="backups/\$(date +%Y%m%d_%H%M%S)"
                        mkdir -p \$BACKUP_DIR

                        # Backup database volumes if they exist
                        if [ -d "data" ]; then
                            echo "Creating backup of persistent data..."
                            sudo tar -czf "\$BACKUP_DIR/data_backup.tar.gz" data/ || echo "Backup failed, continuing..."
                        fi

                        # Keep only last 5 backups
                        ls -t backups/ | tail -n +6 | xargs -r rm -rf
                        echo "Backup completed"
                    '
                    """
                }
            }
        }

        stage('Stop Services Gracefully') {
            steps {
                sshagent(credentials: [SSH_CREDENTIALS]) {
                    sh """
                    ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
                        set -e
                        cd ${DEPLOY_PATH}

                        echo "Stopping services gracefully..."
                        docker compose down --timeout 30 || echo "No running services found"
                        
                        # Clean up orphaned containers
                        docker container prune -f || true
                        
                        echo "Services stopped"
                    '
                    """
                }
            }
        }

        stage('Build Services') {
            steps {
                sshagent(credentials: [SSH_CREDENTIALS]) {
                    sh """
                    ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
                        set -e
                        cd ${DEPLOY_PATH}

                        echo "Building all services..."
                        docker compose build --no-cache --parallel
                        
                        echo "Listing built images..."
                        docker images | grep codenuk || echo "No codenuk images found"
                    '
                    """
                }
            }
        }

        stage('Start Infrastructure Services') {
            steps {
                sshagent(credentials: [SSH_CREDENTIALS]) {
                    sh """
                    ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
                        set -e
                        cd ${DEPLOY_PATH}

                        echo "Starting infrastructure services..."
                        # Start databases and infrastructure first
                        docker compose up -d postgres redis mongodb rabbitmq neo4j chromadb
                                          
                        echo "Infrastructure services status:"
                        docker compose ps
                    '
                    """
                    
                }
            }
        }


        stage('Deploy Application Services') {
            steps {
                sshagent(credentials: [SSH_CREDENTIALS]) {
                    sh """
                    ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
                        set -e
                        cd ${DEPLOY_PATH}

                        echo "Starting application services..."
                        docker compose up -d
                        
                        echo "Waiting for application services to be ready..."
                        sleep 60
                        
                        echo "All services status:"
                        docker compose ps
                        
                        echo "Checking service health..."
                        docker compose ps --format "table {{.Name}}\\t{{.Status}}\\t{{.Ports}}"
                    '
                    """
                }
            }
        }

        stage('Health Check') {
            steps {
                sshagent(credentials: [SSH_CREDENTIALS]) {
                    sh """
                    ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
                        set -e
                        cd ${DEPLOY_PATH}

                        echo "Performing comprehensive health check..."
                        
                        # Check if all services are running
                        FAILED_SERVICES=\$(docker compose ps --services --filter "status=exited")
                        if [ -n "\$FAILED_SERVICES" ]; then
                            echo "Failed services: \$FAILED_SERVICES"
                            docker compose logs \$FAILED_SERVICES
                            exit 1
                        fi

                        # Test database connectivity
                        echo "Testing database connectivity..."
                        docker compose exec -T postgres pg_isready -U pipeline_admin -d dev_pipeline || exit 1
                        
                        # Test Redis connectivity
                        echo "Testing Redis connectivity..."
                        docker compose exec -T redis redis-cli ping || exit 1
                        
                        # Test API Gateway endpoint (if available)
                        echo "Testing API Gateway health..."
                        timeout 30 bash -c "until curl -f https://dashboard.codenuk.com/health 2>/dev/null; do echo \\"Waiting for API Gateway...\\"; sleep 5; done" || echo "API Gateway health check timeout"
                        
                        echo "Container resource usage:"
                        docker stats --no-stream --format "table {{.Container}}\\t{{.CPUPerc}}\\t{{.MemUsage}}"
                        
                        echo "Volume usage:"
                        docker volume ls | grep -E "(postgres|redis|mongodb|rabbitmq|neo4j|chromadb|n8n)_data"
                        
                        echo "Network connectivity:"
                        docker network ls | grep pipeline_network
                        
                        echo "Deployment verification completed successfully"
                    '
                    """
                }
            }
        }

        stage('Service Logs Check') {
            steps {
                sshagent(credentials: [SSH_CREDENTIALS]) {
                    sh """
                    ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
                        set -e
                        cd ${DEPLOY_PATH}

                        echo "Checking recent logs for critical errors..."
                        
                        # Check for critical errors in all services
                        SERVICES=\$(docker compose ps --services --filter "status=running")
                        for service in \$SERVICES; do
                            echo "=== \$service logs ==="
                            docker compose logs --tail=10 \$service || echo "No logs for \$service"
                        done
                        
                        echo "Log check completed"
                    '
                    """
                }
            }
        }

        stage('Performance Verification') {
            steps {
                sshagent(credentials: [SSH_CREDENTIALS]) {
                    sh """
                    ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
                        set -e
                        cd ${DEPLOY_PATH}

                        echo "Performance and resource verification..."
                        
                        # Check system resources
                        echo "System resources:"
                        free -h
                        df -h
                        
                        # Check Docker system usage
                        echo "Docker system usage:"
                        docker system df
                        
                        # Verify persistent volumes
                        echo "Persistent volume verification:"
                        docker volume inspect \${PROJECT_NAME}_postgres_data > /dev/null 2>&1 && echo "PostgreSQL data volume: OK" || echo "PostgreSQL data volume: MISSING"
                        docker volume inspect \${PROJECT_NAME}_redis_data > /dev/null 2>&1 && echo "Redis data volume: OK" || echo "Redis data volume: MISSING"
                        docker volume inspect \${PROJECT_NAME}_mongodb_data > /dev/null 2>&1 && echo "MongoDB data volume: OK" || echo "MongoDB data volume: MISSING"
                        
                        echo "Performance verification completed"
                    '
                    """
                }
            }
        }
    }

    post {
        always {
            script {
                // Collect logs from remote server for debugging
                sshagent(credentials: [SSH_CREDENTIALS]) {
                    sh """
                    ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
                        cd ${DEPLOY_PATH}
                        echo "=== Final Service Status ==="
                        docker compose ps || echo "Could not get service status"
                        
                        echo "=== Docker System Info ==="
                        docker system df || echo "Could not get system info"
                    ' || echo "Failed to collect final status"
                    """
                }
            }
            cleanWs()
        }
        
        success {
            script {
                sshagent(credentials: [SSH_CREDENTIALS]) {
                    def serviceStatus = sh(
                        script: """
                        ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
                            cd ${DEPLOY_PATH}
                            docker compose ps --format "table {{.Name}}\\t{{.Status}}\\t{{.Ports}}"
                        '
                        """,
                        returnStdout: true
                    ).trim()

                    mail to: "${EMAIL_RECIPIENT}",
                         subject: "✅ Jenkins - ${PROJECT_NAME} Microservices Deployment Successful",
                         body: """The deployment of '${PROJECT_NAME}' microservices to ${REMOTE_SERVER} was successful.

Build Number: ${BUILD_NUMBER}
Build URL: ${BUILD_URL}
Deployment Time: ${new Date()}
Commit: ${env.GIT_COMMIT ?: 'Unknown'}

All microservices have been deployed using Docker Compose with persistent data volumes.

Service Status:
${serviceStatus}

Key Features:
- Database persistence maintained across deployments
- All services deployed and running
- Health checks passed
- Graceful service restart completed

Access URLs:
- API Gateway: http://${REMOTE_SERVER.split('@')[1]}:8000
- Dashboard: http://${REMOTE_SERVER.split('@')[1]}:8008
- N8N Workflow: http://${REMOTE_SERVER.split('@')[1]}:5678

For detailed logs, visit: ${BUILD_URL}console
"""
        }
        }
        }
        
        failure {
            script {
                def errorLogs = ""
                try {
                    sshagent(credentials: [SSH_CREDENTIALS]) {
                        errorLogs = sh(
                            script: """
                            ssh -o StrictHostKeyChecking=no ${REMOTE_SERVER} '
                                cd ${DEPLOY_PATH}
                                echo "=== Failed Services ==="
                                docker compose ps --filter "status=exited"
                                echo "=== Recent Error Logs ==="
                                docker compose logs --tail=50 2>&1 || echo "Could not fetch logs"
                            '
                            """,
                            returnStdout: true
                        ).trim()
                    }
                } catch (Exception e) {
                    errorLogs = "Could not fetch error logs: ${e.message}"
                }

                mail to: "${EMAIL_RECIPIENT}",
                     subject: "❌ Jenkins - ${PROJECT_NAME} Microservices Deployment Failed",
                     body: """Microservices deployment failed for '${PROJECT_NAME}' on ${REMOTE_SERVER}.

Build Number: ${BUILD_NUMBER}
Build URL: ${BUILD_URL}console
Failure Time: ${new Date()}

Error Details:
${errorLogs}

Please review the full logs at: ${BUILD_URL}console

Common troubleshooting steps:
1. Check Docker service status on the server
2. Verify .env file configuration
3. Check available disk space and memory
4. Review individual service logs
5. Ensure all required ports are available

To manually investigate:
ssh ${REMOTE_SERVER}
cd ${DEPLOY_PATH}
docker compose logs [service-name]
docker compose ps
"""
            }
        }
        
        unstable {
            mail to: "${EMAIL_RECIPIENT}",
                 subject: "⚠️ Jenkins - ${PROJECT_NAME} Deployment Unstable",
                 body: """The deployment of '${PROJECT_NAME}' completed but some issues were detected.

Build Number: ${BUILD_NUMBER}
Build URL: ${BUILD_URL}console
Time: ${new Date()}

Please review the logs and verify all services are functioning correctly.
"""
        }
    }
}