462 lines
18 KiB
TypeScript
462 lines
18 KiB
TypeScript
import { SecretManagerServiceClient } from '@google-cloud/secret-manager';
|
||
import logger from '@utils/logger';
|
||
import path from 'path';
|
||
import fs from 'fs';
|
||
|
||
/**
|
||
* Google Secret Manager Service
|
||
*
|
||
* This service loads secrets from Google Cloud Secret Manager and merges them
|
||
* with the current process.env, allowing for minimal changes to existing code.
|
||
*
|
||
* Configuration:
|
||
* - Set USE_GOOGLE_SECRET_MANAGER=true to enable (default: false)
|
||
* - Set GCP_PROJECT_ID to your Google Cloud Project ID
|
||
* - Set GCP_SECRET_PREFIX to prefix all secret names (optional, default: empty)
|
||
* - Set GCP_SECRET_MAP_FILE to map secret names to env vars (optional, see format below)
|
||
*
|
||
* Secret Name Mapping:
|
||
* - By default, secrets are mapped to env vars with the same name (uppercase)
|
||
* - Example: secret "db_password" -> process.env.DB_PASSWORD
|
||
* - Use GCP_SECRET_MAP_FILE to provide a JSON mapping file for custom mappings
|
||
*
|
||
* Fallback:
|
||
* - If Google Secret Manager is disabled or fails, falls back to .env file
|
||
* - Existing environment variables take precedence unless explicitly overridden
|
||
*/
|
||
class GoogleSecretManagerService {
|
||
private client: SecretManagerServiceClient | null = null;
|
||
private projectId: string;
|
||
private secretPrefix: string;
|
||
private secretMap: Record<string, string> = {};
|
||
private isInitialized: boolean = false;
|
||
private isLoaded: boolean = false;
|
||
|
||
constructor() {
|
||
this.projectId = process.env.GCP_PROJECT_ID || '';
|
||
this.secretPrefix = process.env.GCP_SECRET_PREFIX || '';
|
||
|
||
// Load secret mapping file if provided
|
||
const mapFile = process.env.GCP_SECRET_MAP_FILE;
|
||
if (mapFile) {
|
||
try {
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
const mapFilePath = path.resolve(mapFile);
|
||
if (fs.existsSync(mapFilePath)) {
|
||
const mapContent = fs.readFileSync(mapFilePath, 'utf8');
|
||
this.secretMap = JSON.parse(mapContent);
|
||
logger.info(`[Secret Manager] Loaded secret mapping from ${mapFilePath}`);
|
||
} else {
|
||
logger.warn(`[Secret Manager] Secret mapping file not found: ${mapFilePath}`);
|
||
}
|
||
} catch (error: any) {
|
||
logger.warn(`[Secret Manager] Failed to load secret mapping file: ${error.message}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Initialize Google Secret Manager client
|
||
*/
|
||
private async initializeClient(): Promise<void> {
|
||
if (this.client) {
|
||
return;
|
||
}
|
||
|
||
try {
|
||
const keyFilePath = process.env.GCP_KEY_FILE || '';
|
||
let originalCredentialsEnv: string | undefined;
|
||
|
||
// If GCP_KEY_FILE is specified, set GOOGLE_APPLICATION_CREDENTIALS temporarily
|
||
if (keyFilePath) {
|
||
const resolvedKeyPath = path.isAbsolute(keyFilePath)
|
||
? keyFilePath
|
||
: path.resolve(process.cwd(), keyFilePath);
|
||
|
||
if (fs.existsSync(resolvedKeyPath)) {
|
||
// Save original value if it exists
|
||
originalCredentialsEnv = process.env.GOOGLE_APPLICATION_CREDENTIALS;
|
||
// Set it to use the key file
|
||
process.env.GOOGLE_APPLICATION_CREDENTIALS = resolvedKeyPath;
|
||
logger.debug(`[Secret Manager] Using key file: ${resolvedKeyPath}`);
|
||
} else {
|
||
logger.warn(`[Secret Manager] Key file not found at: ${resolvedKeyPath}`);
|
||
logger.warn('[Secret Manager] Will attempt to use Application Default Credentials');
|
||
}
|
||
}
|
||
|
||
try {
|
||
// Create client - it will use GOOGLE_APPLICATION_CREDENTIALS if set
|
||
this.client = new SecretManagerServiceClient({
|
||
projectId: this.projectId,
|
||
});
|
||
logger.info('[Secret Manager] ✅ Google Secret Manager client initialized');
|
||
} finally {
|
||
// Restore original GOOGLE_APPLICATION_CREDENTIALS if we changed it
|
||
if (keyFilePath && originalCredentialsEnv !== undefined) {
|
||
if (originalCredentialsEnv) {
|
||
process.env.GOOGLE_APPLICATION_CREDENTIALS = originalCredentialsEnv;
|
||
} else {
|
||
delete process.env.GOOGLE_APPLICATION_CREDENTIALS;
|
||
}
|
||
}
|
||
}
|
||
} catch (error: any) {
|
||
logger.error('[Secret Manager] Failed to initialize client:', error);
|
||
if (error.message?.includes('Could not load the default credentials')) {
|
||
logger.error('[Secret Manager] Authentication failed. Please check:');
|
||
logger.error('[Secret Manager] 1. GCP_KEY_FILE points to a valid service account JSON file');
|
||
logger.error('[Secret Manager] 2. Service account has Secret Manager Secret Accessor role');
|
||
logger.error('[Secret Manager] 3. The key file is readable and not corrupted');
|
||
}
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Get secret value from Google Secret Manager
|
||
*/
|
||
private async getSecret(secretName: string): Promise<string | null> {
|
||
if (!this.client || !this.projectId) {
|
||
return null;
|
||
}
|
||
|
||
const fullSecretName = this.secretPrefix
|
||
? `${this.secretPrefix}-${secretName}`
|
||
: secretName;
|
||
|
||
const name = `projects/${this.projectId}/secrets/${fullSecretName}/versions/latest`;
|
||
|
||
try {
|
||
const [version] = await this.client.accessSecretVersion({ name });
|
||
|
||
if (version.payload?.data) {
|
||
const secretValue = version.payload.data.toString();
|
||
logger.debug(`[Secret Manager] ✅ Fetched secret: ${fullSecretName}`);
|
||
return secretValue;
|
||
}
|
||
|
||
logger.debug(`[Secret Manager] Secret ${fullSecretName} exists but payload is empty`);
|
||
return null;
|
||
} catch (error: any) {
|
||
const isOktaSecret = /OKTA_/i.test(secretName);
|
||
const logLevel = isOktaSecret ? logger.info.bind(logger) : logger.debug.bind(logger);
|
||
|
||
// Handle "not found" errors (code 5 = NOT_FOUND)
|
||
if (error.code === 5 || error.code === 'NOT_FOUND' || error.message?.includes('not found')) {
|
||
logLevel(`[Secret Manager] Secret not found: ${fullSecretName} (project: ${this.projectId})`);
|
||
if (isOktaSecret) {
|
||
logger.info(`[Secret Manager] Searched path: projects/${this.projectId}/secrets/${fullSecretName}/versions/latest`);
|
||
}
|
||
return null;
|
||
}
|
||
|
||
// Handle permission errors (code 7 = PERMISSION_DENIED)
|
||
if (error.code === 7 || error.code === 'PERMISSION_DENIED' || error.message?.includes('Permission denied')) {
|
||
logger.warn(`[Secret Manager] ❌ Permission denied for secret '${fullSecretName}'`);
|
||
if (isOktaSecret) {
|
||
logger.warn(`[Secret Manager] This is an OKTA secret - check service account permissions`);
|
||
}
|
||
logger.warn(`[Secret Manager] Service account needs 'Secret Manager Secret Accessor' role`);
|
||
logger.warn(`[Secret Manager] To grant access, run:`);
|
||
logger.warn(`[Secret Manager] gcloud secrets add-iam-policy-binding ${fullSecretName} \\`);
|
||
logger.warn(`[Secret Manager] --member="serviceAccount:YOUR_SERVICE_ACCOUNT@${this.projectId}.iam.gserviceaccount.com" \\`);
|
||
logger.warn(`[Secret Manager] --role="roles/secretmanager.secretAccessor" \\`);
|
||
logger.warn(`[Secret Manager] --project=${this.projectId}`);
|
||
return null;
|
||
}
|
||
|
||
// Log full error details for debugging (info level for OKTA secrets)
|
||
const errorLogLevel = isOktaSecret ? logger.warn.bind(logger) : logger.warn.bind(logger);
|
||
errorLogLevel(`[Secret Manager] Failed to fetch secret '${fullSecretName}' (project: ${this.projectId})`);
|
||
errorLogLevel(`[Secret Manager] Error code: ${error.code || 'unknown'}, Message: ${error.message || 'no message'}`);
|
||
if (error.details) {
|
||
errorLogLevel(`[Secret Manager] Error details: ${JSON.stringify(error.details)}`);
|
||
}
|
||
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Map secret name to environment variable name
|
||
*/
|
||
private getEnvVarName(secretName: string): string {
|
||
// Check if there's a custom mapping
|
||
if (this.secretMap[secretName]) {
|
||
return this.secretMap[secretName];
|
||
}
|
||
|
||
// Default: convert secret name to uppercase and replace hyphens with underscores
|
||
// Example: "db-password" -> "DB_PASSWORD", "JWT_SECRET" -> "JWT_SECRET"
|
||
return secretName.toUpperCase().replace(/-/g, '_');
|
||
}
|
||
|
||
/**
|
||
* Load all secrets from Google Secret Manager and merge with process.env
|
||
*
|
||
* @param secretNames - Array of secret names to load. If not provided,
|
||
* will attempt to load common secret names based on env.example
|
||
*/
|
||
async loadSecrets(secretNames?: string[]): Promise<void> {
|
||
const useSecretManager = process.env.USE_GOOGLE_SECRET_MANAGER === 'true';
|
||
|
||
if (!useSecretManager) {
|
||
logger.debug('[Secret Manager] Google Secret Manager is disabled (USE_GOOGLE_SECRET_MANAGER != true)');
|
||
return;
|
||
}
|
||
|
||
if (this.isLoaded && !secretNames) {
|
||
logger.debug('[Secret Manager] ℹ️ Secrets already loaded in this process, skipping.');
|
||
return;
|
||
}
|
||
|
||
if (!this.projectId) {
|
||
this.projectId = process.env.GCP_PROJECT_ID || '';
|
||
}
|
||
|
||
if (!this.projectId) {
|
||
logger.warn('[Secret Manager] GCP_PROJECT_ID not set, skipping Google Secret Manager');
|
||
return;
|
||
}
|
||
|
||
try {
|
||
await this.initializeClient();
|
||
|
||
// Default list of secrets to load if not provided
|
||
const secretsToLoad = secretNames || this.getDefaultSecretNames();
|
||
|
||
logger.info(`[Secret Manager] Loading ${secretsToLoad.length} secrets from Google Secret Manager (project: ${this.projectId})...`);
|
||
if (this.secretPrefix) {
|
||
logger.info(`[Secret Manager] Using secret prefix: ${this.secretPrefix}`);
|
||
}
|
||
|
||
const loadedSecrets: Record<string, string> = {};
|
||
const notFoundSecrets: string[] = [];
|
||
let loadedCount = 0;
|
||
|
||
// Log OKTA and EMAIL secrets specifically if they're in the list
|
||
const oktaSecrets = secretsToLoad.filter(name => /^OKTA_/i.test(name));
|
||
const emailSecrets = secretsToLoad.filter(name => /^EMAIL_|^SMTP_/i.test(name));
|
||
if (oktaSecrets.length > 0) {
|
||
logger.info(`[Secret Manager] 🔍 Attempting to load OKTA secrets: ${oktaSecrets.join(', ')}`);
|
||
}
|
||
if (emailSecrets.length > 0) {
|
||
logger.info(`[Secret Manager] 📧 Attempting to load EMAIL secrets: ${emailSecrets.join(', ')}`);
|
||
}
|
||
|
||
// Load each secret
|
||
const uatSecrets = ['OKTA_CLIENT_ID', 'OKTA_CLIENT_SECRET', 'OKTA_API_TOKEN', 'OKTA_DOMAIN', 'DB_PASSWORD'];
|
||
const isUat = process.env.NODE_ENV === 'uat';
|
||
|
||
for (const secretName of secretsToLoad) {
|
||
// Handle UAT-specific secret names if in UAT environment
|
||
let secretNameToFetch = secretName;
|
||
if (isUat && uatSecrets.includes(secretName)) {
|
||
secretNameToFetch = `${secretName}_UAT`;
|
||
logger.info(`[Secret Manager] UAT mode: Fetching source ${secretNameToFetch} for ${secretName}`);
|
||
}
|
||
|
||
const fullSecretName = this.secretPrefix
|
||
? `${this.secretPrefix}-${secretNameToFetch}`
|
||
: secretNameToFetch;
|
||
|
||
const rawValue = await this.getSecret(secretNameToFetch);
|
||
const secretValue = rawValue ? rawValue.trim() : null;
|
||
|
||
if (secretValue !== null) {
|
||
const envVarName = this.getEnvVarName(secretName);
|
||
loadedSecrets[envVarName] = secretValue;
|
||
loadedCount++;
|
||
|
||
logger.info(`[Secret Manager] ✅ Loaded: ${secretNameToFetch} -> process.env.${envVarName}`);
|
||
} else {
|
||
// Track which secrets weren't found for better logging
|
||
notFoundSecrets.push(fullSecretName);
|
||
logger.warn(`[Secret Manager] ❌ Not found: ${secretNameToFetch} (searched as: ${fullSecretName})`);
|
||
}
|
||
}
|
||
|
||
// Merge secrets into process.env (only override if secret exists)
|
||
// Log when overriding existing env vars vs setting new ones
|
||
for (const [envVar, value] of Object.entries(loadedSecrets)) {
|
||
const existingValue = process.env[envVar];
|
||
const isAlreadySet = existingValue !== undefined && existingValue !== '';
|
||
|
||
if (isAlreadySet) {
|
||
logger.debug(`[Secret Manager] ℹ️ Skipping ${envVar}: Already set in local environment`);
|
||
continue;
|
||
}
|
||
|
||
process.env[envVar] = value;
|
||
logger.debug(`[Secret Manager] ✨ Set new env var: ${envVar} (from Secret Manager)`);
|
||
}
|
||
|
||
this.isLoaded = true;
|
||
logger.info(`[Secret Manager] ✅ Successfully loaded ${loadedCount}/${secretsToLoad.length} secrets`);
|
||
|
||
if (loadedCount > 0) {
|
||
const loadedVars = Object.keys(loadedSecrets);
|
||
logger.info(`[Secret Manager] Loaded env vars: ${loadedVars.join(', ')}`);
|
||
} else {
|
||
logger.warn(`[Secret Manager] ⚠️ No secrets were loaded. This might be normal if secrets don't exist yet.`);
|
||
logger.info(`[Secret Manager] To create secrets, use: gcloud secrets create SECRET_NAME --data-file=- --project=${this.projectId}`);
|
||
if (notFoundSecrets.length > 0 && notFoundSecrets.length <= 5) {
|
||
logger.info(`[Secret Manager] Example secrets to create: ${notFoundSecrets.slice(0, 3).join(', ')}`);
|
||
}
|
||
}
|
||
|
||
// Log summary of not found secrets, especially OKTA and EMAIL ones
|
||
if (notFoundSecrets.length > 0) {
|
||
const notFoundOkta = notFoundSecrets.filter(name => /OKTA_/i.test(name));
|
||
const notFoundEmail = notFoundSecrets.filter(name => /EMAIL_|SMTP_/i.test(name));
|
||
|
||
if (notFoundOkta.length > 0) {
|
||
logger.warn(`[Secret Manager] ⚠️ OKTA secrets not found (${notFoundOkta.length}): ${notFoundOkta.join(', ')}`);
|
||
logger.info(`[Secret Manager] 💡 To create OKTA secrets, use:`);
|
||
notFoundOkta.forEach(secretName => {
|
||
logger.info(`[Secret Manager] gcloud secrets create ${secretName} --data-file=- --project=${this.projectId}`);
|
||
});
|
||
}
|
||
|
||
if (notFoundEmail.length > 0) {
|
||
logger.warn(`[Secret Manager] ⚠️ EMAIL secrets not found (${notFoundEmail.length}): ${notFoundEmail.join(', ')}`);
|
||
logger.info(`[Secret Manager] 💡 To create EMAIL secrets, use:`);
|
||
notFoundEmail.forEach(secretName => {
|
||
logger.info(`[Secret Manager] gcloud secrets create ${secretName} --data-file=- --project=${this.projectId}`);
|
||
});
|
||
}
|
||
|
||
const otherNotFound = notFoundSecrets.filter(name => !/OKTA_|EMAIL_|SMTP_/i.test(name));
|
||
if (otherNotFound.length > 0) {
|
||
logger.debug(`[Secret Manager] Other secrets not found (${otherNotFound.length}): ${otherNotFound.slice(0, 10).join(', ')}${otherNotFound.length > 10 ? '...' : ''}`);
|
||
}
|
||
}
|
||
|
||
this.isInitialized = true;
|
||
} catch (error: any) {
|
||
logger.error('[Secret Manager] Failed to load secrets:', error);
|
||
// Don't throw - allow fallback to .env file
|
||
logger.warn('[Secret Manager] Falling back to .env file and existing environment variables');
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Get default list of secret names based on common environment variables
|
||
*/
|
||
private getDefaultSecretNames(): string[] {
|
||
return [
|
||
// Database
|
||
'DB_PASSWORD',
|
||
|
||
// JWT & Session
|
||
'JWT_SECRET',
|
||
'REFRESH_TOKEN_SECRET',
|
||
'SESSION_SECRET',
|
||
|
||
// Okta/SSO
|
||
'OKTA_CLIENT_ID',
|
||
'OKTA_CLIENT_SECRET',
|
||
'OKTA_API_TOKEN',
|
||
'OKTA_DOMAIN',
|
||
|
||
// Email
|
||
'SMTP_HOST',
|
||
'SMTP_PORT',
|
||
'SMTP_USER',
|
||
'SMTP_PASSWORD',
|
||
'EMAIL_FROM',
|
||
];
|
||
}
|
||
|
||
/**
|
||
* Load a single secret value
|
||
* Useful for on-demand secret retrieval
|
||
*/
|
||
async getSecretValue(secretName: string, envVarName?: string): Promise<string | null> {
|
||
const useSecretManager = process.env.USE_GOOGLE_SECRET_MANAGER === 'true';
|
||
|
||
if (!useSecretManager || !this.projectId) {
|
||
return null;
|
||
}
|
||
|
||
try {
|
||
if (!this.client) {
|
||
await this.initializeClient();
|
||
}
|
||
|
||
const secretValue = await this.getSecret(secretName);
|
||
|
||
if (secretValue !== null) {
|
||
const envVar = envVarName || this.getEnvVarName(secretName);
|
||
process.env[envVar] = secretValue;
|
||
logger.debug(`[Secret Manager] ✅ Loaded secret ${secretName} -> ${envVar}`);
|
||
}
|
||
|
||
return secretValue;
|
||
} catch (error: any) {
|
||
logger.error(`[Secret Manager] Failed to get secret ${secretName}:`, error);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Check if secret manager is initialized and ready
|
||
*/
|
||
isReady(): boolean {
|
||
return this.isInitialized;
|
||
}
|
||
|
||
/**
|
||
* List all secrets in the project (for debugging/setup verification)
|
||
* Returns array of secret names
|
||
*/
|
||
async listSecrets(): Promise<string[]> {
|
||
const useSecretManager = process.env.USE_GOOGLE_SECRET_MANAGER === 'true';
|
||
|
||
if (!useSecretManager || !this.projectId) {
|
||
logger.warn('[Secret Manager] Cannot list secrets: Secret Manager not enabled or project ID not set');
|
||
return [];
|
||
}
|
||
|
||
try {
|
||
if (!this.client) {
|
||
await this.initializeClient();
|
||
}
|
||
|
||
if (!this.client) {
|
||
return [];
|
||
}
|
||
|
||
const parent = `projects/${this.projectId}`;
|
||
const [secrets] = await this.client.listSecrets({ parent });
|
||
|
||
const secretNames = secrets.map(secret => {
|
||
// Extract secret name from full path: projects/PROJECT/secrets/NAME -> NAME
|
||
const nameParts = secret.name?.split('/') || [];
|
||
return nameParts[nameParts.length - 1] || '';
|
||
}).filter(Boolean);
|
||
|
||
logger.info(`[Secret Manager] Found ${secretNames.length} secrets in project ${this.projectId}`);
|
||
if (secretNames.length > 0) {
|
||
logger.info(`[Secret Manager] Available secrets: ${secretNames.slice(0, 10).join(', ')}${secretNames.length > 10 ? '...' : ''}`);
|
||
}
|
||
|
||
return secretNames;
|
||
} catch (error: any) {
|
||
logger.error(`[Secret Manager] Failed to list secrets: ${error.message || JSON.stringify(error)}`);
|
||
return [];
|
||
}
|
||
}
|
||
}
|
||
|
||
// Export singleton instance
|
||
export const googleSecretManager = new GoogleSecretManagerService();
|
||
|
||
// Export for easy initialization
|
||
export async function initializeGoogleSecretManager(secretNames?: string[]): Promise<void> {
|
||
await googleSecretManager.loadSecrets(secretNames);
|
||
}
|
||
|