-- Migration 003: Optimize Repository Files Storage with JSON
-- This migration transforms the repository_files table to use JSON arrays
-- for storing multiple files per directory instead of individual rows per file

-- Step 1: Enable required extensions
CREATE EXTENSION IF NOT EXISTS pg_trgm;

-- Step 2: Create backup table for existing data
CREATE TABLE IF NOT EXISTS repository_files_backup AS 
SELECT * FROM repository_files;

-- Step 3: Drop existing indexes that will be recreated
DROP INDEX IF EXISTS idx_repo_files_repo_id;
DROP INDEX IF EXISTS idx_repo_files_directory_id;
DROP INDEX IF EXISTS idx_repo_files_storage_id;
DROP INDEX IF EXISTS idx_repo_files_extension;
DROP INDEX IF EXISTS idx_repo_files_filename;
DROP INDEX IF EXISTS idx_repo_files_relative_path;
DROP INDEX IF EXISTS idx_repo_files_is_binary;

-- Step 4: Drop existing triggers
DROP TRIGGER IF EXISTS update_repository_files_updated_at ON repository_files;

-- Step 5: Drop the existing table
DROP TABLE IF EXISTS repository_files CASCADE;

-- Step 6: Create the new optimized repository_files table
CREATE TABLE IF NOT EXISTS repository_files (
  id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
  repository_id UUID REFERENCES all_repositories(id) ON DELETE CASCADE,
  storage_id UUID REFERENCES repository_storage(id) ON DELETE CASCADE,
  directory_id UUID REFERENCES repository_directories(id) ON DELETE SET NULL,
  
  -- Directory path information
  relative_path TEXT NOT NULL, -- path from repository root
  absolute_path TEXT NOT NULL, -- full local filesystem path
  
  -- JSON array containing all files in this directory
  files JSONB NOT NULL DEFAULT '[]'::jsonb,
  
  -- Aggregated directory statistics
  files_count INTEGER DEFAULT 0,
  total_size_bytes BIGINT DEFAULT 0,
  file_extensions TEXT[] DEFAULT '{}', -- Array of unique file extensions
  
  -- Directory metadata
  last_scan_at TIMESTAMP DEFAULT NOW(),
  scan_status VARCHAR(50) DEFAULT 'completed', -- pending, scanning, completed, error
  
  -- Timestamps
  created_at TIMESTAMP DEFAULT NOW(),
  updated_at TIMESTAMP DEFAULT NOW(),
  
  -- Constraints
  UNIQUE(directory_id), -- One record per directory
  CONSTRAINT valid_files_count CHECK (files_count >= 0),
  CONSTRAINT valid_total_size CHECK (total_size_bytes >= 0)
);

-- Step 7: Create function to update file statistics automatically
CREATE OR REPLACE FUNCTION update_repository_files_stats()
RETURNS TRIGGER AS $$
BEGIN
  -- Update files_count
  NEW.files_count := jsonb_array_length(NEW.files);
  
  -- Update total_size_bytes
  SELECT COALESCE(SUM((file->>'file_size_bytes')::bigint), 0)
  INTO NEW.total_size_bytes
  FROM jsonb_array_elements(NEW.files) AS file;
  
  -- Update file_extensions array
  SELECT ARRAY(
    SELECT DISTINCT file->>'file_extension'
    FROM jsonb_array_elements(NEW.files) AS file
    WHERE file->>'file_extension' IS NOT NULL
  )
  INTO NEW.file_extensions;
  
  RETURN NEW;
END;
$$ LANGUAGE plpgsql;

-- Step 8: Create triggers
CREATE TRIGGER update_repository_files_stats_trigger
  BEFORE INSERT OR UPDATE ON repository_files
  FOR EACH ROW EXECUTE FUNCTION update_repository_files_stats();

CREATE TRIGGER update_repository_files_updated_at 
  BEFORE UPDATE ON repository_files
  FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();

-- Step 9: Migrate existing data from backup table
INSERT INTO repository_files (
  repository_id,
  storage_id,
  directory_id,
  relative_path,
  absolute_path,
  files,
  files_count,
  total_size_bytes,
  file_extensions,
  last_scan_at,
  scan_status,
  created_at,
  updated_at
)
SELECT 
  rf.repository_id,
  rf.storage_id,
  rf.directory_id,
  -- Use directory path from repository_directories table
  COALESCE(rd.relative_path, ''),
  COALESCE(rd.absolute_path, ''),
  -- Aggregate files into JSON array
  jsonb_agg(
    jsonb_build_object(
      'filename', rf.filename,
      'file_extension', rf.file_extension,
      'relative_path', rf.relative_path,
      'absolute_path', rf.absolute_path,
      'file_size_bytes', rf.file_size_bytes,
      'file_hash', rf.file_hash,
      'mime_type', rf.mime_type,
      'is_binary', rf.is_binary,
      'encoding', rf.encoding,
      'github_sha', rf.github_sha,
      'created_at', rf.created_at,
      'updated_at', rf.updated_at
    )
  ) as files,
  -- Statistics will be calculated by trigger
  0 as files_count,
  0 as total_size_bytes,
  '{}' as file_extensions,
  NOW() as last_scan_at,
  'completed' as scan_status,
  MIN(rf.created_at) as created_at,
  MAX(rf.updated_at) as updated_at
FROM repository_files_backup rf
LEFT JOIN repository_directories rd ON rf.directory_id = rd.id
WHERE rf.directory_id IS NOT NULL
GROUP BY 
  rf.repository_id,
  rf.storage_id,
  rf.directory_id,
  rd.relative_path,
  rd.absolute_path;

-- Step 10: Create optimized indexes
CREATE INDEX IF NOT EXISTS idx_repo_files_repo_id ON repository_files(repository_id);
CREATE INDEX IF NOT EXISTS idx_repo_files_directory_id ON repository_files(directory_id);
CREATE INDEX IF NOT EXISTS idx_repo_files_storage_id ON repository_files(storage_id);
CREATE INDEX IF NOT EXISTS idx_repo_files_relative_path ON repository_files(relative_path);
CREATE INDEX IF NOT EXISTS idx_repo_files_scan_status ON repository_files(scan_status);
CREATE INDEX IF NOT EXISTS idx_repo_files_last_scan ON repository_files(last_scan_at);

-- JSONB indexes for efficient file queries
CREATE INDEX IF NOT EXISTS idx_repo_files_files_gin ON repository_files USING gin(files);
CREATE INDEX IF NOT EXISTS idx_repo_files_filename ON repository_files USING gin((files->>'filename') gin_trgm_ops);
CREATE INDEX IF NOT EXISTS idx_repo_files_extension ON repository_files USING gin((files->>'file_extension') gin_trgm_ops);
CREATE INDEX IF NOT EXISTS idx_repo_files_is_binary ON repository_files USING gin((files->>'is_binary') gin_trgm_ops);

-- Array indexes
CREATE INDEX IF NOT EXISTS idx_repo_files_extensions ON repository_files USING gin(file_extensions);

-- Step 11: Update repository_directories files_count to match new structure
UPDATE repository_directories rd
SET files_count = COALESCE(
  (SELECT rf.files_count 
   FROM repository_files rf 
   WHERE rf.directory_id = rd.id), 
  0
);

-- Step 12: Update repository_storage total_files_count
UPDATE repository_storage rs
SET total_files_count = COALESCE(
  (SELECT SUM(rf.files_count) 
   FROM repository_files rf 
   WHERE rf.storage_id = rs.id), 
  0
);

-- Step 13: Verify migration
DO $$
DECLARE
  backup_count INTEGER;
  new_count INTEGER;
  total_files_backup INTEGER;
  total_files_new INTEGER;
BEGIN
  -- Count records
  SELECT COUNT(*) INTO backup_count FROM repository_files_backup;
  SELECT COUNT(*) INTO new_count FROM repository_files;
  
  -- Count total files
  SELECT COUNT(*) INTO total_files_backup FROM repository_files_backup;
  SELECT SUM(files_count) INTO total_files_new FROM repository_files;
  
  -- Log results
  RAISE NOTICE 'Migration completed:';
  RAISE NOTICE 'Backup records: %', backup_count;
  RAISE NOTICE 'New directory records: %', new_count;
  RAISE NOTICE 'Total files in backup: %', total_files_backup;
  RAISE NOTICE 'Total files in new structure: %', total_files_new;
  
  -- Verify data integrity
  IF total_files_backup = total_files_new THEN
    RAISE NOTICE 'Data integrity verified: All files migrated successfully';
  ELSE
    RAISE WARNING 'Data integrity issue: File count mismatch';
  END IF;
END $$;

-- Step 14: Create helper functions for common queries
CREATE OR REPLACE FUNCTION get_files_in_directory(dir_uuid UUID)
RETURNS TABLE(
  filename TEXT,
  file_extension TEXT,
  relative_path TEXT,
  file_size_bytes BIGINT,
  mime_type TEXT,
  is_binary BOOLEAN
) AS $$
BEGIN
  RETURN QUERY
  SELECT 
    file->>'filename' as filename,
    file->>'file_extension' as file_extension,
    file->>'relative_path' as relative_path,
    (file->>'file_size_bytes')::bigint as file_size_bytes,
    file->>'mime_type' as mime_type,
    (file->>'is_binary')::boolean as is_binary
  FROM repository_files rf, jsonb_array_elements(rf.files) as file
  WHERE rf.directory_id = dir_uuid;
END;
$$ LANGUAGE plpgsql;

CREATE OR REPLACE FUNCTION find_files_by_extension(ext TEXT)
RETURNS TABLE(
  directory_path TEXT,
  filename TEXT,
  relative_path TEXT,
  file_size_bytes BIGINT
) AS $$
BEGIN
  RETURN QUERY
  SELECT 
    rf.relative_path as directory_path,
    file->>'filename' as filename,
    file->>'relative_path' as relative_path,
    (file->>'file_size_bytes')::bigint as file_size_bytes
  FROM repository_files rf, jsonb_array_elements(rf.files) as file
  WHERE file->>'file_extension' = ext;
END;
$$ LANGUAGE plpgsql;

-- Step 15: Add comments for documentation
COMMENT ON TABLE repository_files IS 'Optimized table storing files as JSON arrays grouped by directory';
COMMENT ON COLUMN repository_files.files IS 'JSON array containing all files in this directory with complete metadata';
COMMENT ON COLUMN repository_files.files_count IS 'Automatically calculated count of files in this directory';
COMMENT ON COLUMN repository_files.total_size_bytes IS 'Automatically calculated total size of all files in this directory';
COMMENT ON COLUMN repository_files.file_extensions IS 'Array of unique file extensions in this directory';

-- Migration completed successfully
SELECT 'Migration 003 completed: Repository files optimized with JSON storage' as status;
