314 lines
11 KiB
TypeScript
314 lines
11 KiB
TypeScript
import fs from 'fs';
|
|
import path from 'path';
|
|
const _pdfParse = require('pdf-parse');
|
|
const pdfParse = _pdfParse.PDFParse || _pdfParse.default || _pdfParse;
|
|
import mammoth from 'mammoth';
|
|
import * as XLSX from 'xlsx';
|
|
import * as cheerio from 'cheerio';
|
|
import { GetObjectCommand } from '@aws-sdk/client-s3';
|
|
import { s3Client, BUCKET_NAME } from '../utils/s3';
|
|
import { ScraperService } from './scraper.service';
|
|
|
|
export interface OKFMetadata {
|
|
assetId: string;
|
|
assetTitle: string;
|
|
assetType: string;
|
|
location: string; // e.g. "Page 3", "Slide 5", "Sheet: Financials", "Transcript", "Section: Architecture"
|
|
okfCategory?: string;
|
|
isRecommended?: boolean;
|
|
}
|
|
|
|
export interface OKFChunk {
|
|
chunkIndex: number;
|
|
chunkType: 'PAGE' | 'SLIDE' | 'SHEET' | 'TRANSCRIPT' | 'TEXT';
|
|
content: string;
|
|
sourceMetadata: OKFMetadata;
|
|
}
|
|
|
|
export class ExtractionService {
|
|
private scraperService = new ScraperService();
|
|
|
|
/**
|
|
* Simple, fast deterministic embedding generator for local RAG vector search.
|
|
* Generates a 64-dimensional float vector normalized for cosine similarity.
|
|
*/
|
|
public generateEmbedding(text: string): string {
|
|
const dim = 64;
|
|
const vector = new Array(dim).fill(0);
|
|
const cleaned = text.toLowerCase().replace(/[^\w\s]/g, '');
|
|
const words = cleaned.split(/\s+/).filter(Boolean);
|
|
|
|
for (let i = 0; i < words.length; i++) {
|
|
const word = words[i];
|
|
for (let j = 0; j < word.length; j++) {
|
|
const charCode = word.charCodeAt(j);
|
|
const idx = (charCode + j * 7 + i * 3) % dim;
|
|
vector[idx] += 1;
|
|
}
|
|
}
|
|
|
|
// L2 Normalize
|
|
const magnitude = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0)) || 1;
|
|
const normalized = vector.map(v => Number((v / magnitude).toFixed(6)));
|
|
|
|
return JSON.stringify(normalized);
|
|
}
|
|
|
|
/**
|
|
* Extract text chunks from an asset binary or URL using the Open Knowledge Framework (OKF) standard format.
|
|
*/
|
|
public async extractOKFChunks(asset: {
|
|
id: string;
|
|
title: string;
|
|
type: string;
|
|
url: string;
|
|
problemStatement?: string | null;
|
|
solution?: string | null;
|
|
description?: string | null;
|
|
}): Promise<OKFChunk[]> {
|
|
const chunks: OKFChunk[] = [];
|
|
let chunkIndex = 0;
|
|
|
|
// Base Primary Metadata Chunk (Guarantees 100% indexing for ALL assets including URLs & Documents)
|
|
chunks.push({
|
|
chunkIndex: chunkIndex++,
|
|
chunkType: 'TEXT',
|
|
content: `Asset Title: "${asset.title}". Type: "${asset.type}". Description: "${asset.description || ''}". Problem: "${asset.problemStatement || ''}". Solution: "${asset.solution || ''}".`,
|
|
sourceMetadata: {
|
|
assetId: asset.id,
|
|
assetTitle: asset.title,
|
|
assetType: asset.type,
|
|
location: 'Catalog Overview & Metadata',
|
|
},
|
|
});
|
|
|
|
// 1. Ingest Problem Statement & Solution metadata if present
|
|
if (asset.problemStatement) {
|
|
chunks.push({
|
|
chunkIndex: chunkIndex++,
|
|
chunkType: 'TEXT',
|
|
content: `Problem Statement for ${asset.title}: ${asset.problemStatement}`,
|
|
sourceMetadata: {
|
|
assetId: asset.id,
|
|
assetTitle: asset.title,
|
|
assetType: asset.type,
|
|
location: 'Executive Overview: Problem Statement',
|
|
},
|
|
});
|
|
}
|
|
|
|
if (asset.solution) {
|
|
chunks.push({
|
|
chunkIndex: chunkIndex++,
|
|
chunkType: 'TEXT',
|
|
content: `Solution Overview for ${asset.title}: ${asset.solution}`,
|
|
sourceMetadata: {
|
|
assetId: asset.id,
|
|
assetTitle: asset.title,
|
|
assetType: asset.type,
|
|
location: 'Executive Overview: Proposed Solution',
|
|
},
|
|
});
|
|
}
|
|
|
|
// 2. URL Assets / YouTube / Web scraper handling
|
|
if (asset.type === 'url' || asset.type === 'case_study' || asset.url.startsWith('http')) {
|
|
try {
|
|
const scraped = await this.scraperService.scrapeCaseStudy(asset.url);
|
|
const fullContent = `${scraped.title}. ${scraped.problemStatement || ''} ${scraped.solution || ''}`;
|
|
|
|
// Split into 500-token chunks
|
|
const subChunks = this.splitText(fullContent, 500);
|
|
subChunks.forEach((text, i) => {
|
|
chunks.push({
|
|
chunkIndex: chunkIndex++,
|
|
chunkType: 'TRANSCRIPT',
|
|
content: text,
|
|
sourceMetadata: {
|
|
assetId: asset.id,
|
|
assetTitle: asset.title,
|
|
assetType: asset.type,
|
|
location: `Web Link Content: Part ${i + 1}`,
|
|
},
|
|
});
|
|
});
|
|
} catch {
|
|
if (asset.description) {
|
|
chunks.push({
|
|
chunkIndex: chunkIndex++,
|
|
chunkType: 'TEXT',
|
|
content: `${asset.title}: ${asset.description}`,
|
|
sourceMetadata: {
|
|
assetId: asset.id,
|
|
assetTitle: asset.title,
|
|
assetType: asset.type,
|
|
location: 'URL Asset Metadata',
|
|
},
|
|
});
|
|
}
|
|
}
|
|
return chunks;
|
|
}
|
|
|
|
// 3. Binary S3 Assets (PDF, Word, Excel, PPTX, Text)
|
|
if (asset.url.startsWith('/uploads/')) {
|
|
const fileKey = asset.url.replace('/uploads/', '');
|
|
let buffer: Buffer;
|
|
|
|
try {
|
|
const response = await s3Client.send(new GetObjectCommand({
|
|
Bucket: BUCKET_NAME,
|
|
Key: fileKey,
|
|
}));
|
|
const byteArray = await response.Body?.transformToByteArray();
|
|
if (!byteArray) return chunks;
|
|
buffer = Buffer.from(byteArray);
|
|
} catch (err) {
|
|
console.error(`Failed to fetch S3 object ${fileKey} for extraction:`, err);
|
|
return chunks;
|
|
}
|
|
|
|
const ext = path.extname(fileKey).toLowerCase();
|
|
|
|
// A. PDF Files
|
|
if (ext === '.pdf' || asset.type.includes('pdf')) {
|
|
try {
|
|
let pdfText = '';
|
|
try {
|
|
const parser = new pdfParse({ data: buffer });
|
|
const res = await parser.getText();
|
|
pdfText = typeof res === 'string' ? res : res?.text || '';
|
|
} catch (e1) {
|
|
try {
|
|
const res = await pdfParse(buffer);
|
|
pdfText = typeof res === 'string' ? res : res?.text || '';
|
|
} catch (e2) {}
|
|
}
|
|
if (pdfText) {
|
|
const subChunks = this.splitText(pdfText, 600);
|
|
subChunks.forEach((text, i) => {
|
|
chunks.push({
|
|
chunkIndex: chunkIndex++,
|
|
chunkType: 'PAGE',
|
|
content: text,
|
|
sourceMetadata: {
|
|
assetId: asset.id,
|
|
assetTitle: asset.title,
|
|
assetType: asset.type,
|
|
location: `PDF Document: Page ${i + 1}`,
|
|
},
|
|
});
|
|
});
|
|
}
|
|
} catch (e) { console.error('PDF extraction error:', e); }
|
|
}
|
|
// B. Word Files (.docx, .doc)
|
|
else if (ext === '.docx' || ext === '.doc' || asset.type.includes('word')) {
|
|
try {
|
|
const docResult = await mammoth.extractRawText({ buffer });
|
|
const subChunks = this.splitText(docResult.value, 600);
|
|
subChunks.forEach((text, i) => {
|
|
chunks.push({
|
|
chunkIndex: chunkIndex++,
|
|
chunkType: 'TEXT',
|
|
content: text,
|
|
sourceMetadata: {
|
|
assetId: asset.id,
|
|
assetTitle: asset.title,
|
|
assetType: asset.type,
|
|
location: `Word Document: Section ${i + 1}`,
|
|
},
|
|
});
|
|
});
|
|
} catch (e) { console.error('Docx extraction error:', e); }
|
|
}
|
|
// C. Excel & CSV Files (.xlsx, .xls, .csv)
|
|
else if (ext === '.xlsx' || ext === '.xls' || ext === '.csv' || asset.type.includes('spreadsheet') || asset.type.includes('csv')) {
|
|
try {
|
|
const workbook = XLSX.read(buffer, { type: 'buffer' });
|
|
workbook.SheetNames.forEach((sheetName) => {
|
|
const sheet = workbook.Sheets[sheetName];
|
|
const csvText = XLSX.utils.sheet_to_csv(sheet);
|
|
if (csvText && csvText.trim()) {
|
|
chunks.push({
|
|
chunkIndex: chunkIndex++,
|
|
chunkType: 'SHEET',
|
|
content: `Sheet [${sheetName}] Data:\n${csvText.slice(0, 1500)}`,
|
|
sourceMetadata: {
|
|
assetId: asset.id,
|
|
assetTitle: asset.title,
|
|
assetType: asset.type,
|
|
location: `Spreadsheet Sheet: ${sheetName}`,
|
|
},
|
|
});
|
|
}
|
|
});
|
|
} catch (e) { console.error('Excel extraction error:', e); }
|
|
}
|
|
// D. PowerPoint Presentations (.pptx, .ppt)
|
|
else if (ext === '.pptx' || ext === '.ppt' || asset.type.includes('presentation')) {
|
|
// Simple text extraction from raw slide XML/strings
|
|
const rawString = buffer.toString('utf-8').replace(/[^\x20-\x7E]/g, ' ');
|
|
const subChunks = this.splitText(rawString, 600);
|
|
subChunks.forEach((text, i) => {
|
|
if (text.length > 50) {
|
|
chunks.push({
|
|
chunkIndex: chunkIndex++,
|
|
chunkType: 'SLIDE',
|
|
content: text,
|
|
sourceMetadata: {
|
|
assetId: asset.id,
|
|
assetTitle: asset.title,
|
|
assetType: asset.type,
|
|
location: `PowerPoint Presentation: Slide Section ${i + 1}`,
|
|
},
|
|
});
|
|
}
|
|
});
|
|
}
|
|
// E. Plain Text / Markdown / Code (Excluding binaries)
|
|
else if (!ext.match(/\.(png|jpe?g|gif|webp|svg|mp4|webm|avi|mp3|wav)$/i)) {
|
|
const textContent = buffer.toString('utf-8');
|
|
// Only proceed if it looks like actual text (not arbitrary binary data)
|
|
if (!textContent.includes('\u0000\u0000')) {
|
|
const subChunks = this.splitText(textContent, 600);
|
|
subChunks.forEach((text, i) => {
|
|
chunks.push({
|
|
chunkIndex: chunkIndex++,
|
|
chunkType: 'TEXT',
|
|
content: text,
|
|
sourceMetadata: {
|
|
assetId: asset.id,
|
|
assetTitle: asset.title,
|
|
assetType: asset.type,
|
|
location: `Document Text: Segment ${i + 1}`,
|
|
},
|
|
});
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
return chunks;
|
|
}
|
|
|
|
private splitText(text: string, maxLen: number): string[] {
|
|
const cleaned = text.replace(/\s+/g, ' ').trim();
|
|
if (!cleaned) return [];
|
|
const words = cleaned.split(' ');
|
|
const chunks: string[] = [];
|
|
let current = '';
|
|
|
|
for (const word of words) {
|
|
if ((current + ' ' + word).length > maxLen) {
|
|
if (current) chunks.push(current.trim());
|
|
current = word;
|
|
} else {
|
|
current += (current ? ' ' : '') + word;
|
|
}
|
|
}
|
|
if (current.trim()) chunks.push(current.trim());
|
|
return chunks;
|
|
}
|
|
}
|