ساخت یک سیستم تولید تقویت شده بازیابی ساده با ذخیره سازی Supabase و تعبیه Openai در Next.js

npm install @supabase/supabase-js @ai-sdk/openai ai xlsx pdf-parse mammoth tesseract.js

'use client'

import { useState } from 'react'
import { useDropzone } from 'react-dropzone'
import { uploadMedia } from '@/lib/upload-media'

// The Artifact will be the organization file that is being uploaded
interface Artifact {
  id?: number
  file?: File
  name: string
  description: string
  uploaded: boolean
}

export default function FileUpload() {
  const [isUploading, setIsUploading] = useState(false)

  // Handle file upload and embedding generation
  const uploadArtifact = async (artifact: Artifact) => {
    if (!artifact.file) return
    setIsUploading(true)

    try {
      // First, upload file to Supabase storage
      const mediaPath = await uploadMedia(artifact.file)

      if (!mediaPath) {
        throw new Error('Failed to upload file')
      }

      // Generate embeddings via API route
      const formData = new FormData()
      formData.append("mediaPath", mediaPath)

      const response = await fetch("/api/generate-embeddings", {
        method: "POST",
        body: formData,
      })

      const data = await response.json()

      if (!data.success) {
        throw new Error('Failed to generate embeddings')
      }

      // Success! You can now store the embeddings in your database
      console.log('Embeddings generated:', data.embedding)
      //please store your generated embeddings in your database :)
    } catch (error) {
      console.error('Error:', error)
    } finally {
      setIsUploading(false)
    }
  }

  // Set up drag & drop
  const onDrop = useCallback((acceptedFiles: File[]) => {
    const validFileTypes = [
      'text/plain',
      'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
      'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
      'application/pdf'
    ]

    const validFiles = acceptedFiles.filter(file => validFileTypes.includes(file.type))

    if (validFiles.length > 0) {
      // Process each valid file
      validFiles.forEach(file => {
        uploadArtifact({
          file,
          name: file.name,
          description: '',
          uploaded: false
        })
      })
    }
  }, [])

  const { getRootProps, getInputProps } = useDropzone({
    onDrop,
    accept: {
      'text/plain': ['.txt'],
      'application/vnd.openxmlformats-officedocument.wordprocessingml.document': ['.docx'],
      'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': ['.xlsx'],
      'application/pdf': ['.pdf']
    }
  })

  return (
    

      
      {isUploading && 
Processing files...
}
    
  )
}

// lib/upload-media.ts
import { supabase } from '@/lib/utils'

export async function uploadMedia(file: File) {
  // Upload file to Supabase storage with a unique name
  const { data, error } = await supabase.storage
    .from('media')
    .upload(`${Date.now()}-${file.name}`, file)

  if (error) {
    console.error('Error uploading file:', error)
    return null
  }

  return data.path
}

// app/api/generate-embeddings/route.ts
import { createEmbedding } from '@/lib/create-embeddings'
import { type NextRequest, NextResponse } from 'next/server'
export const maxDuration = 60
export async function POST(req: NextRequest) {
  try {
    const formData = await req.formData()
    const filePath = formData.get('mediaPath') as string

    if (!filePath) {
      return NextResponse.json(
        { error: 'Missing file path' },
        { status: 400 }
      )
    }
    // Generate embeddings for the file content
    const embedding = await createEmbedding(filePath)
    return NextResponse.json({ success: true, embedding })
  } catch (error) {
    console.error('Error:', error)
    return NextResponse.json(
      { error: 'Internal server error' },
      { status: 500 }
    )
  }
}

// lib/create-embeddings.ts
import { createOpenAI } from '@ai-sdk/openai';
import { createClient } from '@supabase/supabase-js';
import { embed } from 'ai';
import { execSync } from 'child_process';
import fs from 'fs';
import mammoth from 'mammoth';
import os from 'os';
import path from 'path';
import PdfParse from 'pdf-parse';
import Tesseract from 'tesseract.js';
import * as XLSX from 'xlsx';
import { chunkDocumentWithOverlap } from './chunkDocument';

const supabase = createClient(
  process.env.SUPABASE_URL!,
  process.env.SUPABASE_ANON_KEY!
);

const openai = createOpenAI({
  apiKey: process.env.OPENAI_API_KEY,
});

// Main function to create embeddings
export async function createEmbedding(filePath: string) {
  const { data, error } = await supabase.storage
    .from('media')
    .download(filePath)
  if (error || !data) throw new Error('Failed to download file')
  // Extract text from the file
  const text = await extractTextFromFile(data, filePath)

  // Split text into chunks for better processing
  const chunks = chunkDocumentWithOverlap(text)

  // Generate embeddings for each chunk
  const chunksWithEmbeddings = await Promise.all(
    chunks.map(async (chunk) => {
      const { embedding } = await embed({
        model: openai.embedding('text-embedding-3-small'),
        value: chunk,
      })
      return { embedding, content: chunk }
    })
  )
  return chunksWithEmbeddings
}

async function extractTextFromFile(
  fileData: Blob,
  filePath: string
): Promise {
  const fileExtension = path.extname(filePath).toLowerCase();
  const tempFilePath = path.join(os.tmpdir(), `temp_file${fileExtension}`);

  console.log(`Temp file path: ${tempFilePath}`);

  // Write the blob to a temporary file
  try {
    await fs.promises.writeFile(
      tempFilePath,
      Buffer.from(await fileData.arrayBuffer())
    );
    console.log('File written successfully.');
  } catch (err) {
    console.error('Error writing file:', err);
    throw new Error('Failed to write temporary file.');
  }

  // Check if the file exists
  const fileExists = fs.existsSync(tempFilePath);
  if (!fileExists) {
    console.error('Temp file was not created.');
    throw new Error('File was not created successfully.');
  }

  console.log('File exists:', fileExists);

  try {
    switch (fileExtension) {
      case '.pdf':
        return await extractTextFromPDF(tempFilePath);
      case '.docx':
        return await extractTextFromDOCX(tempFilePath);
      case '.xlsx':
        return extractTextFromXLSX(tempFilePath);
      case '.txt':
        console.log('Reading .txt file...');
        const textContent = await fs.promises.readFile(tempFilePath, 'utf-8');
        console.log('Extracted text:', textContent);
        return textContent;
      default:
        throw new Error('Unsupported file type');
    }
  } catch (err) {
    console.error('Error extracting text:', err);
    throw new Error('Failed to extract text.');
  } finally {
    // Clean up the temporary file
    try {
      await fs.promises.unlink(tempFilePath);
      console.log('Temp file deleted.');
    } catch (err) {
      console.error('Error deleting temp file:', err);
    }
  }
}

//function to clean up the text
const cleanText = (text : string) => {
  return text
    .replace(/\x00/g, '') // Remove NULL bytes
    .replace(/[^\x20-\x7E\n]/g, ''); // Remove non-ASCII characters (optional)
};

// extract the text from pdf
async function extractTextFromPDF(filePath: string): Promise {
  try {
    console.log('Trying pdf-parse...');
    const dataBuffer = await fs.promises.readFile(filePath);
    const data = await PdfParse(dataBuffer);

    let text = data.text.replace(/\uFFFD/g, '').trim(); // Remove invalid characters

    if (text && text.length > 20) {
      console.log('Extracted text successfully using pdf-parse.');
      return cleanText(text);
    }

    throw new Error('Extracted text is empty or invalid.');
  } catch (error) {
    console.warn('pdf-parse failed:', error);
  }

  // Fallback: Using pdftotext (requires poppler-utils installed)
  try {
    console.log('Trying pdftotext...');
    const extractedText = execSync(`pdftotext -layout "${filePath}" -`, {
      encoding: 'utf-8',
    }).trim();
    if (extractedText.length > 20) {
      console.log('Extracted text successfully using pdftotext.');
      return extractedText;
    }

    throw new Error('pdftotext extraction returned empty text.');
  } catch (error) {
    console.warn('pdftotext failed:', error);
  }

  // Final Fallback: OCR with Tesseract.js (for scanned PDFs)
  try {
    console.log('Trying OCR (Tesseract.js)...');
    const {
      data: { text },
    } = await Tesseract.recognize(filePath, 'eng', {
      logger: (m) => console.log(m), // Log OCR progress
    });

    if (text.length > 20) {
      console.log('Extracted text successfully using OCR.');
      return text;
    }

    throw new Error('OCR extraction returned empty text.');
  } catch (error) {
    console.error('OCR extraction failed:', error);
  }

  throw new Error('Failed to extract text from PDF using all methods.');
}

//extract text from docx files
async function extractTextFromDOCX(filePath: string): Promise {
  try {
    const result = await mammoth.extractRawText({ path: filePath });
    return result.value;
  } catch (error) {
    console.error('Error parsing DOCX:', error);
    throw new Error('Failed to extract text from DOCX');
  }
}

//extract text from xlsx files
function extractTextFromXLSX(filePath: string): string {
  try {
    const workbook = XLSX.readFile(filePath);
    let text="";
    workbook.SheetNames.forEach((sheetName) => {
      const sheet = workbook.Sheets[sheetName];
      text += XLSX.utils.sheet_to_csv(sheet) + '\n\n';
    });
    return text;
  } catch (error) {
    console.error('Error parsing XLSX:', error);
    throw new Error('Failed to extract text from XLSX');
  }
}

//lib/chunkDocumentWithOverlap
const MAX_CHUNK_SIZE = 2000;
const OVERLAP = 200;

export function chunkDocumentWithOverlap(text: string): string[] {
  const chunks: string[] = [];
  let startIndex = 0;

  while (startIndex < text.length) {
    let endIndex = Math.min(startIndex + MAX_CHUNK_SIZE, text.length);

    console.log(
      'Start:',
      startIndex,
      'End:',
      endIndex,
      'Total Length:',
      text.length,
      'Chunks Count:',
      chunks.length
    );

    if (endIndex < text.length) {
      // Try to find a space to break at, moving left from `endIndex`
      let breakPoint = endIndex;
      while (breakPoint > startIndex && text[breakPoint] !== ' ') {
        breakPoint--;
      }

      // If no space was found, keep the original `endIndex`
      if (breakPoint > startIndex) {
        endIndex = breakPoint + 1; // Include the space
      }
    }

    chunks.push(text.slice(startIndex, endIndex));

    // Move `startIndex` forward but ensure we make progress
    startIndex = endIndex;
  }

  return chunks;
}

// lib/similarity.ts
import { openai } from '@ai-sdk/openai';

//make sure OPENAI_API_KEY is set up in your env
// Calculate similarity between two vectors using cosine similarity
export function cosineSimilarity(a: number[], b: number[]): number {
  // Calculate dot product
  const dotProduct = a.reduce((sum, _, i) => sum + a[i] * b[i], 0)

  // Calculate magnitudes
  const magnitudeA = Math.sqrt(a.reduce((sum, val) => sum + val * val, 0))
  const magnitudeB = Math.sqrt(b.reduce((sum, val) => sum + val * val, 0))

  // Return similarity score
  return dotProduct / (magnitudeA * magnitudeB)
}

// Find most similar documents
export async function findMostSimilarArtifacts(
  input: string,
  artifacts: Chunk[],
  count: number
): Promise {
  // Get embedding for user input
  const { embedding } = await embed({
    model: openai.embedding('text-embedding-3-small'),
    value: input,
  });


  // Calculate similarity scores and sort
  return artifacts
    .map((artifact) => ({
      ...artifact,
      similarity: cosineSimilarity(embedding, artifact.embedding),
    }))
    .sort((a, b) => b.similarity - a.similarity)
    .slice(0, count)
}

// components/DocumentChat.tsx
'use client'

import { useChat } from 'ai/react'
import { useState } from 'react'
import { Button } from "@/components/ui/button"
import { Card, CardContent, CardFooter, CardHeader, CardTitle } from "@/components/ui/card"
import { Input } from "@/components/ui/input"
import { ScrollArea } from "@/components/ui/scroll-area"
import { Send } from 'lucide-react'

export default function DocumentChat() {
  // Initialize the chat hook with our API endpoint
  const { messages, input, handleInputChange, handleSubmit, isLoading } = useChat({
    api: '/api/chat',
    // Initialize with a helpful system message
    initialMessages: [
      {
        id: 'welcome',
        role: 'system',
        content: "I'm your document assistant. Ask me anything about your uploaded documents!",
      },
    ],
  })

  return (
    
      
        Document Assistant
      

      
        
          

            {messages.map((message) => (
              
            ))}
            {isLoading && (
              
            )}
          
        
      

      
        
      
    
  )
}

// app/api/chat/route.ts
import { createOpenAI } from '@ai-sdk/openai'
import { embed, streamText } from 'ai'
import { NextResponse, type NextRequest } from 'next/server'
import { findMostSimilarArtifacts } from '@/lib/similarity'

// Initialize OpenAI
const openai = createOpenAI({
  apiKey: process.env.OPENAI_API_KEY,
})

export async function POST(req: NextRequest) {
  try {
    const { messages } = await req.json()
    const lastMessage = messages[messages.length - 1]

//get storedArtifacts from your storage first and then find the similar artifacts to be used as context.
const relevantDocs = await findMostSimilarArtifacts(lastMessage, storedArtifacts, 3)

const contextPromises = relevantDocs.map(async (a) => {
    return `${a.articleName} (${a.content})`
})
//get the context to use 
  const context = (await Promise.all(contextPromises)).join("\n\n")
const result = streamText({
      model: openai('gpt-4o-mini'),
      messages,
      temperature: 0.7,
      maxTokens: 1000,
      system: `You are an AI assistant helping with retrieval augmentation. Provide concise and relevant information based on the context provided. the context : ${context}, the question context which contains the questions and answer from the questionnaire answered about the school ${questionContext}, you can also use that to generate a proper response`,
    });

    return result.toDataStreamResponse();
    }catch (error) {
    console.error('Error in chat route:', error);
    return new Response('An error occurred while processing your request', {
      status: 500,
    });
  }
}