ساختن یک چت بابات متخصص IBM Aix با استفاده از Rag و Faiss

pip install streamlit langchain openai faiss-cpu bs4 requests PyMuPDF

import requests
from bs4 import BeautifulSoup

def scrape_aix_docs(url):
    """
    Fetches IBM AIX documentation from a given URL and extracts text content.
    """
    response = requests.get(url)
    soup = BeautifulSoup(response.text, "html.parser")
    text_content = "\n".join([p.get_text() for p in soup.find_all("p")])
    with open("aix_docs.txt", "w", encoding="utf-8") as file:
        file.write(text_content)
    print("AIX documentation scraped and saved!")

scrape_aix_docs("https://www.ibm.com/docs/en/aix/7.3?topic=commands")

import fitz  # PyMuPDF

def extract_text_from_pdf(pdf_path):
    """
    Extracts text from a given PDF file and appends it to aix_docs.txt.
    """
    output_text = ""
    with fitz.open(pdf_path) as doc:
        for page in doc:
            output_text += page.get_text()
    with open("aix_docs.txt", "a", encoding="utf-8") as file:
        file.write(output_text)
    print("Extracted text from PDF saved!")

extract_text_from_pdf("aix_redbook.pdf")

from langchain.text_splitter import CharacterTextSplitter
from langchain.embeddings import OpenAIEmbeddings
from langchain.vectorstores import FAISS
from langchain.document_loaders import TextLoader

def build_vector_db():
    """
    Processes AIX documentation, splits it into smaller chunks, 
    generates embeddings, and stores them in FAISS vector database.
    """
    with open("aix_docs.txt", "r", encoding="utf-8") as file:
        aix_text = file.read()
    text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
    split_texts = text_splitter.split_text(aix_text)
    with open("aix_docs_cleaned.txt", "w", encoding="utf-8") as file:
        for chunk in split_texts:
            file.write(chunk + "\n\n")
    loader = TextLoader("aix_docs_cleaned.txt")
    documents = loader.load()
    embedding_model = OpenAIEmbeddings()
    vector_db = FAISS.from_documents(documents, embedding_model)
    vector_db.save_local("aix_vector_db")
    print("Vector database built successfully!")

build_vector_db()

import streamlit as st
from langchain.llms import OpenAI
from langchain.chains import RetrievalQA

# Load FAISS vector DB
embedding_model = OpenAIEmbeddings()
vector_db = FAISS.load_local("aix_vector_db", embedding_model)
retriever = vector_db.as_retriever()

# Setup RAG Chain
llm = OpenAI()
qa_chain = RetrievalQA.from_chain_type(llm=llm, retriever=retriever)

# Streamlit UI
st.title("IBM AIX Expert Chatbot")
st.write("Ask me anything about IBM AIX!")

user_input = st.text_input("Enter your question:")
if st.button("Submit"):
    if user_input:
        response = qa_chain.run(user_input)
        st.write(response)
    else:
        st.write("Please enter a question.")

streamlit run Aix_Rag_Chatbot.py

# Build the Docker image
docker build -t aix-rag-chatbot .

# Run the container
docker run -p 8501:8501 aix-rag-chatbot