Agentic Rag PipeLine Structed With Vious Rag Techniques

Author: Abeshith

.gitignore

@@ -0,0 +1,11 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+.env

.python-version

@@ -0,0 +1 @@
+3.11

README.md

@@ -0,0 +1,59 @@
+import os
+from dotenv import load_dotenv
+from project.pipeline.agents import AgentWorkflow
+from project.logger.logging import get_logger
+
+load_dotenv()
+
+logger = get_logger(__name__)
+
+
+def setup_langsmith():
+    langsmith_api_key = os.getenv("LANGSMITH_API_KEY")
+    if langsmith_api_key:
+        os.environ["LANGCHAIN_TRACING_V2"] = "true"
+        os.environ["LANGCHAIN_ENDPOINT"] = "https://api.smith.langchain.com"
+        os.environ["LANGCHAIN_API_KEY"] = langsmith_api_key
+        os.environ["LANGCHAIN_PROJECT"] = "rag-corrective-pipeline"
+        logger.info("LangSmith tracing enabled")
+    else:
+        logger.warning("LANGSMITH_API_KEY not found, tracing disabled")
+
+
+def main():
+    setup_langsmith()
+    logger.info("Starting RAG application...")
+    
+    agent = AgentWorkflow()
+    
+    logger.info("Setting up pipeline with Attention Is All You Need paper...")
+    agent.setup(use_attention_paper=True)
+    
+    agent.save_graph("workflow.png")
+    logger.info("Workflow graph saved")
+    
+    questions = [
+        "What is the attention mechanism in transformers?",
+        "Explain the multi-head attention.",
+        "What are the advantages of the transformer architecture?"
+    ]
+    
+    print("\n" + "="*80)
+    print("RAG PIPELINE WITH CORRECTIVE RAG (CRAG)")
+    print("="*80 + "\n")
+    
+    for i, question in enumerate(questions, 1):
+        print(f"\n{'='*80}")
+        print(f"Question {i}: {question}")
+        print(f"{'='*80}\n")
+        
+        answer = agent.run(question)
+        
+        print(f"\nAnswer:\n{answer}\n")
+        print(f"{'='*80}\n")
+    
+    logger.info("RAG application completed successfully")
+
+
+if __name__ == "__main__":
+    main()

data/attention-is-all-you-need.pdf

@@ -0,0 +1,208 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "171dc240",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dotenv import load_dotenv\n",
+    "import os\n",
+    "load_dotenv() \n",
+    "os.environ['GROQ_API_KEY'] = os.getenv('GROQ_API_KEY')\n",
+    "os.environ['GOOGLE_API_KEY'] = os.getenv('GOOGLE_API_KEY')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "efbca25c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.document_loaders import TextLoader\n",
+    "\n",
+    "loader = TextLoader('..\\data\\state_of_the_union.txt', encoding='utf8')\n",
+    "documents = loader.load()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "203b53b3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Madam Speaker, Madam Vice President, our First Lady and Second Gentleman. Members of Congress and the Cabinet. Justices of the Supreme Court. My fellow Americans.  \n",
+      "\n",
+      "Last year COVID-19 kept us apart. This year we are finally together again. \n",
+      "\n",
+      "Tonight, we meet as Democrats Republicans and Independents. But most importantly as Americans. \n",
+      "\n",
+      "With a duty to one another to the American people to the Constitution. \n",
+      "\n",
+      "And with an unwavering resolve that freedom will always triumph over tyranny. \n",
+      "\n",
+      "Six day\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(documents[0].page_content[:500])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "76bdd56f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
+    "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n",
+    "chunks = text_splitter.split_documents(documents)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "3fd6b5dd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.embeddings import FastEmbedEmbeddings\n",
+    "embeddings = FastEmbedEmbeddings(model_name=\"BAAI/bge-small-en-v1.5\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "9d79271e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.vectorstores import FAISS\n",
+    "\n",
+    "vectorstore = FAISS.from_documents(chunks, embeddings)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "53ec2306",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "retriever = vectorstore.as_retriever(search_type=\"mmr\", search_kwargs={\"k\":3})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "1c9181f3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_groq import ChatGroq\n",
+    "llm = ChatGroq(model='openai/gpt-oss-120b', temperature=0.1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "11181278",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_core.prompts import ChatPromptTemplate\n",
+    "\n",
+    "template = \"\"\"\n",
+    "You are a helpful AI assistant. Use the following pieces of context to answer the question at the end. \n",
+    "If you don't know the answer, just say that you don't know, don't try to make up an answer.\n",
+    "Use the information to provide a concise and accurate answer.\n",
+    "Question: {question}\n",
+    "context: {context}\n",
+    "\"\"\"\n",
+    "\n",
+    "prompt = ChatPromptTemplate.from_template(template)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "79752ec8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_core.runnables import RunnablePassthrough\n",
+    "from langchain_core.output_parsers import StrOutputParser\n",
+    "rag_chain = (\n",
+    "    {\"context\": retriever, \"question\": RunnablePassthrough()}\n",
+    "    | prompt\n",
+    "    | llm\n",
+    "    | StrOutputParser()\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "5d88e579",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "**Madam Speaker** – the title used for the presiding officer of the U.S. House of Representatives when that officer is a woman (the Speaker of the House at the time of the address).\n",
+      "\n",
+      "**What her (the address’s) speech is about** – the President’s opening remarks to the joint session of Congress. In this portion he:\n",
+      "\n",
+      "* Acknowledges the recent COVID‑19 pandemic and the fact that the nation is now gathering together again.  \n",
+      "* Calls for bipartisan unity – Democrats, Republicans and Independents – as “Americans” first.  \n",
+      "* Re‑affirms the nation’s commitment to the Constitution and to freedom.  \n",
+      "* Condemns Russia’s invasion of Ukraine, describing Vladimir Putin’s attempt to “shake the foundations of the free world” and praising the courage and determination of the Ukrainian people.  \n",
+      "\n",
+      "So, “Madam Speaker” is the female Speaker of the House, and the speech she is hearing focuses on national recovery, bipartisan unity, and a strong stance against Russian aggression in Ukraine.\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(rag_chain.invoke(\"Who is Madam Speaker and What is Her Speech About?\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a56e9e22",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "RAG Project",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

data/state_of_the_union.txt

@@ -0,0 +1,20 @@
+embedding_model: 
+  provider: "fastembedding"
+  model_name : "BAAI/bge-small-en-v1.5"
+
+retriever:
+  search_type: "mmr"
+  top_k: 3
+
+llm:
+  provider: "langchain_groq"
+  model: "openai/gpt-oss-120b"
+  temperature: 0.1
+  max_tokens: 2048
+
+reranker:
+  model_name: "rank-T5-flan"
+  top_k: 3
+  cache_dir: null
+
+