requirements.txt

# Core Python Tools
pydantic>=1.10.0
ftfy>=6.1.1
fitz                       # PyMuPDF, ensure installed via pip install pymupdf
uuid                       # For generating unique identifiers
re                         # Regular expressions
enum                       # Enumerations
dataclasses                # For data structure definitions
collections.OrderedDict    # For maintaining order in dictionaries
typing                     # For type hints (List, Dict, Tuple, Optional)
sys                        # System-specific parameters and functions
os                         # File and directory handling
logging                    # Configurable logging for debugging and tracing
json                       # JSON file parsing and manipulation

unstructured>=0.10.0  # for PDF parsing (partition_pdf)

# Core Dependencies
numpy>=1.21.0
pandas>=1.3.0
scipy>=1.7.0

# Machine Learning & NLP
torch>=1.11.0
transformers>=4.20.0
sentence-transformers>=2.2.0
scikit-learn>=1.1.0
nltk>=3.7
spacy>=3.4.0
evaluate>=0.4.0

# OpenAI & LLM APIs
openai>=0.27.0
anthropic>=0.3.0

# Knowledge Graph Processing
rdflib>=6.2.0
networkx>=2.8.0
owlrl>=6.0.2

# Data Processing & Web Scraping
requests>=2.28.0
beautifulsoup4>=4.11.0
lxml>=4.9.0
jsonlines>=3.1.0

# Evaluation Metrics
rouge-score>=0.1.2
bert-score>=0.3.12
bleu>=0.2.0
sacrebleu>=2.2.0

# Visualization & Plotting
matplotlib>=3.5.0
seaborn>=0.11.0
plotly>=5.10.0

# Jupyter & Development
jupyter>=1.0.0
notebook>=6.4.0
ipywidgets>=7.7.0

# Utilities
tqdm>=4.64.0
python-dotenv>=0.20.0
configparser>=5.2.0

# Statistical Analysis
statsmodels>=0.13.0
pingouin>=0.5.0

# Text Readability & Analysis
textstat>=0.7.3
readability>=0.3.1