diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..7ffa8f7 --- /dev/null +++ b/.env.example @@ -0,0 +1,21 @@ +POSTGRES_USER=appuser +POSTGRES_PASSWORD=secret +POSTGRES_DB=appdb +POSTGRES_HOST=postgres +POSTGRES_PORT=5432 + +DATABASE_URL=postgresql+asyncpg://appuser:secret@postgres:5432/appdb +SYNC_DATABASE_URL=postgresql://appuser:secret@postgres:5432/appdb + +REDIS_URL=redis://redis:6379/0 + +GAPGPT_API_KEY= +OPENAI_API_KEY= + +TELEGRAM_BOT_TOKEN= + +RESUME_PATH=/data/resume.txt + +BACKEND_PORT=8000 +FRONTEND_PORT=3000 +TELBOT_PORT=8080 diff --git a/.gitignore b/.gitignore index 3d67c40..9c3887e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .env -extracted_for_ai.text \ No newline at end of file +__pycache__/ + diff --git a/commands/__pycache__/__init__.cpython-310.pyc b/commands/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 111eba7..0000000 Binary files a/commands/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/commands/__pycache__/query_handler.cpython-310.pyc b/commands/__pycache__/query_handler.cpython-310.pyc deleted file mode 100644 index 51d2c49..0000000 Binary files a/commands/__pycache__/query_handler.cpython-310.pyc and /dev/null differ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e4e761f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,62 @@ +version: '3.8' +services: + postgres: + image: postgres:15-alpine + env_file: .env + volumes: + - db_data:/var/lib/postgresql/data + + pgbouncer: + image: edoburu/pgbouncer:latest + volumes: + - ./services/pgbouncer/pgbouncer.ini:/etc/pgbouncer/pgbouncer.ini + - ./services/pgbouncer/userlist.txt:/etc/pgbouncer/userlist.txt + env_file: .env + depends_on: + - postgres + + redis: + image: redis:7-alpine + command: ["redis-server","--maxmemory","256mb","--maxmemory-policy","allkeys-lru"] + + backend: + build: ./services/backend + env_file: .env + ports: + - "${BACKEND_PORT}:8000" + depends_on: + - pgbouncer + - redis + + rq-worker: + build: ./services/backend + env_file: .env + entrypoint: ["rq","worker","--url","${REDIS_URL}","default"] + depends_on: + - backend + - redis + + frontend: + build: ./services/frontend + env_file: .env + ports: + - "${FRONTEND_PORT}:3000" + depends_on: + - backend + + telbot: + build: ./services/telbot + env_file: .env + depends_on: + - postgres + command: ["python","bot.py"] + + resume-pipeline: + build: ./services/resume_pipeline + env_file: .env + volumes: + - ./data:/data + command: ["python","run_pipeline.py"] + +volumes: + db_data: diff --git a/requirements.txt b/requirements.txt index ad2f548..0aacb38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,6 @@ langchain_openai sqlalchemy tenacity matplotlib +psycopg2-binary +requests +beautifulsoup4 diff --git a/commands/__init__.py b/services/__init__.py similarity index 100% rename from commands/__init__.py rename to services/__init__.py diff --git a/services/backend/Dockerfile b/services/backend/Dockerfile new file mode 100644 index 0000000..d89bca0 --- /dev/null +++ b/services/backend/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.11-slim AS builder +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +FROM python:3.11-slim +WORKDIR /app +COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages +COPY . . +CMD ["uvicorn","main:app","--host","0.0.0.0","--port","8000"] diff --git a/services/backend/__init__.py b/services/backend/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/backend/db.py b/services/backend/db.py new file mode 100644 index 0000000..7993729 --- /dev/null +++ b/services/backend/db.py @@ -0,0 +1,11 @@ +import os +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession +from sqlalchemy.orm import sessionmaker + +DATABASE_URL = os.getenv("DATABASE_URL") +engine = create_async_engine(DATABASE_URL, echo=False) +AsyncSessionLocal = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + +async def get_db(): + async with AsyncSessionLocal() as session: + yield session diff --git a/services/backend/main.py b/services/backend/main.py new file mode 100644 index 0000000..b6ab15f --- /dev/null +++ b/services/backend/main.py @@ -0,0 +1,11 @@ +from fastapi import FastAPI, Depends +from sqlalchemy import select +from .db import get_db +from .models import Item + +app = FastAPI() + +@app.get("/items") +async def list_items(db=Depends(get_db)): + result = await db.execute(select(Item)) + return result.scalars().all() diff --git a/services/backend/models.py b/services/backend/models.py new file mode 100644 index 0000000..80c4d17 --- /dev/null +++ b/services/backend/models.py @@ -0,0 +1,11 @@ +from sqlalchemy.orm import declarative_base +from sqlalchemy import Column, Integer, String, Text + +Base = declarative_base() + +class Item(Base): + __tablename__ = "items" + id = Column(Integer, primary_key=True) + name = Column(String(100)) + description = Column(Text) + status = Column(String(20)) diff --git a/services/backend/requirements.txt b/services/backend/requirements.txt new file mode 100644 index 0000000..c46f197 --- /dev/null +++ b/services/backend/requirements.txt @@ -0,0 +1,7 @@ +fastapi +uvicorn[standard] +asyncpg +sqlalchemy +psycopg2-binary +redis +rq diff --git a/services/backend/rq_worker.py b/services/backend/rq_worker.py new file mode 100644 index 0000000..ad6d7d0 --- /dev/null +++ b/services/backend/rq_worker.py @@ -0,0 +1,11 @@ +import os +from rq import Worker, Queue, Connection +import redis + +redis_url = os.getenv("REDIS_URL") +conn = redis.from_url(redis_url) + +if __name__ == "__main__": + with Connection(conn): + worker = Worker(["default"]) + worker.work() diff --git a/services/frontend/Dockerfile b/services/frontend/Dockerfile new file mode 100644 index 0000000..75d5a75 --- /dev/null +++ b/services/frontend/Dockerfile @@ -0,0 +1,9 @@ +FROM node:18-alpine AS builder +WORKDIR /app +COPY package.json yarn.lock ./ +RUN yarn install +COPY . . +RUN yarn build + +FROM nginx:alpine +COPY --from=builder /app/build /usr/share/nginx/html diff --git a/services/frontend/package.json b/services/frontend/package.json new file mode 100644 index 0000000..1867c57 --- /dev/null +++ b/services/frontend/package.json @@ -0,0 +1,11 @@ +{ + "name": "frontend", + "version": "1.0.0", + "dependencies": { + "react": "^18.0.0", + "react-dom": "^18.0.0" + }, + "scripts": { + "build": "echo building frontend" + } +} diff --git a/services/frontend/src/components/ItemsTable.jsx b/services/frontend/src/components/ItemsTable.jsx new file mode 100644 index 0000000..9851d41 --- /dev/null +++ b/services/frontend/src/components/ItemsTable.jsx @@ -0,0 +1,3 @@ +export default function ItemsTable() { + return null; +} diff --git a/services/frontend/src/index.jsx b/services/frontend/src/index.jsx new file mode 100644 index 0000000..631f94b --- /dev/null +++ b/services/frontend/src/index.jsx @@ -0,0 +1 @@ +console.log('frontend placeholder'); diff --git a/services/frontend/yarn.lock b/services/frontend/yarn.lock new file mode 100644 index 0000000..a64f6fb --- /dev/null +++ b/services/frontend/yarn.lock @@ -0,0 +1 @@ +# lockfile diff --git a/services/pgbouncer/pgbouncer.ini b/services/pgbouncer/pgbouncer.ini new file mode 100644 index 0000000..9f9d35b --- /dev/null +++ b/services/pgbouncer/pgbouncer.ini @@ -0,0 +1,6 @@ +[databases] +appdb = host=postgres port=5432 dbname=appdb user=appuser password=secret + +[pgbouncer] +listen_port = 6432 +listen_addr = 0.0.0.0 diff --git a/services/pgbouncer/userlist.txt b/services/pgbouncer/userlist.txt new file mode 100644 index 0000000..e652c79 --- /dev/null +++ b/services/pgbouncer/userlist.txt @@ -0,0 +1 @@ +"appuser" "secret" diff --git a/services/postgres/init.sql b/services/postgres/init.sql new file mode 100644 index 0000000..a1b4570 --- /dev/null +++ b/services/postgres/init.sql @@ -0,0 +1,13 @@ +-- optional schema +CREATE TABLE IF NOT EXISTS jobs ( + id SERIAL PRIMARY KEY, + url TEXT, + page_html TEXT, + requirements TEXT +); +CREATE TABLE IF NOT EXISTS items ( + id SERIAL PRIMARY KEY, + name TEXT, + description TEXT, + status TEXT +); diff --git a/services/resume_pipeline/Dockerfile b/services/resume_pipeline/Dockerfile new file mode 100644 index 0000000..ac46952 --- /dev/null +++ b/services/resume_pipeline/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.11-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY . . +CMD ["python","run_pipeline.py"] diff --git a/services/resume_pipeline/__init__.py b/services/resume_pipeline/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/resume_pipeline/database.py b/services/resume_pipeline/database.py new file mode 100644 index 0000000..8bc8cd7 --- /dev/null +++ b/services/resume_pipeline/database.py @@ -0,0 +1,35 @@ +import os +from contextlib import contextmanager +import psycopg2 +from psycopg2.pool import SimpleConnectionPool + +DATABASE_URL = os.getenv("DATABASE_URL") + +if not DATABASE_URL: + raise RuntimeError("DATABASE_URL environment variable is required") + +pool = SimpleConnectionPool(minconn=1, maxconn=5, dsn=DATABASE_URL) + +@contextmanager +def get_conn(): + conn = pool.getconn() + try: + yield conn + finally: + pool.putconn(conn) + +def fetch_job_urls(limit=10): + """Return a list of (id, url) tuples from the jobs table.""" + query = "SELECT id, url FROM jobs WHERE url IS NOT NULL LIMIT %s" + with get_conn() as conn: + with conn.cursor() as cur: + cur.execute(query, (limit,)) + return cur.fetchall() + +def store_job_page(job_id, html, requirements_text): + """Store downloaded HTML and extracted requirements for a job.""" + query = "UPDATE jobs SET page_html=%s, requirements=%s WHERE id=%s" + with get_conn() as conn: + with conn.cursor() as cur: + cur.execute(query, (html, requirements_text, job_id)) + conn.commit() diff --git a/services/resume_pipeline/requirements.txt b/services/resume_pipeline/requirements.txt new file mode 100644 index 0000000..513f296 --- /dev/null +++ b/services/resume_pipeline/requirements.txt @@ -0,0 +1,4 @@ +psycopg2-binary +requests +beautifulsoup4 +openai diff --git a/services/resume_pipeline/resume_generator.py b/services/resume_pipeline/resume_generator.py new file mode 100644 index 0000000..c7e4b5d --- /dev/null +++ b/services/resume_pipeline/resume_generator.py @@ -0,0 +1,17 @@ +import os +import openai + +openai.api_key = os.getenv("OPENAI_API_KEY") + +PROMPT = ( + "You are a helpful assistant that rewrites a resume to match a job description." + "Return the improved resume in plain text." +) + +def generate_resume(resume_text: str, job_requirements: str) -> str: + messages = [ + {"role": "system", "content": PROMPT}, + {"role": "user", "content": f"Resume:\n{resume_text}\n\nJob requirements:\n{job_requirements}"}, + ] + resp = openai.ChatCompletion.create(model="gpt-3.5-turbo", messages=messages) + return resp.choices[0].message.content.strip() diff --git a/services/resume_pipeline/resume_parser.py b/services/resume_pipeline/resume_parser.py new file mode 100644 index 0000000..70310e0 --- /dev/null +++ b/services/resume_pipeline/resume_parser.py @@ -0,0 +1,8 @@ +from pathlib import Path + +def load_resume(path: str) -> str: + """Load resume text from a file.""" + p = Path(path) + if not p.exists(): + raise FileNotFoundError(path) + return p.read_text() diff --git a/services/resume_pipeline/run_pipeline.py b/services/resume_pipeline/run_pipeline.py new file mode 100644 index 0000000..162eeef --- /dev/null +++ b/services/resume_pipeline/run_pipeline.py @@ -0,0 +1,25 @@ +import os +from database import fetch_job_urls, store_job_page +from scraper import download_job_page, parse_requirements +from resume_parser import load_resume +from resume_generator import generate_resume + +RESUME_PATH = os.getenv("RESUME_PATH", "resume.txt") + + +def main(): + resume_text = load_resume(RESUME_PATH) + jobs = fetch_job_urls(limit=1) + for job_id, url in jobs: + html = download_job_page(url) + reqs = parse_requirements(html) + new_resume = generate_resume(resume_text, reqs) + store_job_page(job_id, html, reqs) + out_file = f"resume_{job_id}.txt" + with open(out_file, "w") as f: + f.write(new_resume) + print(f"Generated resume for job {job_id} -> {out_file}") + + +if __name__ == "__main__": + main() diff --git a/services/resume_pipeline/scraper.py b/services/resume_pipeline/scraper.py new file mode 100644 index 0000000..043a35a --- /dev/null +++ b/services/resume_pipeline/scraper.py @@ -0,0 +1,26 @@ +import requests +from bs4 import BeautifulSoup +from typing import Tuple + +HEADERS = {"User-Agent": "Mozilla/5.0"} + +def download_job_page(url: str) -> str: + resp = requests.get(url, headers=HEADERS, timeout=10) + resp.raise_for_status() + return resp.text + +def parse_requirements(html: str) -> str: + soup = BeautifulSoup(html, "html.parser") + # Simple heuristic: extract bullet lists under sections containing 'requirement' + sections = soup.find_all(text=lambda t: t and 'require' in t.lower()) + if not sections: + return "" + # Gather sibling lists or paragraphs + requirements = [] + for s in sections: + parent = s.parent + ul = parent.find_next("ul") + if ul: + items = [li.get_text(" ", strip=True) for li in ul.find_all("li")] + requirements.extend(items) + return "\n".join(requirements) diff --git a/services/telbot/Dockerfile b/services/telbot/Dockerfile new file mode 100644 index 0000000..3aa49a4 --- /dev/null +++ b/services/telbot/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.11-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY . . +CMD ["python","bot.py"] diff --git a/services/telbot/__init__.py b/services/telbot/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bot.py b/services/telbot/bot.py similarity index 100% rename from bot.py rename to services/telbot/bot.py diff --git a/services/telbot/commands/__init__.py b/services/telbot/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/commands/query_handler.py b/services/telbot/commands/query_handler.py similarity index 100% rename from commands/query_handler.py rename to services/telbot/commands/query_handler.py diff --git a/commands/query_handler.py.old b/services/telbot/commands/query_handler.py.old similarity index 100% rename from commands/query_handler.py.old rename to services/telbot/commands/query_handler.py.old diff --git a/commands/query_handler.py.old2 b/services/telbot/commands/query_handler.py.old2 similarity index 100% rename from commands/query_handler.py.old2 rename to services/telbot/commands/query_handler.py.old2 diff --git a/commands/query_handler.py.withopenaiapi b/services/telbot/commands/query_handler.py.withopenaiapi similarity index 100% rename from commands/query_handler.py.withopenaiapi rename to services/telbot/commands/query_handler.py.withopenaiapi diff --git a/services/telbot/requirements.txt b/services/telbot/requirements.txt new file mode 100644 index 0000000..0aacb38 --- /dev/null +++ b/services/telbot/requirements.txt @@ -0,0 +1,16 @@ +python-telegram-bot==13.7 +pandas +numpy +plotly +python-dotenv +openai +langchain +langchain_community +langchain_experimental +langchain_openai +sqlalchemy +tenacity +matplotlib +psycopg2-binary +requests +beautifulsoup4 diff --git a/test_openai_api.py b/test_openai_api.py index db6bbd9..c25634b 100644 --- a/test_openai_api.py +++ b/test_openai_api.py @@ -1,49 +1,30 @@ #!/usr/bin/env python3 import os -import sys +import pytest try: import openai -except ImportError: - print("Please install the OpenAI library first:\n pip install openai") - sys.exit(1) +except ImportError: # pragma: no cover - dependency missing + openai = None + def test_openai_api(): - # 1. Load API key + if openai is None: + pytest.skip("openai package not installed") api_key = os.getenv("OPENAI_API_KEY") if not api_key: - print("ERROR: OPENAI_API_KEY environment variable is not set.") - return False + pytest.skip("OPENAI_API_KEY not configured") openai.api_key = api_key + resp = openai.models.list() + assert hasattr(resp, "data") - # 2. Try to list models using the new API - try: - print("Listing models…") - resp = openai.models.list() - # resp is a SyncPage of Model objects, so use .data - models = resp.data - print(f"✅ Successfully retrieved {len(models)} models.") - except Exception as e: - print(f"❌ Failed to list models: {e}") - return False - - # 3. Send a quick chat completion - try: - print("Sending test completion…") - chat = openai.ChatCompletion.create( - model="gpt-3.5-turbo", # widely available model - messages=[{"role": "user", "content": "Say hello in one word."}], - temperature=0 - ) - content = chat.choices[0].message.content.strip() - print(f"✅ Completion succeeded: “{content}”") - except Exception as e: - print(f"❌ Chat completion failed: {e}") - return False - - return True + chat = openai.ChatCompletion.create( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Say hello in one word."}], + temperature=0, + ) + assert chat.choices[0].message.content if __name__ == "__main__": - success = test_openai_api() - sys.exit(0 if success else 1) + test_openai_api() diff --git a/tests/test_scraper.py b/tests/test_scraper.py new file mode 100644 index 0000000..4352094 --- /dev/null +++ b/tests/test_scraper.py @@ -0,0 +1,13 @@ +from services.resume_pipeline.scraper import parse_requirements + +sample_html = """ +
+