diff --git a/DeepCode/.github/ISSUE_TEMPLATE/bug_report.yml b/DeepCode/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 00000000..daf9200e --- /dev/null +++ b/DeepCode/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,61 @@ +name: Bug Report +description: File a bug report +title: "[Bug]:" +labels: ["bug", "triage"] + +body: + - type: checkboxes + id: existingcheck + attributes: + label: Do you need to file an issue? + description: Please help us manage our time by avoiding duplicates and common bugs with the steps below. + options: + - label: I have searched the existing issues and this bug is not already filed. + - label: I believe this is a legitimate bug, not just a question or feature request. + - type: textarea + id: description + attributes: + label: Describe the bug + description: A clear and concise description of what the bug is. + placeholder: What went wrong? + - type: textarea + id: reproduce + attributes: + label: Steps to reproduce + description: Steps to reproduce the behavior. + placeholder: How can we replicate the issue? + - type: textarea + id: expected_behavior + attributes: + label: Expected Behavior + description: A clear and concise description of what you expected to happen. + placeholder: What should have happened? + - type: textarea + id: configused + attributes: + label: DeepCode Config Used + description: The DeepCode configuration used for the run. + placeholder: The settings content or DeepCode configuration + value: | + # Paste your config here + - type: textarea + id: screenshotslogs + attributes: + label: Logs and screenshots + description: If applicable, add screenshots and logs to help explain your problem. + placeholder: Add logs and screenshots here + - type: textarea + id: additional_information + attributes: + label: Additional Information + description: | + - DeepCode Version: e.g., v0.1.1 + - Operating System: e.g., Windows 10, Ubuntu 20.04 + - Python Version: e.g., 3.8 + - Related Issues: e.g., #1 + - Any other relevant information. + value: | + - DeepCode Version: + - Operating System: + - Python Version: + - Related Issues: diff --git a/DeepCode/.github/ISSUE_TEMPLATE/config.yml b/DeepCode/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..3ba13e0c --- /dev/null +++ b/DeepCode/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false diff --git a/DeepCode/.github/ISSUE_TEMPLATE/feature_request.yml b/DeepCode/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 00000000..3c127d59 --- /dev/null +++ b/DeepCode/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,26 @@ +name: Feature Request +description: File a feature request +labels: ["enhancement"] +title: "[Feature Request]:" + +body: + - type: checkboxes + id: existingcheck + attributes: + label: Do you need to file a feature request? + description: Please help us manage our time by avoiding duplicates and common feature request with the steps below. + options: + - label: I have searched the existing feature request and this feature request is not already filed. + - label: I believe this is a legitimate feature request, not just a question or bug. + - type: textarea + id: feature_request_description + attributes: + label: Feature Request Description + description: A clear and concise description of the feature request you would like. + placeholder: What this feature request add more or improve? + - type: textarea + id: additional_context + attributes: + label: Additional Context + description: Add any other context or screenshots about the feature request here. + placeholder: Any additional information diff --git a/DeepCode/.github/ISSUE_TEMPLATE/question.yml b/DeepCode/.github/ISSUE_TEMPLATE/question.yml new file mode 100644 index 00000000..26fed23d --- /dev/null +++ b/DeepCode/.github/ISSUE_TEMPLATE/question.yml @@ -0,0 +1,26 @@ +name: Question +description: Ask a general question +labels: ["question"] +title: "[Question]:" + +body: + - type: checkboxes + id: existingcheck + attributes: + label: Do you need to ask a question? + description: Please help us manage our time by avoiding duplicates and common questions with the steps below. + options: + - label: I have searched the existing question and discussions and this question is not already answered. + - label: I believe this is a legitimate question, not just a bug or feature request. + - type: textarea + id: question + attributes: + label: Your Question + description: A clear and concise description of your question. + placeholder: What is your question? + - type: textarea + id: context + attributes: + label: Additional Context + description: Provide any additional context or details that might help us understand your question better. + placeholder: Add any relevant information here diff --git a/DeepCode/.github/dependabot.yml b/DeepCode/.github/dependabot.yml new file mode 100644 index 00000000..9d866e39 --- /dev/null +++ b/DeepCode/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "pip" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "weekly" diff --git a/DeepCode/.github/pull_request_template.md b/DeepCode/.github/pull_request_template.md new file mode 100644 index 00000000..9cfe531e --- /dev/null +++ b/DeepCode/.github/pull_request_template.md @@ -0,0 +1,32 @@ + + +## Description + +[Briefly describe the changes made in this pull request.] + +## Related Issues + +[Reference any related issues or tasks addressed by this pull request.] + +## Changes Made + +[List the specific changes made in this pull request.] + +## Checklist + +- [ ] Changes tested locally +- [ ] Code reviewed +- [ ] Documentation updated (if necessary) +- [ ] Unit tests added (if applicable) + +## Additional Notes + +[Add any additional notes or context for the reviewer(s).] diff --git a/DeepCode/.github/workflows/linting.yaml b/DeepCode/.github/workflows/linting.yaml new file mode 100644 index 00000000..aa054369 --- /dev/null +++ b/DeepCode/.github/workflows/linting.yaml @@ -0,0 +1,30 @@ +name: Linting and Formatting + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + lint-and-format: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pre-commit + + - name: Run pre-commit + run: pre-commit run --all-files --show-diff-on-failure diff --git a/DeepCode/.github/workflows/pypi-publish.yml b/DeepCode/.github/workflows/pypi-publish.yml new file mode 100644 index 00000000..d4e144af --- /dev/null +++ b/DeepCode/.github/workflows/pypi-publish.yml @@ -0,0 +1,52 @@ +name: Upload DeepCode Package + +on: + release: + types: [published] + +permissions: + contents: read + +jobs: + release-build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Build release distributions + run: | + python -m pip install build + python -m build + + - name: Upload distributions + uses: actions/upload-artifact@v4 + with: + name: release-dists + path: dist/ + + pypi-publish: + runs-on: ubuntu-latest + needs: + - release-build + permissions: + id-token: write + + environment: + name: pypi + + steps: + - name: Retrieve release distributions + uses: actions/download-artifact@v4 + with: + name: release-dists + path: dist/ + + - name: Publish release distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/ diff --git a/DeepCode/.gitignore b/DeepCode/.gitignore new file mode 100644 index 00000000..0f34269d --- /dev/null +++ b/DeepCode/.gitignore @@ -0,0 +1,66 @@ +# Python-related files +__pycache__/ +*.py[cod] +*.egg-info/ +.eggs/ +*.tgz +*.tar.gz +*.ini + +# Virtual Environment +.venv/ +env/ +venv/ +*.env* +.env_example + + +# Build / Distribution +dist/ +build/ +site/ + +# Logs / Reports +*.log +*.log.* +*.logfire +*.coverage/ +log/ + +# Caches +.cache/ +.mypy_cache/ +.pytest_cache/ +.ruff_cache/ +.gradio/ +.history/ +temp/ + +# IDE / Editor Files +.idea/ +.vscode/ +.vscode/settings.json + +# Framework-specific files +local_neo4jWorkDir/ +neo4jWorkDir/ + +# Data & Storage +inputs/ +rag_storage/ +examples/input/ +examples/output/ +deepcode-mcp/agent_folders + +# Miscellaneous +.DS_Store +TODO.md +ignore_this.txt +*.ignore.* + +# unit-test files +test_* +run_indexer_with_filtering.py + +# Cline files +memory-bank/ diff --git a/DeepCode/.pre-commit-config.yaml b/DeepCode/.pre-commit-config.yaml new file mode 100644 index 00000000..2634f83c --- /dev/null +++ b/DeepCode/.pre-commit-config.yaml @@ -0,0 +1,21 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: requirements-txt-fixer + + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.4 + hooks: + - id: ruff-format + - id: ruff + args: [--fix, --ignore=E402] + + - repo: https://github.com/mgedmin/check-manifest + rev: "0.49" + hooks: + - id: check-manifest + stages: [manual] diff --git a/DeepCode/LICENSE b/DeepCode/LICENSE new file mode 100644 index 00000000..b3ba37ce --- /dev/null +++ b/DeepCode/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 โœจData Intelligence Lab@HKUโœจ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/DeepCode/MANIFEST.in b/DeepCode/MANIFEST.in new file mode 100644 index 00000000..36569a3d --- /dev/null +++ b/DeepCode/MANIFEST.in @@ -0,0 +1,19 @@ +include README.md +include LICENSE +include requirements.txt +include __init__.py +include *.png +include *.yaml +recursive-include config *.yaml +recursive-include prompts * +recursive-include schema * +recursive-include ui *.py +recursive-include cli *.py +recursive-include utils *.py +recursive-include tools *.py +recursive-include workflows *.py +global-exclude *.pyc +global-exclude .git* +global-exclude .history* +global-exclude .ruff_cache* +global-exclude __pycache__* diff --git a/DeepCode/README.md b/DeepCode/README.md new file mode 100644 index 00000000..cebb54b5 --- /dev/null +++ b/DeepCode/README.md @@ -0,0 +1,117 @@ +# DeepCode + +This is an implementation of the paper ["DeepCode: Open Agentic Coding"](https://www.alphaxiv.org/abs/2512.07921). DeepCode is a fully autonomous multi-agent framework for high-fidelity document-to-repository synthesis, solving the context bottleneck in LLM-based coding through principled information-flow management. + +DeepCode achieves state-of-the-art performance on the PaperBench benchmark, decisively outperforming leading commercial agents such as Cursor and Claude Code, and crucially, surpassing PhD-level human experts from top institutes on key reproduction metrics. Read the paper [here](https://www.alphaxiv.org/abs/2512.07921). + +## Quickstart + +### One-Line Setup with speedrun_setup.sh + +The easiest way to get started is using our `speedrun_setup.sh` script that handles everything: + +```bash +# Run the setup wizard +bash speedrun_setup.sh +``` + +The script automatically: +- Installs `uv` package manager if not present +- Creates Python virtual environment with `uv venv` +- Installs all dependencies +- Configures search servers (Brave or Bocha-MCP) +- Sets up LLM provider (OpenAI, Anthropic, or Google) +- Configures API keys +- Sets up Windows MCP servers (Windows only) + +### Manual Setup + +If you prefer manual setup or want to install the DeepCode package directly, refer to: [Quick Start Guide](https://github.com/HKUDS/DeepCode?tab=readme-ov-file#-quick-start) + +### Launching DeepCode + +After setup, you can launch DeepCode in two ways: + +**Web Interface:** +```bash +python deepcode.py +``` + +**CLI Interface:** +```bash +python cli/main_cli.py +``` + +## Features + +### Core Capabilities + +- **๐Ÿ“„ Document Parsing**: Processes research papers (PDF/Markdown) into structured semantic chunks +- **๐Ÿง  Multi-Agent Planning**: Synthesizes detailed implementation blueprints using Concept, Algorithm, and Planning agents +- **๐Ÿ’พ CodeMem**: Graph-based memory system maintaining cross-file consistency +- **๐Ÿ” CodeRAG**: Retrieval-Augmented Generation for external knowledge injection +- **โœ… Verification Swarm**: Two-stage verification (Static Analysis & Dynamic Execution) +- **๐Ÿณ Docker Sandbox**: Secure execution environment for running generated code + +### Supported LLM Providers + +- **OpenAI** (GPT-4o, GPT-4, etc.) +- **Anthropic** (Claude Sonnet, Claude Opus, etc.) +- **Google** (Gemini Pro, etc.) + +### Search Integration + +- **Brave Search**: Web search for finding similar repositories and code examples +- **Bocha-MCP**: Alternative search server option + +## Architecture + +DeepCode uses a multi-agent architecture: + +- **๐ŸŽฏ Central Orchestration Agent**: Coordinates workflow execution and strategic decisions +- **๐Ÿ“ Requirement Analysis Agent**: Performs deep semantic analysis of user requirements +- **๐Ÿ“„ Document Segmentation Agent**: Processes complex technical documents and research papers +- **๐Ÿ—๏ธ Code Planning Agent**: Executes architectural design and technology stack optimization +- **๐Ÿ” Code Reference Mining Agent**: Discovers relevant repositories and frameworks +- **๐Ÿ“š Code Indexing Agent**: Builds comprehensive knowledge graphs of discovered codebases +- **๐Ÿงฌ Code Implementation Agent**: Synthesizes collected information into executable code + +All agents communicate via **Model Context Protocol (MCP)** for standardized tool integration. + +## Self-Reproduction: DeepCode on DeepCode + +As a demonstration of its capabilities, DeepCode was run on its own paper ("DeepCode: Open Agentic Coding"). The framework successfully processed the 23-page research paper and generated a complete reproduction repository. The repo was generated with gemini-3-pro-preview, Brave Search, and with fast mode enabled. + +### Generated Repository Structure + +The self-reproduction created a fully functional implementation in `deepcode_lab/papers/1/generate_code/deepcode_repro/`: + +**Core Components:** +- `src/core/document_parser.py` - Hierarchical content segmentation +- `src/core/memory.py` - CodeMem graph-based memory system +- `src/core/rag_engine.py` - CodeRAG retrieval engine +- `src/core/blueprint.py` - Blueprint schema definitions + +**Agent Implementations:** +- `src/agents/planning.py` - Concept, Algorithm, and Planning agents +- `src/agents/coding.py` - Code generation and summarization agents +- `src/agents/verification.py` - Static analysis and sandbox verification agents +- `src/agents/base.py` - LLM wrapper with MCP tool integration + +**Infrastructure:** +- `docker/Dockerfile` - Secure sandbox environment +- `experiments/run_paperbench.py` - PaperBench evaluation script +- `experiments/validate_repro.py` - Validation harness +- Complete `README.md` with installation and usage instructions + +## Troubleshooting + +- **UV not found**: Make sure `uv` is installed and in your PATH. Restart your terminal after installation. +- **Python version**: DeepCode requires Python 3.13+. Use `uv venv --python=3.13` to create the environment. +- **API key errors**: Verify your API keys are correctly set in `mcp_agent.secrets.yaml` and match your selected LLM provider. +- **Windows MCP servers**: Ensure npm is installed and MCP servers are installed globally before running the Windows configuration step. +- **Virtual environment**: Always activate the virtual environment before running DeepCode: `source .venv/bin/activate` (Linux/macOS) or `.venv\Scripts\activate` (Windows) + +## Source Code + +This implementation is based on the original DeepCode framework. Source code: https://github.com/HKUDS/DeepCode diff --git a/DeepCode/__init__.py b/DeepCode/__init__.py new file mode 100644 index 00000000..2594402a --- /dev/null +++ b/DeepCode/__init__.py @@ -0,0 +1,21 @@ +""" +DeepCode - AI Research Engine + +๐Ÿงฌ Next-Generation AI Research Automation Platform +โšก Transform research papers into working code automatically +""" + +__version__ = "1.0.8" +__author__ = "DeepCode Team" +__url__ = "https://github.com/HKUDS/DeepCode" + +# Import main components for easy access +from utils import FileProcessor, DialogueLogger + +__all__ = [ + "FileProcessor", + "DialogueLogger", + "__version__", + "__author__", + "__url__", +] diff --git a/DeepCode/assets/625302d6ce6e8bd6be7c3e6969cdab4e.JPG b/DeepCode/assets/625302d6ce6e8bd6be7c3e6969cdab4e.JPG new file mode 100644 index 00000000..65791a63 Binary files /dev/null and b/DeepCode/assets/625302d6ce6e8bd6be7c3e6969cdab4e.JPG differ diff --git a/DeepCode/assets/Deepcode.png b/DeepCode/assets/Deepcode.png new file mode 100644 index 00000000..0db865d7 Binary files /dev/null and b/DeepCode/assets/Deepcode.png differ diff --git a/DeepCode/assets/icons/access_credentials.png b/DeepCode/assets/icons/access_credentials.png new file mode 100644 index 00000000..799c54be Binary files /dev/null and b/DeepCode/assets/icons/access_credentials.png differ diff --git a/DeepCode/assets/icons/chat_interface.png b/DeepCode/assets/icons/chat_interface.png new file mode 100644 index 00000000..9854a3e8 Binary files /dev/null and b/DeepCode/assets/icons/chat_interface.png differ diff --git a/DeepCode/assets/icons/core_metrics.png b/DeepCode/assets/icons/core_metrics.png new file mode 100644 index 00000000..7e7d86ab Binary files /dev/null and b/DeepCode/assets/icons/core_metrics.png differ diff --git a/DeepCode/assets/icons/download.png b/DeepCode/assets/icons/download.png new file mode 100644 index 00000000..051014c6 Binary files /dev/null and b/DeepCode/assets/icons/download.png differ diff --git a/DeepCode/assets/icons/feature_cognition.png b/DeepCode/assets/icons/feature_cognition.png new file mode 100644 index 00000000..cfcea207 Binary files /dev/null and b/DeepCode/assets/icons/feature_cognition.png differ diff --git a/DeepCode/assets/icons/feature_hyper.png b/DeepCode/assets/icons/feature_hyper.png new file mode 100644 index 00000000..4f22e163 Binary files /dev/null and b/DeepCode/assets/icons/feature_hyper.png differ diff --git a/DeepCode/assets/icons/feature_secure.png b/DeepCode/assets/icons/feature_secure.png new file mode 100644 index 00000000..e3b2ccbe Binary files /dev/null and b/DeepCode/assets/icons/feature_secure.png differ diff --git a/DeepCode/assets/icons/feature_synthesis.png b/DeepCode/assets/icons/feature_synthesis.png new file mode 100644 index 00000000..2fc4ff30 Binary files /dev/null and b/DeepCode/assets/icons/feature_synthesis.png differ diff --git a/DeepCode/assets/icons/operation_result.png b/DeepCode/assets/icons/operation_result.png new file mode 100644 index 00000000..be55f145 Binary files /dev/null and b/DeepCode/assets/icons/operation_result.png differ diff --git a/DeepCode/assets/icons/processing_core.png b/DeepCode/assets/icons/processing_core.png new file mode 100644 index 00000000..004c8679 Binary files /dev/null and b/DeepCode/assets/icons/processing_core.png differ diff --git a/DeepCode/assets/icons/runtime_status.png b/DeepCode/assets/icons/runtime_status.png new file mode 100644 index 00000000..21dd605d Binary files /dev/null and b/DeepCode/assets/icons/runtime_status.png differ diff --git a/DeepCode/assets/icons/status_error.png b/DeepCode/assets/icons/status_error.png new file mode 100644 index 00000000..f3f43b9d Binary files /dev/null and b/DeepCode/assets/icons/status_error.png differ diff --git a/DeepCode/assets/icons/status_success.png b/DeepCode/assets/icons/status_success.png new file mode 100644 index 00000000..563a396e Binary files /dev/null and b/DeepCode/assets/icons/status_success.png differ diff --git a/DeepCode/assets/icons/system_diagnostics.png b/DeepCode/assets/icons/system_diagnostics.png new file mode 100644 index 00000000..097fbe1d Binary files /dev/null and b/DeepCode/assets/icons/system_diagnostics.png differ diff --git a/DeepCode/assets/icons/tab_link.png b/DeepCode/assets/icons/tab_link.png new file mode 100644 index 00000000..60100ed2 Binary files /dev/null and b/DeepCode/assets/icons/tab_link.png differ diff --git a/DeepCode/assets/icons/tab_pdf.png b/DeepCode/assets/icons/tab_pdf.png new file mode 100644 index 00000000..5196ca46 Binary files /dev/null and b/DeepCode/assets/icons/tab_pdf.png differ diff --git a/DeepCode/assets/icons/tab_prompt.png b/DeepCode/assets/icons/tab_prompt.png new file mode 100644 index 00000000..100b27f7 Binary files /dev/null and b/DeepCode/assets/icons/tab_prompt.png differ diff --git a/DeepCode/assets/icons/troubleshooting.png b/DeepCode/assets/icons/troubleshooting.png new file mode 100644 index 00000000..a5a6dad4 Binary files /dev/null and b/DeepCode/assets/icons/troubleshooting.png differ diff --git a/DeepCode/assets/icons/workflow_monitor.png b/DeepCode/assets/icons/workflow_monitor.png new file mode 100644 index 00000000..13db5339 Binary files /dev/null and b/DeepCode/assets/icons/workflow_monitor.png differ diff --git a/DeepCode/assets/logo.png b/DeepCode/assets/logo.png new file mode 100644 index 00000000..807cc297 Binary files /dev/null and b/DeepCode/assets/logo.png differ diff --git a/DeepCode/assets/result_main.jpg b/DeepCode/assets/result_main.jpg new file mode 100644 index 00000000..bdbeb915 Binary files /dev/null and b/DeepCode/assets/result_main.jpg differ diff --git a/DeepCode/assets/result_main02.jpg b/DeepCode/assets/result_main02.jpg new file mode 100644 index 00000000..a578e56c Binary files /dev/null and b/DeepCode/assets/result_main02.jpg differ diff --git a/DeepCode/cli/__init__.py b/DeepCode/cli/__init__.py new file mode 100644 index 00000000..313bd6f3 --- /dev/null +++ b/DeepCode/cli/__init__.py @@ -0,0 +1,18 @@ +""" +CLI Module for DeepCode Agent +DeepCodeๆ™บ่ƒฝไฝ“CLIๆจกๅ— + +ๅŒ…ๅซไปฅไธ‹็ป„ไปถ / Contains the following components: +- cli_app: CLIๅบ”็”จไธป็จ‹ๅบ / CLI application main program +- cli_interface: CLI็•Œ้ข็ป„ไปถ / CLI interface components +- cli_launcher: CLIๅฏๅŠจๅ™จ / CLI launcher +""" + +__version__ = "1.0.0" +__author__ = "DeepCode Team - Data Intelligence Lab @ HKU" + +from .cli_app import main as cli_main +from .cli_interface import CLIInterface +from .cli_launcher import main as launcher_main + +__all__ = ["cli_main", "CLIInterface", "launcher_main"] diff --git a/DeepCode/cli/cli_app.py b/DeepCode/cli/cli_app.py new file mode 100644 index 00000000..93f1b495 --- /dev/null +++ b/DeepCode/cli/cli_app.py @@ -0,0 +1,500 @@ +#!/usr/bin/env python3 +""" +DeepCode - CLI Application Main Program +ๆทฑๅบฆไปฃ็  - CLIๅบ”็”จไธป็จ‹ๅบ + +๐Ÿงฌ Open-Source Code Agent by Data Intelligence Lab @ HKU +โšก Revolutionizing research reproducibility through collaborative AI +""" + +import os +import sys +import asyncio +import time +import json + +# ็ฆๆญข็”Ÿๆˆ.pycๆ–‡ไปถ +os.environ["PYTHONDONTWRITEBYTECODE"] = "1" + +# ๆทปๅŠ ้กน็›ฎๆ น็›ฎๅฝ•ๅˆฐ่ทฏๅพ„ +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +if parent_dir not in sys.path: + sys.path.insert(0, parent_dir) + +# ๅฏผๅ…ฅMCPๅบ”็”จๅ’Œๅทฅไฝœๆต + +from cli.workflows import CLIWorkflowAdapter +from cli.cli_interface import CLIInterface, Colors + + +class CLIApp: + """CLIๅบ”็”จไธป็ฑป - ๅ‡็บง็‰ˆๆ™บ่ƒฝไฝ“็ผ–ๆŽ’ๅผ•ๆ“Ž""" + + def __init__(self): + self.cli = CLIInterface() + self.workflow_adapter = CLIWorkflowAdapter(cli_interface=self.cli) + self.app = None # Will be initialized by workflow adapter + self.logger = None + self.context = None + # Document segmentation will be managed by CLI interface + + async def initialize_mcp_app(self): + """ๅˆๅง‹ๅŒ–MCPๅบ”็”จ - ไฝฟ็”จๅทฅไฝœๆต้€‚้…ๅ™จ""" + # Workflow adapter will handle MCP initialization + return await self.workflow_adapter.initialize_mcp_app() + + async def cleanup_mcp_app(self): + """ๆธ…็†MCPๅบ”็”จ - ไฝฟ็”จๅทฅไฝœๆต้€‚้…ๅ™จ""" + await self.workflow_adapter.cleanup_mcp_app() + + async def process_requirement_analysis_non_interactive(self, initial_idea: str): + """ๅค„็†้œ€ๆฑ‚ๅˆ†ๆžๅทฅไฝœๆต๏ผˆ้žไบคไบ’ๅผ๏ผŒ็”จไบŽๅ‘ฝไปค่กŒๅ‚ๆ•ฐ๏ผ‰ (NEW: matching UI version)""" + try: + self.cli.print_separator() + self.cli.print_status( + "๐Ÿง  Starting requirement analysis workflow...", "info" + ) + + # Step 1: Generate guiding questions + self.cli.print_status( + "๐Ÿค– Generating AI-guided questions to refine your requirements...", + "processing", + ) + + questions_result = ( + await self.workflow_adapter.execute_requirement_analysis_workflow( + user_input=initial_idea, analysis_mode="generate_questions" + ) + ) + + if questions_result["status"] != "success": + self.cli.print_status( + f"โŒ Failed to generate questions: {questions_result.get('error', 'Unknown error')}", + "error", + ) + return questions_result + + # Step 2: Display questions + questions_json = questions_result["result"] + self.cli.display_guiding_questions(questions_json) + + # For non-interactive mode, we can't get user answers, so we provide a summary + self.cli.print_status( + "โ„น๏ธ In non-interactive mode, using initial idea for implementation", + "info", + ) + self.cli.print_status( + "๐Ÿ’ก For guided analysis, please use interactive mode (python main_cli.py)", + "info", + ) + + # Proceed directly with the initial idea as the requirement + self.cli.print_status( + "๐Ÿš€ Starting code implementation based on initial requirements...", + "processing", + ) + + implementation_result = await self.process_input(initial_idea, "chat") + + return { + "status": "success", + "questions_generated": questions_result, + "implementation": implementation_result, + } + + except Exception as e: + error_msg = str(e) + self.cli.print_error_box("Requirement Analysis Error", error_msg) + self.cli.print_status( + f"Error during requirement analysis: {error_msg}", "error" + ) + + return {"status": "error", "error": error_msg} + + async def process_requirement_analysis(self): + """ๅค„็†้œ€ๆฑ‚ๅˆ†ๆžๅทฅไฝœๆต๏ผˆไบคไบ’ๅผ๏ผ‰ (NEW: matching UI version)""" + try: + # Step 1: Get initial requirements from user + self.cli.print_separator() + self.cli.print_status( + "๐Ÿง  Starting requirement analysis workflow...", "info" + ) + + user_input = self.cli.get_requirement_analysis_input() + + if not user_input: + self.cli.print_status("Requirement analysis cancelled", "warning") + return {"status": "cancelled"} + + # Step 2: Generate guiding questions + self.cli.print_status( + "๐Ÿค– Generating AI-guided questions to refine your requirements...", + "processing", + ) + + questions_result = ( + await self.workflow_adapter.execute_requirement_analysis_workflow( + user_input=user_input, analysis_mode="generate_questions" + ) + ) + + if questions_result["status"] != "success": + self.cli.print_status( + f"โŒ Failed to generate questions: {questions_result.get('error', 'Unknown error')}", + "error", + ) + return questions_result + + # Step 3: Display questions and get user answers + questions_json = questions_result["result"] + self.cli.display_guiding_questions(questions_json) + + # Ask if user wants to answer the questions + proceed = ( + input( + f"\n{Colors.BOLD}{Colors.YELLOW}Would you like to answer these questions? (y/n):{Colors.ENDC} " + ) + .strip() + .lower() + ) + + if proceed != "y": + self.cli.print_status( + "You can still use the initial requirements for chat input", + "info", + ) + return {"status": "partial", "initial_requirements": user_input} + + user_answers = self.cli.get_question_answers(questions_json) + + # Step 4: Generate requirement summary + self.cli.print_status( + "๐Ÿ“„ Generating detailed requirement document...", "processing" + ) + + summary_result = ( + await self.workflow_adapter.execute_requirement_analysis_workflow( + user_input=user_input, + analysis_mode="summarize_requirements", + user_answers=user_answers, + ) + ) + + if summary_result["status"] != "success": + self.cli.print_status( + f"โŒ Failed to generate summary: {summary_result.get('error', 'Unknown error')}", + "error", + ) + return summary_result + + # Step 5: Display requirement summary + requirement_summary = summary_result["result"] + should_proceed = self.cli.display_requirement_summary(requirement_summary) + + if should_proceed: + # Step 6: Proceed with chat-based implementation + self.cli.print_status( + "๐Ÿš€ Starting code implementation based on analyzed requirements...", + "processing", + ) + + implementation_result = await self.process_input( + requirement_summary, "chat" + ) + + return { + "status": "success", + "requirement_analysis": summary_result, + "implementation": implementation_result, + } + else: + self.cli.print_status( + "Requirement analysis completed. Implementation skipped.", "info" + ) + return { + "status": "success", + "requirement_analysis": summary_result, + "implementation": None, + } + + except Exception as e: + error_msg = str(e) + self.cli.print_error_box("Requirement Analysis Error", error_msg) + self.cli.print_status( + f"Error during requirement analysis: {error_msg}", "error" + ) + + return {"status": "error", "error": error_msg} + + async def process_input(self, input_source: str, input_type: str): + """ๅค„็†่พ“ๅ…ฅๆบ๏ผˆURLๆˆ–ๆ–‡ไปถ๏ผ‰- ไฝฟ็”จๅ‡็บง็‰ˆๆ™บ่ƒฝไฝ“็ผ–ๆŽ’ๅผ•ๆ“Ž""" + try: + # Document segmentation configuration is managed by CLI interface + + self.cli.print_separator() + self.cli.print_status( + "๐Ÿš€ Starting intelligent agent orchestration...", "processing" + ) + + # ๆ˜พ็คบๅค„็†้˜ถๆฎต๏ผˆๆ นๆฎ้…็ฝฎๅ†ณๅฎš๏ผ‰ + chat_mode = input_type == "chat" + self.cli.display_processing_stages( + 0, self.cli.enable_indexing, chat_mode=chat_mode + ) + + # ไฝฟ็”จๅทฅไฝœๆต้€‚้…ๅ™จ่ฟ›่กŒๅค„็† + result = await self.workflow_adapter.process_input_with_orchestration( + input_source=input_source, + input_type=input_type, + enable_indexing=self.cli.enable_indexing, + ) + + if result["status"] == "success": + # ๆ˜พ็คบๅฎŒๆˆ็Šถๆ€ + if chat_mode: + final_stage = 4 + else: + final_stage = 8 if self.cli.enable_indexing else 5 + self.cli.display_processing_stages( + final_stage, self.cli.enable_indexing, chat_mode=chat_mode + ) + self.cli.print_status( + "๐ŸŽ‰ Agent orchestration completed successfully!", "complete" + ) + + # ๆ˜พ็คบ็ป“ๆžœ + self.display_results( + result.get("analysis_result", ""), + result.get("download_result", ""), + result.get("repo_result", ""), + result.get("pipeline_mode", "comprehensive"), + ) + else: + self.cli.print_status( + f"โŒ Processing failed: {result.get('error', 'Unknown error')}", + "error", + ) + + # ๆทปๅŠ ๅˆฐๅŽ†ๅฒ่ฎฐๅฝ• + self.cli.add_to_history(input_source, result) + + return result + + except Exception as e: + error_msg = str(e) + self.cli.print_error_box("Agent Orchestration Error", error_msg) + self.cli.print_status(f"Error during orchestration: {error_msg}", "error") + + # ๆทปๅŠ ้”™่ฏฏๅˆฐๅŽ†ๅฒ่ฎฐๅฝ• + error_result = {"status": "error", "error": error_msg} + self.cli.add_to_history(input_source, error_result) + + return error_result + + def display_results( + self, + analysis_result: str, + download_result: str, + repo_result: str, + pipeline_mode: str = "comprehensive", + ): + """ๆ˜พ็คบๅค„็†็ป“ๆžœ""" + self.cli.print_results_header() + + # ๆ˜พ็คบๆตๆฐด็บฟๆจกๅผ + if pipeline_mode == "chat": + mode_display = "๐Ÿ’ฌ Chat Planning Mode" + elif pipeline_mode == "comprehensive": + mode_display = "๐Ÿง  Comprehensive Mode" + else: + mode_display = "โšก Optimized Mode" + print( + f"{Colors.BOLD}{Colors.PURPLE}๐Ÿค– PIPELINE MODE: {mode_display}{Colors.ENDC}" + ) + self.cli.print_separator("โ”€", 79, Colors.PURPLE) + + print(f"{Colors.BOLD}{Colors.OKCYAN}๐Ÿ“Š ANALYSIS PHASE RESULTS:{Colors.ENDC}") + self.cli.print_separator("โ”€", 79, Colors.CYAN) + + # ๅฐ่ฏ•่งฃๆžๅนถๆ ผๅผๅŒ–ๅˆ†ๆž็ป“ๆžœ + try: + if analysis_result.strip().startswith("{"): + parsed_analysis = json.loads(analysis_result) + print(json.dumps(parsed_analysis, indent=2, ensure_ascii=False)) + else: + print( + analysis_result[:1000] + "..." + if len(analysis_result) > 1000 + else analysis_result + ) + except Exception: + print( + analysis_result[:1000] + "..." + if len(analysis_result) > 1000 + else analysis_result + ) + + print(f"\n{Colors.BOLD}{Colors.PURPLE}๐Ÿ“ฅ DOWNLOAD PHASE RESULTS:{Colors.ENDC}") + self.cli.print_separator("โ”€", 79, Colors.PURPLE) + print( + download_result[:1000] + "..." + if len(download_result) > 1000 + else download_result + ) + + print( + f"\n{Colors.BOLD}{Colors.GREEN}โš™๏ธ IMPLEMENTATION PHASE RESULTS:{Colors.ENDC}" + ) + self.cli.print_separator("โ”€", 79, Colors.GREEN) + print(repo_result[:1000] + "..." if len(repo_result) > 1000 else repo_result) + + # ๅฐ่ฏ•ๆๅ–็”Ÿๆˆ็š„ไปฃ็ ็›ฎๅฝ•ไฟกๆฏ + if "Code generated in:" in repo_result: + code_dir = ( + repo_result.split("Code generated in:")[-1].strip().split("\n")[0] + ) + print( + f"\n{Colors.BOLD}{Colors.YELLOW}๐Ÿ“ Generated Code Directory: {Colors.ENDC}{code_dir}" + ) + + # ๆ˜พ็คบๅค„็†ๅฎŒๆˆ็š„ๅทฅไฝœๆต้˜ถๆฎต + print( + f"\n{Colors.BOLD}{Colors.OKCYAN}๐Ÿ”„ COMPLETED WORKFLOW STAGES:{Colors.ENDC}" + ) + + if pipeline_mode == "chat": + stages = [ + "๐Ÿš€ Engine Initialization", + "๐Ÿ’ฌ Requirements Analysis", + "๐Ÿ—๏ธ Workspace Setup", + "๐Ÿ“ Implementation Plan Generation", + "โš™๏ธ Code Implementation", + ] + else: + stages = [ + "๐Ÿ“„ Document Processing", + "๐Ÿ” Reference Analysis", + "๐Ÿ“‹ Plan Generation", + "๐Ÿ“ฆ Repository Download", + "๐Ÿ—‚๏ธ Codebase Indexing", + "โš™๏ธ Code Implementation", + ] + + for stage in stages: + print(f" โœ… {stage}") + + self.cli.print_separator() + + async def run_interactive_session(self): + """่ฟ่กŒไบคไบ’ๅผไผš่ฏ""" + # ๆธ…ๅฑๅนถๆ˜พ็คบๅฏๅŠจ็•Œ้ข + self.cli.clear_screen() + self.cli.print_logo() + self.cli.print_welcome_banner() + + # ๅˆๅง‹ๅŒ–MCPๅบ”็”จ + await self.initialize_mcp_app() + + try: + # ไธปไบคไบ’ๅพช็Žฏ + while self.cli.is_running: + self.cli.create_menu() + choice = self.cli.get_user_input() + + if choice in ["q", "quit", "exit"]: + self.cli.print_goodbye() + break + + elif choice in ["u", "url"]: + url = self.cli.get_url_input() + if url: + await self.process_input(url, "url") + + elif choice in ["f", "file"]: + file_path = self.cli.upload_file_gui() + if file_path: + await self.process_input(f"file://{file_path}", "file") + + elif choice in ["t", "chat", "text"]: + chat_input = self.cli.get_chat_input() + if chat_input: + await self.process_input(chat_input, "chat") + + elif choice in ["r", "req", "requirement", "requirements"]: + # NEW: Requirement Analysis workflow + await self.process_requirement_analysis() + + elif choice in ["h", "history"]: + self.cli.show_history() + + elif choice in ["c", "config", "configure"]: + # Show configuration menu - all settings managed by CLI interface + self.cli.show_configuration_menu() + + else: + self.cli.print_status( + "Invalid choice. Please select U, F, T, R, C, H, or Q.", + "warning", + ) + + # ่ฏข้—ฎๆ˜ฏๅฆ็ปง็ปญ + if self.cli.is_running and choice in [ + "u", + "f", + "t", + "r", + "chat", + "text", + "req", + "requirement", + "requirements", + ]: + if not self.cli.ask_continue(): + self.cli.is_running = False + self.cli.print_status("Session ended by user", "info") + + except KeyboardInterrupt: + print(f"\n{Colors.WARNING}โš ๏ธ Process interrupted by user{Colors.ENDC}") + except Exception as e: + print(f"\n{Colors.FAIL}โŒ Unexpected error: {str(e)}{Colors.ENDC}") + finally: + # ๆธ…็†่ต„ๆบ + await self.cleanup_mcp_app() + + +async def main(): + """ไธปๅ‡ฝๆ•ฐ""" + start_time = time.time() + + try: + # ๅˆ›ๅปบๅนถ่ฟ่กŒCLIๅบ”็”จ + app = CLIApp() + await app.run_interactive_session() + + except KeyboardInterrupt: + print(f"\n{Colors.WARNING}โš ๏ธ Application interrupted by user{Colors.ENDC}") + except Exception as e: + print(f"\n{Colors.FAIL}โŒ Application error: {str(e)}{Colors.ENDC}") + finally: + end_time = time.time() + print( + f"\n{Colors.BOLD}{Colors.CYAN}โฑ๏ธ Total runtime: {end_time - start_time:.2f} seconds{Colors.ENDC}" + ) + + # ๆธ…็†็ผ“ๅญ˜ๆ–‡ไปถ + print(f"{Colors.YELLOW}๐Ÿงน Cleaning up cache files...{Colors.ENDC}") + if os.name == "nt": # Windows + os.system( + "powershell -Command \"Get-ChildItem -Path . -Filter '__pycache__' -Recurse -Directory | Remove-Item -Recurse -Force\" 2>nul" + ) + else: # Unix/Linux/macOS + os.system('find . -type d -name "__pycache__" -exec rm -r {} + 2>/dev/null') + + print( + f"{Colors.OKGREEN}โœจ Goodbye! Thanks for using DeepCode CLI! โœจ{Colors.ENDC}" + ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/DeepCode/cli/cli_interface.py b/DeepCode/cli/cli_interface.py new file mode 100644 index 00000000..e3df1c48 --- /dev/null +++ b/DeepCode/cli/cli_interface.py @@ -0,0 +1,1053 @@ +#!/usr/bin/env python3 +""" +Enhanced CLI Interface Module for DeepCode +ๅขžๅผบ็‰ˆCLI็•Œ้ขๆจกๅ— - ไธ“ไธบDeepCode่ฎพ่ฎก +""" + +import os +import time +import platform +from typing import Optional + + +class Colors: + """ANSI color codes for terminal styling""" + + HEADER = "\033[95m" + OKBLUE = "\033[94m" + OKCYAN = "\033[96m" + OKGREEN = "\033[92m" + WARNING = "\033[93m" + FAIL = "\033[91m" + ENDC = "\033[0m" + BOLD = "\033[1m" + UNDERLINE = "\033[4m" + + # Gradient colors + PURPLE = "\033[35m" + MAGENTA = "\033[95m" + BLUE = "\033[34m" + CYAN = "\033[36m" + GREEN = "\033[32m" + YELLOW = "\033[33m" + + +class CLIInterface: + """Enhanced CLI interface with modern styling for DeepCode""" + + def __init__(self): + self.uploaded_file = None + self.is_running = True + self.processing_history = [] + self.enable_indexing = ( + False # Default configuration (matching UI: fast mode by default) + ) + + # Load segmentation config from the same source as UI + self._load_segmentation_config() + + # Initialize tkinter availability + self._init_tkinter() + + def _load_segmentation_config(self): + """Load segmentation configuration from mcp_agent.config.yaml""" + try: + from utils.llm_utils import get_document_segmentation_config + + seg_config = get_document_segmentation_config() + self.segmentation_enabled = seg_config.get("enabled", True) + self.segmentation_threshold = seg_config.get("size_threshold_chars", 50000) + except Exception as e: + print(f"โš ๏ธ Warning: Failed to load segmentation config: {e}") + # Fall back to defaults + self.segmentation_enabled = True + self.segmentation_threshold = 50000 + + def _save_segmentation_config(self): + """Save segmentation configuration to mcp_agent.config.yaml""" + import yaml + import os + + # Get the project root directory (where mcp_agent.config.yaml is located) + current_file = os.path.abspath(__file__) + cli_dir = os.path.dirname(current_file) # cli directory + project_root = os.path.dirname(cli_dir) # project root + config_path = os.path.join(project_root, "mcp_agent.config.yaml") + + try: + # Read current config + with open(config_path, "r", encoding="utf-8") as f: + config = yaml.safe_load(f) + + # Update document segmentation settings + if "document_segmentation" not in config: + config["document_segmentation"] = {} + + config["document_segmentation"]["enabled"] = self.segmentation_enabled + config["document_segmentation"]["size_threshold_chars"] = ( + self.segmentation_threshold + ) + + # Write updated config + with open(config_path, "w", encoding="utf-8") as f: + yaml.dump(config, f, default_flow_style=False, allow_unicode=True) + + print( + f"{Colors.OKGREEN}โœ… Document segmentation configuration updated{Colors.ENDC}" + ) + + except Exception as e: + print( + f"{Colors.WARNING}โš ๏ธ Failed to update segmentation config: {str(e)}{Colors.ENDC}" + ) + + def _init_tkinter(self): + """Initialize tkinter availability check""" + # Check tkinter availability for file dialogs + self.tkinter_available = True + try: + import tkinter as tk + + # Test if tkinter can create a window + test_root = tk.Tk() + test_root.withdraw() + test_root.destroy() + except Exception: + self.tkinter_available = False + + def clear_screen(self): + """Clear terminal screen""" + os.system("cls" if os.name == "nt" else "clear") + + def print_logo(self): + """Print enhanced ASCII logo for DeepCode CLI""" + logo = f""" +{Colors.CYAN}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•— +โ•‘ โ•‘ +โ•‘ {Colors.BOLD}{Colors.MAGENTA}โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—{Colors.CYAN} โ•‘ +โ•‘ {Colors.BOLD}{Colors.PURPLE}โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•”โ•โ•โ•โ•โ•โ–ˆโ–ˆโ•”โ•โ•โ•โ•โ•โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•”โ•โ•โ•โ•โ•โ–ˆโ–ˆโ•”โ•โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•”โ•โ•โ•โ•โ•{Colors.CYAN} โ•‘ +โ•‘ {Colors.BOLD}{Colors.BLUE}โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— {Colors.CYAN} โ•‘ +โ•‘ {Colors.BOLD}{Colors.OKBLUE}โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ• โ–ˆโ–ˆโ•”โ•โ•โ• โ–ˆโ–ˆโ•”โ•โ•โ•โ• โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ• {Colors.CYAN} โ•‘ +โ•‘ {Colors.BOLD}{Colors.OKCYAN}โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘ โ•šโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ•šโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—{Colors.CYAN} โ•‘ +โ•‘ {Colors.BOLD}{Colors.GREEN}โ•šโ•โ•โ•โ•โ•โ• โ•šโ•โ•โ•โ•โ•โ•โ•โ•šโ•โ•โ•โ•โ•โ•โ•โ•šโ•โ• โ•šโ•โ•โ•โ•โ•โ• โ•šโ•โ•โ•โ•โ•โ• โ•šโ•โ•โ•โ•โ•โ• โ•šโ•โ•โ•โ•โ•โ•โ•{Colors.CYAN} โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.BOLD}{Colors.GREEN}๐Ÿงฌ OPEN-SOURCE CODE AGENT โ€ข DATA INTELLIGENCE LAB @ HKU ๐Ÿš€ {Colors.CYAN}โ•‘ +โ•‘ {Colors.BOLD}{Colors.GREEN}โšก REVOLUTIONIZING RESEARCH REPRODUCIBILITY โšก {Colors.CYAN}โ•‘ +โ•‘ โ•‘ +โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{Colors.ENDC} +""" + print(logo) + + def print_welcome_banner(self): + """Print enhanced welcome banner""" + banner = f""" +{Colors.BOLD}{Colors.CYAN}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•— +โ•‘ WELCOME TO DEEPCODE CLI โ•‘ +โ• โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ฃ +โ•‘ {Colors.YELLOW}Open-Source Code Agent | Data Intelligence Lab @ HKU | MIT License {Colors.CYAN}โ•‘ +โ•‘ {Colors.GREEN}Status: Ready | Engine: Multi-Agent Architecture Initialized {Colors.CYAN}โ•‘ +โ•‘ {Colors.PURPLE}Mission: Revolutionizing Research Reproducibility {Colors.CYAN}โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.BOLD}{Colors.OKCYAN}๐Ÿ’Ž CORE CAPABILITIES:{Colors.ENDC} {Colors.CYAN}โ•‘ +โ•‘ {Colors.BOLD}{Colors.OKCYAN}โ–ถ Automated Paper-to-Code Reproduction {Colors.CYAN}โ•‘ +โ•‘ {Colors.BOLD}{Colors.OKCYAN}โ–ถ Collaborative Multi-Agent Architecture {Colors.CYAN}โ•‘ +โ•‘ {Colors.BOLD}{Colors.OKCYAN}โ–ถ Intelligent Code Implementation & Validation {Colors.CYAN}โ•‘ +โ•‘ {Colors.BOLD}{Colors.OKCYAN}โ–ถ Future Vision: One Sentence โ†’ Complete Codebase {Colors.CYAN}โ•‘ +โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{Colors.ENDC} +""" + print(banner) + + def print_separator(self, char="โ•", length=79, color=Colors.CYAN): + """Print a styled separator line""" + print(f"{color}{char * length}{Colors.ENDC}") + + def print_status(self, message: str, status_type: str = "info"): + """Print status message with appropriate styling""" + status_styles = { + "success": f"{Colors.OKGREEN}โœ…", + "error": f"{Colors.FAIL}โŒ", + "warning": f"{Colors.WARNING}โš ๏ธ ", + "info": f"{Colors.OKBLUE}โ„น๏ธ ", + "processing": f"{Colors.YELLOW}โณ", + "upload": f"{Colors.PURPLE}๐Ÿ“", + "download": f"{Colors.CYAN}๐Ÿ“ฅ", + "analysis": f"{Colors.MAGENTA}๐Ÿ”", + "implementation": f"{Colors.GREEN}โš™๏ธ ", + "complete": f"{Colors.OKGREEN}๐ŸŽ‰", + } + + icon = status_styles.get(status_type, status_styles["info"]) + timestamp = time.strftime("%H:%M:%S") + print( + f"[{Colors.BOLD}{timestamp}{Colors.ENDC}] {icon} {Colors.BOLD}{message}{Colors.ENDC}" + ) + + def create_menu(self): + """Create enhanced interactive menu""" + # Display current configuration + pipeline_mode = "๐Ÿง  COMPREHENSIVE" if self.enable_indexing else "โšก OPTIMIZED" + index_status = "โœ… Enabled" if self.enable_indexing else "๐Ÿ”ถ Disabled" + segmentation_mode = ( + "๐Ÿ“„ SMART" if self.segmentation_enabled else "๐Ÿ“‹ TRADITIONAL" + ) + + menu = f""" +{Colors.BOLD}{Colors.CYAN}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•— +โ•‘ MAIN MENU โ•‘ +โ• โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ฃ +โ•‘ {Colors.OKGREEN}๐ŸŒ [U] Process URL {Colors.CYAN}โ”‚ {Colors.PURPLE}๐Ÿ“ [F] Upload File {Colors.CYAN}โ”‚ {Colors.MAGENTA}๐Ÿ’ฌ [T] Chat Input{Colors.CYAN} โ•‘ +โ•‘ {Colors.BLUE}๐Ÿง  [R] Req. Analysis {Colors.CYAN}โ”‚ {Colors.OKCYAN}โš™๏ธ [C] Configure {Colors.CYAN}โ”‚ {Colors.YELLOW}๐Ÿ“Š [H] History{Colors.CYAN} โ•‘ +โ•‘ {Colors.FAIL}โŒ [Q] Quit{Colors.CYAN} โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.BOLD}๐Ÿค– Current Pipeline Mode: {pipeline_mode}{Colors.CYAN} โ•‘ +โ•‘ {Colors.BOLD}๐Ÿ—‚๏ธ Codebase Indexing: {index_status}{Colors.CYAN} โ•‘ +โ•‘ {Colors.BOLD}๐Ÿ“„ Document Processing: {segmentation_mode}{Colors.CYAN} โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.YELLOW}๐Ÿ“ URL Processing:{Colors.CYAN} โ•‘ +โ•‘ {Colors.YELLOW} โ–ถ Enter research paper URL (arXiv, IEEE, ACM, etc.) {Colors.CYAN}โ•‘ +โ•‘ {Colors.YELLOW} โ–ถ Supports direct PDF links and academic paper pages {Colors.CYAN}โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.PURPLE}๐Ÿ“ File Processing:{Colors.CYAN} โ•‘ +โ•‘ {Colors.PURPLE} โ–ถ Upload PDF, DOCX, PPTX, HTML, or TXT files {Colors.CYAN}โ•‘ +โ•‘ {Colors.PURPLE} โ–ถ Intelligent file format detection and processing {Colors.CYAN}โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.MAGENTA}๐Ÿ’ฌ Chat Input:{Colors.CYAN} โ•‘ +โ•‘ {Colors.MAGENTA} โ–ถ Describe your coding requirements in natural language {Colors.CYAN}โ•‘ +โ•‘ {Colors.MAGENTA} โ–ถ AI generates implementation plan and code automatically {Colors.CYAN}โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.BLUE}๐Ÿง  Requirement Analysis (NEW):{Colors.CYAN} โ•‘ +โ•‘ {Colors.BLUE} โ–ถ Get AI-guided questions to refine your requirements {Colors.CYAN}โ•‘ +โ•‘ {Colors.BLUE} โ–ถ Generate detailed requirement documents from your answers {Colors.CYAN}โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.OKCYAN}๐Ÿ”„ Processing Pipeline:{Colors.CYAN} โ•‘ +โ•‘ {Colors.OKCYAN} โ–ถ Intelligent agent orchestration โ†’ Code synthesis {Colors.CYAN}โ•‘ +โ•‘ {Colors.OKCYAN} โ–ถ Multi-agent coordination with progress tracking {Colors.CYAN}โ•‘ +โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{Colors.ENDC} +""" + print(menu) + + def get_user_input(self): + """Get user input with styled prompt""" + print(f"\n{Colors.BOLD}{Colors.OKCYAN}โžค Your choice: {Colors.ENDC}", end="") + return input().strip().lower() + + def upload_file_gui(self) -> Optional[str]: + """Enhanced file upload interface with better error handling""" + if not self.tkinter_available: + self.print_status( + "GUI file dialog not available - using manual input", "warning" + ) + return self._get_manual_file_path() + + def select_file(): + try: + import tkinter as tk + from tkinter import filedialog + + root = tk.Tk() + root.withdraw() + root.attributes("-topmost", True) + + file_types = [ + ("Research Papers", "*.pdf;*.docx;*.doc"), + ("PDF Files", "*.pdf"), + ("Word Documents", "*.docx;*.doc"), + ("PowerPoint Files", "*.pptx;*.ppt"), + ("HTML Files", "*.html;*.htm"), + ("Text Files", "*.txt;*.md"), + ("All Files", "*.*"), + ] + + if platform.system() == "Darwin": + file_types = [ + ("Research Papers", ".pdf .docx .doc"), + ("PDF Files", ".pdf"), + ("Word Documents", ".docx .doc"), + ("PowerPoint Files", ".pptx .ppt"), + ("HTML Files", ".html .htm"), + ("Text Files", ".txt .md"), + ("All Files", ".*"), + ] + + file_path = filedialog.askopenfilename( + title="Select Research File - DeepCode CLI", + filetypes=file_types, + initialdir=os.getcwd(), + ) + + root.destroy() + return file_path + + except Exception as e: + self.print_status(f"File dialog error: {str(e)}", "error") + return self._get_manual_file_path() + + self.print_status("Opening file browser dialog...", "upload") + file_path = select_file() + + if file_path: + self.print_status( + f"File selected: {os.path.basename(file_path)}", "success" + ) + return file_path + else: + self.print_status("No file selected", "warning") + return None + + def _get_manual_file_path(self) -> Optional[str]: + """Get file path through manual input with validation""" + self.print_separator("โ”€", 79, Colors.YELLOW) + print(f"{Colors.BOLD}{Colors.YELLOW}๐Ÿ“ Manual File Path Input{Colors.ENDC}") + print( + f"{Colors.CYAN}Please enter the full path to your research paper file:{Colors.ENDC}" + ) + print( + f"{Colors.CYAN}Supported formats: PDF, DOCX, PPTX, HTML, TXT, MD{Colors.ENDC}" + ) + self.print_separator("โ”€", 79, Colors.YELLOW) + + while True: + print(f"\n{Colors.BOLD}{Colors.OKCYAN}๐Ÿ“‚ File path: {Colors.ENDC}", end="") + file_path = input().strip() + + if not file_path: + self.print_status( + "Empty path entered. Please try again or press Ctrl+C to cancel.", + "warning", + ) + continue + + file_path = os.path.expanduser(file_path) + file_path = os.path.abspath(file_path) + + if not os.path.exists(file_path): + self.print_status(f"File not found: {file_path}", "error") + retry = ( + input(f"{Colors.YELLOW}Try again? (y/n): {Colors.ENDC}") + .strip() + .lower() + ) + if retry != "y": + return None + continue + + if not os.path.isfile(file_path): + self.print_status(f"Path is not a file: {file_path}", "error") + continue + + supported_extensions = { + ".pdf", + ".docx", + ".doc", + ".pptx", + ".ppt", + ".html", + ".htm", + ".txt", + ".md", + } + file_ext = os.path.splitext(file_path)[1].lower() + + if file_ext not in supported_extensions: + self.print_status(f"Unsupported file format: {file_ext}", "warning") + proceed = ( + input(f"{Colors.YELLOW}Process anyway? (y/n): {Colors.ENDC}") + .strip() + .lower() + ) + if proceed != "y": + continue + + self.print_status( + f"File validated: {os.path.basename(file_path)}", "success" + ) + return file_path + + def get_url_input(self) -> str: + """Enhanced URL input with validation""" + self.print_separator("โ”€", 79, Colors.GREEN) + print(f"{Colors.BOLD}{Colors.GREEN}๐ŸŒ URL Input Interface{Colors.ENDC}") + print( + f"{Colors.CYAN}Enter a research paper URL from supported platforms:{Colors.ENDC}" + ) + print( + f"{Colors.CYAN}โ€ข arXiv (arxiv.org) โ€ข IEEE Xplore (ieeexplore.ieee.org){Colors.ENDC}" + ) + print( + f"{Colors.CYAN}โ€ข ACM Digital Library โ€ข SpringerLink โ€ข Nature โ€ข Science{Colors.ENDC}" + ) + print( + f"{Colors.CYAN}โ€ข Direct PDF links โ€ข Academic publisher websites{Colors.ENDC}" + ) + self.print_separator("โ”€", 79, Colors.GREEN) + + while True: + print(f"\n{Colors.BOLD}{Colors.OKCYAN}๐Ÿ”— URL: {Colors.ENDC}", end="") + url = input().strip() + + if not url: + self.print_status( + "Empty URL entered. Please try again or press Ctrl+C to cancel.", + "warning", + ) + continue + + if not url.startswith(("http://", "https://")): + self.print_status("URL must start with http:// or https://", "error") + retry = ( + input(f"{Colors.YELLOW}Try again? (y/n): {Colors.ENDC}") + .strip() + .lower() + ) + if retry != "y": + return "" + continue + + academic_domains = [ + "arxiv.org", + "ieeexplore.ieee.org", + "dl.acm.org", + "link.springer.com", + "nature.com", + "science.org", + "scholar.google.com", + "researchgate.net", + "semanticscholar.org", + ] + + is_academic = any(domain in url.lower() for domain in academic_domains) + if not is_academic and not url.lower().endswith(".pdf"): + self.print_status( + "URL doesn't appear to be from a known academic platform", "warning" + ) + proceed = ( + input(f"{Colors.YELLOW}Process anyway? (y/n): {Colors.ENDC}") + .strip() + .lower() + ) + if proceed != "y": + continue + + self.print_status(f"URL validated: {url}", "success") + return url + + def get_chat_input(self) -> str: + """Enhanced chat input interface for coding requirements""" + self.print_separator("โ”€", 79, Colors.PURPLE) + print(f"{Colors.BOLD}{Colors.PURPLE}๐Ÿ’ฌ Chat Input Interface{Colors.ENDC}") + print( + f"{Colors.CYAN}Describe your coding requirements in natural language.{Colors.ENDC}" + ) + print( + f"{Colors.CYAN}Our AI will analyze your needs and generate a comprehensive implementation plan.{Colors.ENDC}" + ) + self.print_separator("โ”€", 79, Colors.PURPLE) + + # Display examples to help users + print(f"\n{Colors.BOLD}{Colors.YELLOW}๐Ÿ’ก Examples:{Colors.ENDC}") + print(f"{Colors.CYAN}Academic Research:{Colors.ENDC}") + print( + " โ€ข 'I need to implement a reinforcement learning algorithm for robotic control'" + ) + print( + " โ€ข 'Create a neural network for image classification with attention mechanisms'" + ) + print(f"{Colors.CYAN}Engineering Projects:{Colors.ENDC}") + print( + " โ€ข 'Develop a web application for project management with user authentication'" + ) + print(" โ€ข 'Create a data visualization dashboard for sales analytics'") + print(f"{Colors.CYAN}Mixed Projects:{Colors.ENDC}") + print( + " โ€ข 'Implement a machine learning model with a web interface for real-time predictions'" + ) + + self.print_separator("โ”€", 79, Colors.PURPLE) + + print( + f"\n{Colors.BOLD}{Colors.OKCYAN}โœ๏ธ Enter your coding requirements below:{Colors.ENDC}" + ) + print( + f"{Colors.YELLOW}(Type your description, press Enter twice when finished, or Ctrl+C to cancel){Colors.ENDC}" + ) + + lines = [] + empty_line_count = 0 + + while True: + try: + if len(lines) == 0: + print(f"{Colors.BOLD}> {Colors.ENDC}", end="") + else: + print(f"{Colors.BOLD} {Colors.ENDC}", end="") + + line = input() + + if line.strip() == "": + empty_line_count += 1 + if empty_line_count >= 2: + # Two consecutive empty lines means user finished input + break + lines.append("") # Keep empty line for formatting + else: + empty_line_count = 0 + lines.append(line) + + except KeyboardInterrupt: + print(f"\n{Colors.WARNING}Input cancelled by user{Colors.ENDC}") + return "" + + # Join all lines and clean up + user_input = "\n".join(lines).strip() + + if not user_input: + self.print_status("No input provided", "warning") + return "" + + if len(user_input) < 20: + self.print_status( + "Input too short. Please provide more detailed requirements (at least 20 characters)", + "warning", + ) + retry = ( + input(f"{Colors.YELLOW}Try again? (y/n): {Colors.ENDC}").strip().lower() + ) + if retry == "y": + return self.get_chat_input() # Recursive call for retry + return "" + + # Display input summary + word_count = len(user_input.split()) + char_count = len(user_input) + + print(f"\n{Colors.BOLD}{Colors.GREEN}๐Ÿ“‹ Input Summary:{Colors.ENDC}") + print(f" โ€ข {Colors.CYAN}Word count: {word_count}{Colors.ENDC}") + print(f" โ€ข {Colors.CYAN}Character count: {char_count}{Colors.ENDC}") + + # Show preview + preview = user_input[:200] + "..." if len(user_input) > 200 else user_input + print(f"\n{Colors.BOLD}{Colors.CYAN}๐Ÿ“„ Preview:{Colors.ENDC}") + print(f"{Colors.YELLOW}{preview}{Colors.ENDC}") + + # Confirm with user + confirm = ( + input( + f"\n{Colors.BOLD}{Colors.OKCYAN}Proceed with this input? (y/n): {Colors.ENDC}" + ) + .strip() + .lower() + ) + if confirm != "y": + retry = ( + input(f"{Colors.YELLOW}Edit input? (y/n): {Colors.ENDC}") + .strip() + .lower() + ) + if retry == "y": + return self.get_chat_input() # Recursive call for retry + return "" + + self.print_status( + f"Chat input captured: {word_count} words, {char_count} characters", + "success", + ) + return user_input + + def show_progress_bar(self, message: str, duration: float = 2.0): + """Show animated progress bar""" + print(f"\n{Colors.BOLD}{Colors.CYAN}{message}{Colors.ENDC}") + + bar_length = 50 + for i in range(bar_length + 1): + percent = (i / bar_length) * 100 + filled = "โ–ˆ" * i + empty = "โ–‘" * (bar_length - i) + + print( + f"\r{Colors.OKGREEN}[{filled}{empty}] {percent:3.0f}%{Colors.ENDC}", + end="", + flush=True, + ) + time.sleep(duration / bar_length) + + print(f"\n{Colors.OKGREEN}โœ“ {message} completed{Colors.ENDC}") + + def show_spinner(self, message: str, duration: float = 1.0): + """Show spinner animation""" + spinner_chars = "โ ‹โ ™โ นโ ธโ ผโ ดโ ฆโ งโ ‡โ " + end_time = time.time() + duration + + print( + f"{Colors.BOLD}{Colors.CYAN}{message}... {Colors.ENDC}", end="", flush=True + ) + + i = 0 + while time.time() < end_time: + print( + f"\r{Colors.BOLD}{Colors.CYAN}{message}... {Colors.YELLOW}{spinner_chars[i % len(spinner_chars)]}{Colors.ENDC}", + end="", + flush=True, + ) + time.sleep(0.1) + i += 1 + + print( + f"\r{Colors.BOLD}{Colors.CYAN}{message}... {Colors.OKGREEN}โœ“{Colors.ENDC}" + ) + + def display_processing_stages( + self, + current_stage: int = 0, + enable_indexing: bool = True, + chat_mode: bool = False, + ): + """Display processing pipeline stages with current progress""" + if chat_mode: + # Chat mode - simplified workflow for user requirements + stages = [ + ("๐Ÿš€", "Initialize", "Setting up chat engine"), + ("๐Ÿ’ฌ", "Planning", "Analyzing requirements"), + ("๐Ÿ—๏ธ", "Setup", "Creating workspace"), + ("๐Ÿ“", "Save Plan", "Saving implementation plan"), + ("โš™๏ธ", "Implement", "Generating code"), + ] + pipeline_mode = "CHAT PLANNING" + elif enable_indexing: + # Full pipeline with all stages + stages = [ + ("๐Ÿš€", "Initialize", "Setting up AI engine"), + ("๐Ÿ“Š", "Analyze", "Analyzing research content"), + ("๐Ÿ“ฅ", "Download", "Processing document"), + ("๐Ÿ“‹", "Plan", "Generating code architecture"), + ("๐Ÿ”", "References", "Analyzing references"), + ("๐Ÿ“ฆ", "Repos", "Downloading repositories"), + ("๐Ÿ—‚๏ธ", "Index", "Building code index"), + ("โš™๏ธ", "Implement", "Implementing code"), + ] + pipeline_mode = "COMPREHENSIVE" + else: + # Fast mode - skip indexing related stages + stages = [ + ("๐Ÿš€", "Initialize", "Setting up AI engine"), + ("๐Ÿ“Š", "Analyze", "Analyzing research content"), + ("๐Ÿ“ฅ", "Download", "Processing document"), + ("๐Ÿ“‹", "Plan", "Generating code architecture"), + ("โš™๏ธ", "Implement", "Implementing code"), + ] + pipeline_mode = "OPTIMIZED" + + print( + f"\n{Colors.BOLD}{Colors.CYAN}๐Ÿ“‹ {pipeline_mode} PIPELINE STATUS{Colors.ENDC}" + ) + self.print_separator("โ”€", 79, Colors.CYAN) + + for i, (icon, name, desc) in enumerate(stages): + if i < current_stage: + status = f"{Colors.OKGREEN}โœ“ COMPLETED{Colors.ENDC}" + elif i == current_stage: + status = f"{Colors.YELLOW}โณ IN PROGRESS{Colors.ENDC}" + else: + status = f"{Colors.CYAN}โธ๏ธ PENDING{Colors.ENDC}" + + print( + f"{icon} {Colors.BOLD}{name:<12}{Colors.ENDC} โ”‚ {desc:<25} โ”‚ {status}" + ) + + self.print_separator("โ”€", 79, Colors.CYAN) + + def print_results_header(self): + """Print results section header""" + header = f""" +{Colors.BOLD}{Colors.OKGREEN}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•— +โ•‘ PROCESSING RESULTS โ•‘ +โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{Colors.ENDC} +""" + print(header) + + def print_error_box(self, title: str, error_msg: str): + """Print formatted error box""" + print( + f"\n{Colors.FAIL}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—" + ) + print(f"โ•‘ {Colors.BOLD}ERROR: {title:<50}{Colors.FAIL} โ•‘") + print("โ• โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ฃ") + + words = error_msg.split() + lines = [] + current_line = "" + + for word in words: + if len(current_line + word) <= 54: + current_line += word + " " + else: + lines.append(current_line.strip()) + current_line = word + " " + if current_line: + lines.append(current_line.strip()) + + for line in lines: + print(f"โ•‘ {line:<56} โ•‘") + + print( + f"โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{Colors.ENDC}" + ) + + def cleanup_cache(self): + """ๆธ…็†Python็ผ“ๅญ˜ๆ–‡ไปถ / Clean up Python cache files""" + try: + self.print_status("Cleaning up cache files...", "info") + # ๆธ…็†__pycache__็›ฎๅฝ• + os.system('find . -type d -name "__pycache__" -exec rm -r {} + 2>/dev/null') + # ๆธ…็†.pycๆ–‡ไปถ + os.system('find . -name "*.pyc" -delete 2>/dev/null') + self.print_status("Cache cleanup completed", "success") + except Exception as e: + self.print_status(f"Cache cleanup failed: {e}", "warning") + + def print_goodbye(self): + """Print goodbye message""" + # ๆธ…็†็ผ“ๅญ˜ๆ–‡ไปถ + self.cleanup_cache() + + goodbye = f""" +{Colors.BOLD}{Colors.CYAN}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•— +โ•‘ GOODBYE โ•‘ +โ• โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ฃ +โ•‘ {Colors.OKGREEN}๐ŸŽ‰ Thank you for using DeepCode CLI! {Colors.CYAN}โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.YELLOW}๐Ÿงฌ Join our community in revolutionizing research reproducibility {Colors.CYAN}โ•‘ +โ•‘ {Colors.PURPLE}โšก Together, we're building the future of automated code generation {Colors.CYAN}โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.OKCYAN}๐Ÿ’ก Questions? Contribute to our open-source mission at GitHub {Colors.CYAN}โ•‘ +โ•‘ {Colors.GREEN}๐Ÿงน Cache files cleaned up for optimal performance {Colors.CYAN}โ•‘ +โ•‘ โ•‘ +โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{Colors.ENDC} +""" + print(goodbye) + + def get_requirement_analysis_input(self) -> str: + """Enhanced requirement analysis input interface (NEW: matching UI version)""" + self.print_separator("โ”€", 79, Colors.BLUE) + print( + f"{Colors.BOLD}{Colors.BLUE}๐Ÿง  Requirement Analysis Interface{Colors.ENDC}" + ) + print( + f"{Colors.CYAN}Describe your project idea or requirements briefly.{Colors.ENDC}" + ) + print( + f"{Colors.CYAN}Our AI will generate guiding questions to help you refine your vision.{Colors.ENDC}" + ) + self.print_separator("โ”€", 79, Colors.BLUE) + + # Display examples + print(f"\n{Colors.BOLD}{Colors.YELLOW}๐Ÿ’ก Examples:{Colors.ENDC}") + print( + f"{Colors.CYAN} โ€ข 'I want to build a machine learning system for image recognition'{Colors.ENDC}" + ) + print( + f"{Colors.CYAN} โ€ข 'Create a web app for project management with real-time collaboration'{Colors.ENDC}" + ) + print( + f"{Colors.CYAN} โ€ข 'Develop a data analysis pipeline for financial forecasting'{Colors.ENDC}" + ) + + self.print_separator("โ”€", 79, Colors.BLUE) + + print( + f"\n{Colors.BOLD}{Colors.OKCYAN}โœ๏ธ Enter your initial requirements below:{Colors.ENDC}" + ) + print( + f"{Colors.YELLOW}(Type your description, press Enter twice when finished, or Ctrl+C to cancel){Colors.ENDC}" + ) + + lines = [] + empty_line_count = 0 + + while True: + try: + if len(lines) == 0: + print(f"{Colors.BOLD}> {Colors.ENDC}", end="") + else: + print(f"{Colors.BOLD} {Colors.ENDC}", end="") + + line = input() + + if line.strip() == "": + empty_line_count += 1 + if empty_line_count >= 2: + break + lines.append("") + else: + empty_line_count = 0 + lines.append(line) + + except KeyboardInterrupt: + print(f"\n{Colors.WARNING}Input cancelled by user{Colors.ENDC}") + return "" + + user_input = "\n".join(lines).strip() + + if not user_input: + self.print_status("No input provided", "warning") + return "" + + if len(user_input) < 20: + self.print_status( + "Input too short. Please provide more details (at least 20 characters)", + "warning", + ) + retry = ( + input(f"{Colors.YELLOW}Try again? (y/n): {Colors.ENDC}").strip().lower() + ) + if retry == "y": + return self.get_requirement_analysis_input() + return "" + + # Display input summary + word_count = len(user_input.split()) + char_count = len(user_input) + + print(f"\n{Colors.BOLD}{Colors.GREEN}๐Ÿ“‹ Input Summary:{Colors.ENDC}") + print(f" โ€ข {Colors.CYAN}Word count: {word_count}{Colors.ENDC}") + print(f" โ€ข {Colors.CYAN}Character count: {char_count}{Colors.ENDC}") + + # Show preview + preview = user_input[:200] + "..." if len(user_input) > 200 else user_input + print(f"\n{Colors.BOLD}{Colors.CYAN}๐Ÿ“„ Preview:{Colors.ENDC}") + print(f"{Colors.YELLOW}{preview}{Colors.ENDC}") + + # Confirm + confirm = ( + input( + f"\n{Colors.BOLD}{Colors.OKCYAN}Proceed with this input? (y/n): {Colors.ENDC}" + ) + .strip() + .lower() + ) + if confirm != "y": + retry = ( + input(f"{Colors.YELLOW}Edit input? (y/n): {Colors.ENDC}") + .strip() + .lower() + ) + if retry == "y": + return self.get_requirement_analysis_input() + return "" + + self.print_status( + f"Requirement input captured: {word_count} words, {char_count} characters", + "success", + ) + return user_input + + def display_guiding_questions(self, questions_json: str): + """Display AI-generated guiding questions (NEW: matching UI version)""" + import json + + try: + questions = json.loads(questions_json) + + self.print_separator("โ•", 79, Colors.GREEN) + print( + f"\n{Colors.BOLD}{Colors.GREEN}๐Ÿค– AI-Generated Guiding Questions{Colors.ENDC}" + ) + print( + f"{Colors.CYAN}Please answer these questions to help refine your requirements:{Colors.ENDC}\n" + ) + self.print_separator("โ”€", 79, Colors.GREEN) + + for i, q in enumerate(questions, 1): + print( + f"\n{Colors.BOLD}{Colors.YELLOW}Question {i}:{Colors.ENDC} {Colors.CYAN}{q}{Colors.ENDC}" + ) + + self.print_separator("โ•", 79, Colors.GREEN) + + except json.JSONDecodeError: + self.print_status("Failed to parse questions", "error") + print(questions_json) + + def get_question_answers(self, questions_json: str) -> dict: + """Get user answers to guiding questions (NEW: matching UI version)""" + import json + + try: + questions = json.loads(questions_json) + answers = {} + + print( + f"\n{Colors.BOLD}{Colors.BLUE}๐Ÿ“ Answer the following questions:{Colors.ENDC}" + ) + print( + f"{Colors.CYAN}(Type your answer and press Enter for each question){Colors.ENDC}\n" + ) + + for i, question in enumerate(questions, 1): + print( + f"\n{Colors.BOLD}{Colors.YELLOW}Q{i}:{Colors.ENDC} {Colors.CYAN}{question}{Colors.ENDC}" + ) + print(f"{Colors.BOLD}{Colors.OKCYAN}Your answer:{Colors.ENDC} ", end="") + + answer = input().strip() + answers[f"question_{i}"] = answer + + if answer: + self.print_status(f"Answer {i} recorded", "success") + else: + self.print_status(f"Answer {i} left blank", "warning") + + return answers + + except json.JSONDecodeError: + self.print_status("Failed to parse questions", "error") + return {} + + def display_requirement_summary(self, summary: str): + """Display generated requirement document (NEW: matching UI version)""" + self.print_separator("โ•", 79, Colors.GREEN) + print( + f"\n{Colors.BOLD}{Colors.GREEN}๐Ÿ“„ Generated Requirement Document{Colors.ENDC}\n" + ) + self.print_separator("โ”€", 79, Colors.GREEN) + + print(f"{Colors.CYAN}{summary}{Colors.ENDC}") + + self.print_separator("โ•", 79, Colors.GREEN) + + # Ask if user wants to proceed with implementation + proceed = ( + input( + f"\n{Colors.BOLD}{Colors.YELLOW}Would you like to proceed with code implementation based on these requirements? (y/n):{Colors.ENDC} " + ) + .strip() + .lower() + ) + + return proceed == "y" + + def ask_continue(self) -> bool: + """Ask if user wants to continue with another paper""" + self.print_separator("โ”€", 79, Colors.YELLOW) + print(f"\n{Colors.BOLD}{Colors.YELLOW}๐Ÿ”„ Process another paper?{Colors.ENDC}") + choice = input(f"{Colors.OKCYAN}Continue? (y/n): {Colors.ENDC}").strip().lower() + return choice in ["y", "yes", "1", "true"] + + def add_to_history(self, input_source: str, result: dict): + """Add processing result to history""" + entry = { + "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), + "input_source": input_source, + "status": result.get("status", "unknown"), + "result": result, + } + self.processing_history.append(entry) + + def show_history(self): + """Display processing history""" + if not self.processing_history: + self.print_status("No processing history available", "info") + return + + print(f"\n{Colors.BOLD}{Colors.CYAN}๐Ÿ“š PROCESSING HISTORY{Colors.ENDC}") + self.print_separator("โ”€", 79, Colors.CYAN) + + for i, entry in enumerate(self.processing_history, 1): + status_icon = "โœ…" if entry["status"] == "success" else "โŒ" + source = entry["input_source"] + if len(source) > 50: + source = source[:47] + "..." + + print(f"{i}. {status_icon} {entry['timestamp']} | {source}") + + self.print_separator("โ”€", 79, Colors.CYAN) + + def show_configuration_menu(self): + """Show configuration options menu""" + self.clear_screen() + + # Get segmentation config status + segmentation_enabled = getattr(self, "segmentation_enabled", True) + segmentation_threshold = getattr(self, "segmentation_threshold", 50000) + + print(f""" +{Colors.BOLD}{Colors.CYAN}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•— +โ•‘ CONFIGURATION MENU โ•‘ +โ• โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ฃ +โ•‘ โ•‘ +โ•‘ {Colors.BOLD}๐Ÿค– Agent Orchestration Engine Configuration{Colors.CYAN} โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.OKCYAN}[1] Pipeline Mode:{Colors.CYAN} โ•‘ +โ•‘ {Colors.BOLD}๐Ÿง  Comprehensive Mode{Colors.CYAN} - Full intelligence analysis (Default) โ•‘ +โ•‘ โœ“ Research Analysis + Resource Processing โ•‘ +โ•‘ โœ“ Reference Intelligence Discovery โ•‘ +โ•‘ โœ“ Automated Repository Acquisition โ•‘ +โ•‘ โœ“ Codebase Intelligence Orchestration โ•‘ +โ•‘ โœ“ Intelligent Code Implementation Synthesis โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.BOLD}โšก Optimized Mode{Colors.CYAN} - Fast processing (Skip indexing) โ•‘ +โ•‘ โœ“ Research Analysis + Resource Processing โ•‘ +โ•‘ โœ“ Code Architecture Synthesis โ•‘ +โ•‘ โœ“ Intelligent Code Implementation Synthesis โ•‘ +โ•‘ โœ— Reference Intelligence Discovery (Skipped) โ•‘ +โ•‘ โœ— Repository Acquisition (Skipped) โ•‘ +โ•‘ โœ— Codebase Intelligence Orchestration (Skipped) โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.OKCYAN}[2] Document Processing:{Colors.CYAN} โ•‘ +โ•‘ {Colors.BOLD}๐Ÿ“„ Smart Segmentation{Colors.CYAN} - Intelligent document analysis (Default) โ•‘ +โ•‘ โœ“ Semantic boundary detection โ•‘ +โ•‘ โœ“ Algorithm integrity preservation โ•‘ +โ•‘ โœ“ Formula chain recognition โ•‘ +โ•‘ โœ“ Adaptive character limits โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.BOLD}๐Ÿ“‹ Traditional Processing{Colors.CYAN} - Full document reading โ•‘ +โ•‘ โœ“ Complete document analysis โ•‘ +โ•‘ โœ— Smart segmentation (Disabled) โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.YELLOW}Current Settings:{Colors.CYAN} โ•‘ +โ•‘ Pipeline: {'๐Ÿง  Comprehensive Mode' if self.enable_indexing else 'โšก Optimized Mode'} โ•‘ +โ•‘ Document: {'๐Ÿ“„ Smart Segmentation' if segmentation_enabled else '๐Ÿ“‹ Traditional Processing'} โ•‘ +โ•‘ Threshold: {segmentation_threshold} characters โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.OKGREEN}[T] Toggle Pipeline {Colors.BLUE}[S] Toggle Segmentation {Colors.FAIL}[B] Back{Colors.CYAN} โ•‘ +โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{Colors.ENDC} +""") + + while True: + print( + f"\n{Colors.BOLD}{Colors.OKCYAN}โžค Configuration choice: {Colors.ENDC}", + end="", + ) + choice = input().strip().lower() + + if choice in ["t", "toggle"]: + self.enable_indexing = not self.enable_indexing + mode = "๐Ÿง  Comprehensive" if self.enable_indexing else "โšก Optimized" + self.print_status(f"Pipeline mode switched to: {mode}", "success") + time.sleep(1) + self.show_configuration_menu() + return + + elif choice in ["s", "segmentation"]: + current_state = getattr(self, "segmentation_enabled", True) + self.segmentation_enabled = not current_state + # Save the configuration to file + self._save_segmentation_config() + seg_mode = ( + "๐Ÿ“„ Smart Segmentation" + if self.segmentation_enabled + else "๐Ÿ“‹ Traditional Processing" + ) + self.print_status( + f"Document processing switched to: {seg_mode}", "success" + ) + time.sleep(1) + self.show_configuration_menu() + return + + elif choice in ["b", "back"]: + return + + else: + self.print_status( + "Invalid choice. Please enter 'T', 'S', or 'B'.", "warning" + ) diff --git a/DeepCode/cli/cli_launcher.py b/DeepCode/cli/cli_launcher.py new file mode 100644 index 00000000..255d20f9 --- /dev/null +++ b/DeepCode/cli/cli_launcher.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +""" +DeepCode - CLI Research Engine Launcher +DeepCode - CLI็ ”็ฉถๅผ•ๆ“ŽๅฏๅŠจๅ™จ + +๐Ÿงฌ Open-Source Code Agent by Data Intelligence Lab @ HKU (CLI Edition) +โšก Revolutionizing research reproducibility through collaborative AI via command line +""" + +import sys +from pathlib import Path + + +def check_dependencies(): + """ๆฃ€ๆŸฅๅฟ…่ฆ็š„ไพ่ต–ๆ˜ฏๅฆๅทฒๅฎ‰่ฃ… / Check if necessary dependencies are installed""" + import importlib.util + + print("๐Ÿ” Checking CLI dependencies...") + + missing_deps = [] + + # Check asyncio availability + if importlib.util.find_spec("asyncio") is not None: + print("โœ… Asyncio is available") + else: + missing_deps.append("asyncio") + + # Check PyYAML availability + if importlib.util.find_spec("yaml") is not None: + print("โœ… PyYAML is installed") + else: + missing_deps.append("pyyaml") + + # Check Tkinter availability + if importlib.util.find_spec("tkinter") is not None: + print("โœ… Tkinter is available (for file dialogs)") + else: + print("โš ๏ธ Tkinter not available - file dialogs will use manual input") + + # Check for MCP agent dependencies + if importlib.util.find_spec("mcp_agent.app") is not None: + print("โœ… MCP Agent framework is available") + else: + missing_deps.append("mcp-agent") + + # Check for workflow dependencies + # ๆทปๅŠ ้กน็›ฎๆ น็›ฎๅฝ•ๅˆฐ่ทฏๅพ„ + current_dir = Path(__file__).parent + project_root = current_dir.parent + if str(project_root) not in sys.path: + sys.path.insert(0, str(project_root)) + + if importlib.util.find_spec("workflows.agent_orchestration_engine") is not None: + print("โœ… Workflow modules are available") + else: + print("โš ๏ธ Workflow modules may not be properly configured") + + # Check for CLI components + if importlib.util.find_spec("cli.cli_app") is not None: + print("โœ… CLI application components are available") + else: + print("โŒ CLI application components missing") + missing_deps.append("cli-components") + + if missing_deps: + print("\nโŒ Missing dependencies:") + for dep in missing_deps: + print(f" - {dep}") + print("\nPlease install missing dependencies using:") + print( + f"pip install {' '.join([d for d in missing_deps if d != 'cli-components'])}" + ) + if "cli-components" in missing_deps: + print( + "CLI components appear to be missing - please check the cli/ directory" + ) + return False + + print("โœ… All CLI dependencies satisfied") + return True + + +def print_banner(): + """ๆ˜พ็คบCLIๅฏๅŠจๆจชๅน… / Display CLI startup banner""" + banner = """ +โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•— +โ•‘ โ•‘ +โ•‘ ๐Ÿงฌ DeepCode - Open-Source Code Agent โ•‘ +โ•‘ โ•‘ +โ•‘ โšก DATA INTELLIGENCE LAB @ HKU โšก โ•‘ +โ•‘ โ•‘ +โ•‘ โ•‘ +โ•‘ โ•‘ +โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +""" + print(banner) + + +def main(): + """ไธปๅ‡ฝๆ•ฐ / Main function""" + print_banner() + + # ๆฃ€ๆŸฅไพ่ต– / Check dependencies + if not check_dependencies(): + print("\n๐Ÿšจ Please install missing dependencies and try again.") + sys.exit(1) + + # ่Žทๅ–ๅฝ“ๅ‰่„šๆœฌ็›ฎๅฝ• / Get current script directory + current_dir = Path(__file__).parent + project_root = current_dir.parent + cli_app_path = current_dir / "cli_app.py" + + # ๆฃ€ๆŸฅcli_app.pyๆ˜ฏๅฆๅญ˜ๅœจ / Check if cli_app.py exists + if not cli_app_path.exists(): + print(f"โŒ CLI application file not found: {cli_app_path}") + print("Please ensure the cli/cli_app.py file exists.") + sys.exit(1) + + print(f"\n๐Ÿ“ CLI App location: {cli_app_path}") + print("๐Ÿ–ฅ๏ธ Starting DeepCode CLI interface...") + print("๐Ÿš€ Initializing command line application") + print("=" * 70) + print("๐Ÿ’ก Tip: Follow the interactive prompts to process your research") + print("๐Ÿ›‘ Press Ctrl+C to exit at any time") + print("=" * 70) + + # ๅฏๅŠจCLIๅบ”็”จ / Launch CLI application + try: + # ๅฏผๅ…ฅๅนถ่ฟ่กŒCLIๅบ”็”จ + if str(project_root) not in sys.path: + sys.path.insert(0, str(project_root)) # ๆทปๅŠ ้กน็›ฎๆ น็›ฎๅฝ•ๅˆฐ่ทฏๅพ„ + from cli.cli_app import main as cli_main + + print("\n๐ŸŽฏ Launching CLI application...") + + # ไฝฟ็”จasyncio่ฟ่กŒไธปๅ‡ฝๆ•ฐ + import asyncio + + asyncio.run(cli_main()) + + except KeyboardInterrupt: + print("\n\n๐Ÿ›‘ DeepCode CLI stopped by user") + print("Thank you for using DeepCode CLI! ๐Ÿงฌ") + except ImportError as e: + print(f"\nโŒ Failed to import CLI application: {e}") + print("Please check if all modules are properly installed.") + sys.exit(1) + except Exception as e: + print(f"\nโŒ Unexpected error: {e}") + print("Please check your Python environment and try again.") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/DeepCode/cli/main_cli.py b/DeepCode/cli/main_cli.py new file mode 100644 index 00000000..5ff3578d --- /dev/null +++ b/DeepCode/cli/main_cli.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 +""" +DeepCode CLI - Open-Source Code Agent +ๆทฑๅบฆไปฃ็ CLI - ๅผ€ๆบไปฃ็ ๆ™บ่ƒฝไฝ“ + +๐Ÿงฌ Data Intelligence Lab @ HKU +โšก Revolutionizing Research Reproducibility through Multi-Agent Architecture +""" + +import os +import sys +import asyncio +import argparse + +# ็ฆๆญข็”Ÿๆˆ.pycๆ–‡ไปถ +os.environ["PYTHONDONTWRITEBYTECODE"] = "1" + +# ๆทปๅŠ ้กน็›ฎๆ น็›ฎๅฝ•ๅˆฐ่ทฏๅพ„ +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +if parent_dir not in sys.path: + sys.path.insert(0, parent_dir) + +# ๅฏผๅ…ฅCLIๅบ”็”จ +from cli.cli_app import CLIApp, Colors + + +def print_enhanced_banner(): + """ๆ˜พ็คบๅขžๅผบ็‰ˆๅฏๅŠจๆจชๅน…""" + banner = f""" +{Colors.CYAN}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•— +โ•‘ โ•‘ +โ•‘ {Colors.BOLD}{Colors.MAGENTA}๐Ÿงฌ DeepCode - Open-Source Code Agent{Colors.CYAN} โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.BOLD}{Colors.YELLOW}โšก DATA INTELLIGENCE LAB @ HKU โšก{Colors.CYAN} โ•‘ +โ•‘ โ•‘ +โ•‘ Revolutionizing research reproducibility through collaborative AI โ•‘ +โ•‘ Building the future where code is reproduced from natural language โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.BOLD}{Colors.GREEN}๐Ÿค– Key Features:{Colors.CYAN} โ•‘ +โ•‘ โ€ข Automated paper-to-code reproduction โ•‘ +โ•‘ โ€ข Multi-agent collaborative architecture โ•‘ +โ•‘ โ€ข Open-source and extensible design โ•‘ +โ•‘ โ€ข Join our growing research community โ•‘ +โ•‘ โ•‘ +โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{Colors.ENDC} +""" + print(banner) + + +def check_environment(): + """ๆฃ€ๆŸฅ่ฟ่กŒ็Žฏๅขƒ""" + print(f"{Colors.CYAN}๐Ÿ” Checking environment...{Colors.ENDC}") + + # ๆฃ€ๆŸฅPython็‰ˆๆœฌ + if sys.version_info < (3, 8): + print( + f"{Colors.FAIL}โŒ Python 3.8+ required. Current: {sys.version}{Colors.ENDC}" + ) + return False + + print(f"{Colors.OKGREEN}โœ… Python {sys.version.split()[0]} - OK{Colors.ENDC}") + + # ๆฃ€ๆŸฅๅฟ…่ฆๆจกๅ— + required_modules = [ + ("asyncio", "Async IO support"), + ("pathlib", "Path handling"), + ("typing", "Type hints"), + ] + + missing_modules = [] + for module, desc in required_modules: + try: + __import__(module) + print(f"{Colors.OKGREEN}โœ… {desc} - OK{Colors.ENDC}") + except ImportError: + missing_modules.append(module) + print(f"{Colors.FAIL}โŒ {desc} - Missing{Colors.ENDC}") + + if missing_modules: + print( + f"{Colors.FAIL}โŒ Missing required modules: {', '.join(missing_modules)}{Colors.ENDC}" + ) + return False + + print(f"{Colors.OKGREEN}โœ… Environment check passed{Colors.ENDC}") + return True + + +def parse_arguments(): + """่งฃๆžๅ‘ฝไปค่กŒๅ‚ๆ•ฐ""" + parser = argparse.ArgumentParser( + description="DeepCode CLI - Open-Source Code Agent by Data Intelligence Lab @ HKU", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=f""" +{Colors.BOLD}Examples:{Colors.ENDC} + {Colors.CYAN}python main_cli.py{Colors.ENDC} # Interactive mode + {Colors.CYAN}python main_cli.py --file paper.pdf{Colors.ENDC} # Process file directly + {Colors.CYAN}python main_cli.py --url https://...{Colors.ENDC} # Process URL directly + {Colors.CYAN}python main_cli.py --chat "Build a web app..."{Colors.ENDC} # Process chat requirements + {Colors.CYAN}python main_cli.py --requirement "ML system for..."{Colors.ENDC} # Guided requirement analysis (NEW) + {Colors.CYAN}python main_cli.py --optimized{Colors.ENDC} # Use optimized mode + {Colors.CYAN}python main_cli.py --disable-segmentation{Colors.ENDC} # Disable document segmentation + {Colors.CYAN}python main_cli.py --segmentation-threshold 30000{Colors.ENDC} # Custom segmentation threshold + +{Colors.BOLD}Pipeline Modes:{Colors.ENDC} + {Colors.GREEN}Comprehensive{Colors.ENDC}: Full intelligence analysis with indexing + {Colors.YELLOW}Optimized{Colors.ENDC}: Fast processing without indexing + {Colors.BLUE}Requirement Analysis{Colors.ENDC}: Guided Q&A to refine requirements (NEW) + +{Colors.BOLD}Document Processing:{Colors.ENDC} + {Colors.BLUE}Smart Segmentation{Colors.ENDC}: Intelligent document segmentation for large papers + {Colors.MAGENTA}Supported Formats{Colors.ENDC}: PDF, DOCX, DOC, PPT, PPTX, XLS, XLSX, HTML, TXT, MD + """, + ) + + parser.add_argument( + "--file", "-f", type=str, help="Process a specific file (PDF, DOCX, TXT, etc.)" + ) + + parser.add_argument( + "--url", "-u", type=str, help="Process a research paper from URL" + ) + + parser.add_argument( + "--chat", + "-t", + type=str, + help="Process coding requirements via chat input (provide requirements as argument)", + ) + + parser.add_argument( + "--requirement", + "-r", + type=str, + help="Process requirements via guided analysis (provide initial idea as argument)", + ) + + parser.add_argument( + "--optimized", + "-o", + action="store_true", + help="Use optimized mode (skip indexing for faster processing)", + ) + + parser.add_argument( + "--disable-segmentation", + action="store_true", + help="Disable intelligent document segmentation (use traditional full-document processing)", + ) + + parser.add_argument( + "--segmentation-threshold", + type=int, + default=50000, + help="Document size threshold (characters) to trigger segmentation (default: 50000)", + ) + + parser.add_argument( + "--verbose", "-v", action="store_true", help="Enable verbose output" + ) + + return parser.parse_args() + + +async def run_direct_processing(app: CLIApp, input_source: str, input_type: str): + """็›ดๆŽฅๅค„็†ๆจกๅผ๏ผˆ้žไบคไบ’ๅผ๏ผ‰""" + try: + print( + f"\n{Colors.BOLD}{Colors.CYAN}๐Ÿš€ Starting direct processing mode...{Colors.ENDC}" + ) + print(f"{Colors.CYAN}Input: {input_source}{Colors.ENDC}") + print(f"{Colors.CYAN}Type: {input_type}{Colors.ENDC}") + print( + f"{Colors.CYAN}Mode: {'๐Ÿง  Comprehensive' if app.cli.enable_indexing else 'โšก Optimized'}{Colors.ENDC}" + ) + + # ๅˆๅง‹ๅŒ–ๅบ”็”จ + init_result = await app.initialize_mcp_app() + if init_result["status"] != "success": + print( + f"{Colors.FAIL}โŒ Initialization failed: {init_result['message']}{Colors.ENDC}" + ) + return False + + # ๅค„็†่พ“ๅ…ฅ + result = await app.process_input(input_source, input_type) + + if result["status"] == "success": + print( + f"\n{Colors.BOLD}{Colors.OKGREEN}๐ŸŽ‰ Processing completed successfully!{Colors.ENDC}" + ) + return True + else: + print( + f"\n{Colors.BOLD}{Colors.FAIL}โŒ Processing failed: {result.get('error', 'Unknown error')}{Colors.ENDC}" + ) + return False + + except Exception as e: + print(f"\n{Colors.FAIL}โŒ Direct processing error: {str(e)}{Colors.ENDC}") + return False + finally: + await app.cleanup_mcp_app() + + +async def run_requirement_analysis(app: CLIApp, initial_idea: str): + """้œ€ๆฑ‚ๅˆ†ๆžๆจกๅผ๏ผˆ้žไบคไบ’ๅผ๏ผ‰ - NEW: matching UI version""" + try: + print( + f"\n{Colors.BOLD}{Colors.BLUE}๐Ÿง  Starting requirement analysis mode...{Colors.ENDC}" + ) + print(f"{Colors.CYAN}Initial Idea: {initial_idea}{Colors.ENDC}") + + # ๅˆๅง‹ๅŒ–ๅบ”็”จ + init_result = await app.initialize_mcp_app() + if init_result["status"] != "success": + print( + f"{Colors.FAIL}โŒ Initialization failed: {init_result['message']}{Colors.ENDC}" + ) + return False + + # ๆ‰ง่กŒ้œ€ๆฑ‚ๅˆ†ๆžๅทฅไฝœๆต + result = await app.process_requirement_analysis_non_interactive(initial_idea) + + if result["status"] == "success": + print( + f"\n{Colors.BOLD}{Colors.OKGREEN}๐ŸŽ‰ Requirement analysis completed successfully!{Colors.ENDC}" + ) + return True + else: + print( + f"\n{Colors.BOLD}{Colors.FAIL}โŒ Requirement analysis failed: {result.get('error', 'Unknown error')}{Colors.ENDC}" + ) + return False + + except Exception as e: + print(f"\n{Colors.FAIL}โŒ Requirement analysis error: {str(e)}{Colors.ENDC}") + return False + finally: + await app.cleanup_mcp_app() + + +async def main(): + """ไธปๅ‡ฝๆ•ฐ""" + # ่งฃๆžๅ‘ฝไปค่กŒๅ‚ๆ•ฐ + args = parse_arguments() + + # ๆ˜พ็คบๆจชๅน… + print_enhanced_banner() + + # ๆฃ€ๆŸฅ็Žฏๅขƒ + if not check_environment(): + print( + f"\n{Colors.FAIL}๐Ÿšจ Environment check failed. Please fix the issues and try again.{Colors.ENDC}" + ) + sys.exit(1) + + try: + # ๅˆ›ๅปบCLIๅบ”็”จ + app = CLIApp() + + # ่ฎพ็ฝฎ้…็ฝฎ - ้ป˜่ฎค็ฆ็”จ็ดขๅผ•ๅŠŸ่ƒฝไปฅๅŠ ๅฟซๅค„็†้€Ÿๅบฆ + if args.optimized: + app.cli.enable_indexing = False + print( + f"\n{Colors.YELLOW}โšก Optimized mode enabled - indexing disabled{Colors.ENDC}" + ) + else: + # ้ป˜่ฎคไนŸ็ฆ็”จ็ดขๅผ•ๅŠŸ่ƒฝ + app.cli.enable_indexing = False + print( + f"\n{Colors.YELLOW}โšก Fast mode enabled - indexing disabled by default{Colors.ENDC}" + ) + + # Configure document segmentation settings + if hasattr(args, "disable_segmentation") and args.disable_segmentation: + print( + f"\n{Colors.MAGENTA}๐Ÿ“„ Document segmentation disabled - using traditional processing{Colors.ENDC}" + ) + app.cli.segmentation_enabled = False + app.cli.segmentation_threshold = args.segmentation_threshold + app.cli._save_segmentation_config() + else: + print( + f"\n{Colors.BLUE}๐Ÿ“„ Smart document segmentation enabled (threshold: {args.segmentation_threshold} chars){Colors.ENDC}" + ) + app.cli.segmentation_enabled = True + app.cli.segmentation_threshold = args.segmentation_threshold + app.cli._save_segmentation_config() + + # ๆฃ€ๆŸฅๆ˜ฏๅฆไธบ็›ดๆŽฅๅค„็†ๆจกๅผ + if args.file or args.url or args.chat or args.requirement: + if args.file: + # ้ชŒ่ฏๆ–‡ไปถๅญ˜ๅœจ + if not os.path.exists(args.file): + print(f"{Colors.FAIL}โŒ File not found: {args.file}{Colors.ENDC}") + sys.exit(1) + # ไฝฟ็”จ file:// ๅ‰็ผ€ไฟๆŒไธŽไบคไบ’ๆจกๅผไธ€่‡ด๏ผŒ็กฎไฟๆ–‡ไปถ่ขซๅคๅˆถ่€Œ้ž็งปๅŠจ + file_url = f"file://{os.path.abspath(args.file)}" + success = await run_direct_processing(app, file_url, "file") + elif args.url: + success = await run_direct_processing(app, args.url, "url") + elif args.chat: + # ้ชŒ่ฏchat่พ“ๅ…ฅ้•ฟๅบฆ + if len(args.chat.strip()) < 20: + print( + f"{Colors.FAIL}โŒ Chat input too short. Please provide more detailed requirements (at least 20 characters){Colors.ENDC}" + ) + sys.exit(1) + success = await run_direct_processing(app, args.chat, "chat") + elif args.requirement: + # NEW: Requirement analysis mode + # ้ชŒ่ฏ้œ€ๆฑ‚่พ“ๅ…ฅ้•ฟๅบฆ + if len(args.requirement.strip()) < 10: + print( + f"{Colors.FAIL}โŒ Requirement input too short. Please provide more details (at least 10 characters){Colors.ENDC}" + ) + sys.exit(1) + success = await run_requirement_analysis(app, args.requirement) + + sys.exit(0 if success else 1) + else: + # ไบคไบ’ๅผๆจกๅผ + print(f"\n{Colors.CYAN}๐ŸŽฎ Starting interactive mode...{Colors.ENDC}") + await app.run_interactive_session() + + except KeyboardInterrupt: + print(f"\n{Colors.WARNING}โš ๏ธ Application interrupted by user{Colors.ENDC}") + sys.exit(1) + except Exception as e: + print(f"\n{Colors.FAIL}โŒ Application errors: {str(e)}{Colors.ENDC}") + sys.exit(1) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/DeepCode/cli/workflows/__init__.py b/DeepCode/cli/workflows/__init__.py new file mode 100644 index 00000000..a1240698 --- /dev/null +++ b/DeepCode/cli/workflows/__init__.py @@ -0,0 +1,11 @@ +""" +CLI-specific Workflow Adapters +CLIไธ“็”จๅทฅไฝœๆต้€‚้…ๅ™จ + +This module provides CLI-optimized versions of workflow components that are +specifically adapted for command-line interface usage patterns. +""" + +from .cli_workflow_adapter import CLIWorkflowAdapter + +__all__ = ["CLIWorkflowAdapter"] diff --git a/DeepCode/cli/workflows/cli_workflow_adapter.py b/DeepCode/cli/workflows/cli_workflow_adapter.py new file mode 100644 index 00000000..fca0418d --- /dev/null +++ b/DeepCode/cli/workflows/cli_workflow_adapter.py @@ -0,0 +1,457 @@ +""" +CLI Workflow Adapter for Agent Orchestration Engine +CLIๅทฅไฝœๆต้€‚้…ๅ™จ - ๆ™บ่ƒฝไฝ“็ผ–ๆŽ’ๅผ•ๆ“Ž + +This adapter provides CLI-optimized interface to the latest agent orchestration engine, +with enhanced progress reporting, error handling, and CLI-specific optimizations. + +Version: 2.1 (Updated to match UI version - Added Requirement Analysis) +Changes: +- Default enable_indexing=False for faster processing (matching UI defaults) +- Mode-aware progress callback with detailed stage mapping +- Chat pipeline now accepts enable_indexing parameter +- Improved error handling and resource management +- Enhanced progress display for different modes (fast/comprehensive/chat) +- NEW: Added requirement analysis workflow support +""" + +import os +from typing import Callable, Dict, Any +from mcp_agent.app import MCPApp + + +class CLIWorkflowAdapter: + """ + CLI-optimized workflow adapter for the intelligent agent orchestration engine. + + This adapter provides: + - Enhanced CLI progress reporting + - Optimized error handling for CLI environments + - Streamlined interface for command-line usage + - Integration with the latest agent orchestration engine + """ + + def __init__(self, cli_interface=None): + """ + Initialize CLI workflow adapter. + + Args: + cli_interface: CLI interface instance for progress reporting + """ + self.cli_interface = cli_interface + self.app = None + self.logger = None + self.context = None + + async def initialize_mcp_app(self) -> Dict[str, Any]: + """ + Initialize MCP application for CLI usage (improved version matching UI). + + Returns: + dict: Initialization result + """ + try: + if self.cli_interface: + self.cli_interface.show_spinner( + "๐Ÿš€ Initializing Agent Orchestration Engine", 2.0 + ) + + # Initialize MCP application using async context manager (matching UI pattern) + self.app = MCPApp(name="cli_agent_orchestration") + self.app_context = self.app.run() + agent_app = await self.app_context.__aenter__() + + self.logger = agent_app.logger + self.context = agent_app.context + + # Configure filesystem access + self.context.config.mcp.servers["filesystem"].args.extend([os.getcwd()]) + + if self.cli_interface: + self.cli_interface.print_status( + "๐Ÿง  Agent Orchestration Engine initialized successfully", "success" + ) + + return { + "status": "success", + "message": "MCP application initialized successfully", + } + + except Exception as e: + error_msg = f"Failed to initialize MCP application: {str(e)}" + if self.cli_interface: + self.cli_interface.print_status(error_msg, "error") + return {"status": "error", "message": error_msg} + + async def cleanup_mcp_app(self): + """ + Clean up MCP application resources. + """ + if hasattr(self, "app_context"): + try: + await self.app_context.__aexit__(None, None, None) + if self.cli_interface: + self.cli_interface.print_status( + "๐Ÿงน Resources cleaned up successfully", "info" + ) + except Exception as e: + if self.cli_interface: + self.cli_interface.print_status( + f"โš ๏ธ Cleanup warning: {str(e)}", "warning" + ) + + def create_cli_progress_callback(self, enable_indexing: bool = True) -> Callable: + """ + Create CLI-optimized progress callback function with mode-aware stage mapping. + + This matches the UI version's detailed progress mapping logic. + + Args: + enable_indexing: Whether indexing is enabled (affects stage mapping) + + Returns: + Callable: Progress callback function + """ + + def progress_callback(progress: int, message: str): + if self.cli_interface: + # Mode-aware stage mapping (matching UI version logic) + if enable_indexing: + # Full workflow mapping: Initialize -> Analyze -> Download -> Plan -> References -> Repos -> Index -> Implement + if progress <= 5: + stage = 0 # Initialize + elif progress <= 10: + stage = 1 # Analyze + elif progress <= 25: + stage = 2 # Download + elif progress <= 40: + stage = 3 # Plan + elif progress <= 50: + stage = 4 # References + elif progress <= 60: + stage = 5 # Repos + elif progress <= 70: + stage = 6 # Index + elif progress <= 85: + stage = 7 # Implement + else: + stage = 8 # Complete + else: + # Fast mode mapping: Initialize -> Analyze -> Download -> Plan -> Implement + if progress <= 5: + stage = 0 # Initialize + elif progress <= 10: + stage = 1 # Analyze + elif progress <= 25: + stage = 2 # Download + elif progress <= 40: + stage = 3 # Plan + elif progress <= 85: + stage = 4 # Implement (skip References, Repos, Index) + else: + stage = 4 # Complete + + self.cli_interface.display_processing_stages(stage, enable_indexing) + + # Display status message + self.cli_interface.print_status(message, "processing") + + return progress_callback + + async def execute_full_pipeline( + self, input_source: str, enable_indexing: bool = False + ) -> Dict[str, Any]: + """ + Execute the complete intelligent multi-agent research orchestration pipeline. + + Updated to match UI version: default enable_indexing=False for faster processing. + + Args: + input_source: Research input source (file path, URL, or preprocessed analysis) + enable_indexing: Whether to enable advanced intelligence analysis (default: False) + + Returns: + dict: Comprehensive pipeline execution result + """ + try: + # Import the latest agent orchestration engine + from workflows.agent_orchestration_engine import ( + execute_multi_agent_research_pipeline, + ) + + # Create CLI progress callback with mode awareness + progress_callback = self.create_cli_progress_callback(enable_indexing) + + # Display pipeline start + if self.cli_interface: + if enable_indexing: + mode_msg = "๐Ÿง  comprehensive (with indexing)" + else: + mode_msg = "โšก fast (indexing disabled)" + self.cli_interface.print_status( + f"๐Ÿš€ Starting {mode_msg} agent orchestration pipeline...", + "processing", + ) + self.cli_interface.display_processing_stages(0, enable_indexing) + + # Execute the pipeline + result = await execute_multi_agent_research_pipeline( + input_source=input_source, + logger=self.logger, + progress_callback=progress_callback, + enable_indexing=enable_indexing, + ) + + # Display completion + if self.cli_interface: + final_stage = 8 if enable_indexing else 4 + self.cli_interface.display_processing_stages( + final_stage, enable_indexing + ) + self.cli_interface.print_status( + "๐ŸŽ‰ Agent orchestration pipeline completed successfully!", + "complete", + ) + + return { + "status": "success", + "result": result, + "pipeline_mode": "comprehensive" if enable_indexing else "optimized", + } + + except Exception as e: + error_msg = f"Pipeline execution failed: {str(e)}" + if self.cli_interface: + self.cli_interface.print_status(error_msg, "error") + + return { + "status": "error", + "error": error_msg, + "pipeline_mode": "comprehensive" if enable_indexing else "optimized", + } + + async def execute_requirement_analysis_workflow( + self, user_input: str, analysis_mode: str, user_answers: Dict[str, str] = None + ) -> Dict[str, Any]: + """ + Execute requirement analysis workflow (NEW: matching UI version). + + This workflow helps users refine their requirements through guided questions + and intelligent analysis before starting code implementation. + + Args: + user_input: User's initial requirements or description + analysis_mode: Analysis mode ("generate_questions" or "summarize_requirements") + user_answers: Dictionary of user answers to guiding questions (for summarize mode) + + Returns: + dict: Analysis result with questions or requirement summary + """ + try: + # Import the requirement analysis workflow + from workflows.agent_orchestration_engine import ( + execute_requirement_analysis_workflow, + ) + + # Create CLI progress callback + def analysis_progress_callback(progress: int, message: str): + if self.cli_interface: + self.cli_interface.print_status(message, "processing") + + # Display workflow start + if self.cli_interface: + if analysis_mode == "generate_questions": + self.cli_interface.print_status( + "๐Ÿค– Generating guiding questions for your requirements...", + "processing", + ) + else: + self.cli_interface.print_status( + "๐Ÿ“„ Analyzing and summarizing your detailed requirements...", + "processing", + ) + + # Execute the requirement analysis workflow + result = await execute_requirement_analysis_workflow( + user_input=user_input, + analysis_mode=analysis_mode, + user_answers=user_answers, + logger=self.logger, + progress_callback=analysis_progress_callback, + ) + + # Display completion + if self.cli_interface: + if result["status"] == "success": + if analysis_mode == "generate_questions": + self.cli_interface.print_status( + "โœ… Guiding questions generated successfully!", "success" + ) + else: + self.cli_interface.print_status( + "โœ… Requirements analysis completed successfully!", + "success", + ) + else: + self.cli_interface.print_status( + f"โŒ Analysis failed: {result.get('error', 'Unknown error')}", + "error", + ) + + return result + + except Exception as e: + error_msg = f"Requirement analysis workflow failed: {str(e)}" + if self.cli_interface: + self.cli_interface.print_status(error_msg, "error") + + return {"status": "error", "error": error_msg} + + async def execute_chat_pipeline( + self, user_input: str, enable_indexing: bool = False + ) -> Dict[str, Any]: + """ + Execute the chat-based planning and implementation pipeline. + + Updated to match UI version: accepts enable_indexing parameter. + + Args: + user_input: User's coding requirements and description + enable_indexing: Whether to enable indexing for enhanced code understanding (default: False) + + Returns: + dict: Chat pipeline execution result + """ + try: + # Import the chat-based pipeline + from workflows.agent_orchestration_engine import ( + execute_chat_based_planning_pipeline, + ) + + # Create CLI progress callback for chat mode + def chat_progress_callback(progress: int, message: str): + if self.cli_interface: + # Map progress to CLI stages for chat mode (matching UI logic) + if progress <= 5: + stage = 0 # Initialize + elif progress <= 30: + stage = 1 # Planning + elif progress <= 50: + stage = 2 # Setup + elif progress <= 70: + stage = 3 # Save Plan + else: + stage = 4 # Implement + + self.cli_interface.display_processing_stages(stage, chat_mode=True) + + # Display status message + self.cli_interface.print_status(message, "processing") + + # Display pipeline start + if self.cli_interface: + indexing_note = ( + " (with indexing)" if enable_indexing else " (fast mode)" + ) + self.cli_interface.print_status( + f"๐Ÿš€ Starting chat-based planning pipeline{indexing_note}...", + "processing", + ) + self.cli_interface.display_processing_stages(0, chat_mode=True) + + # Execute the chat pipeline with configurable indexing + result = await execute_chat_based_planning_pipeline( + user_input=user_input, + logger=self.logger, + progress_callback=chat_progress_callback, + enable_indexing=enable_indexing, # Pass through enable_indexing parameter + ) + + # Display completion + if self.cli_interface: + self.cli_interface.display_processing_stages(4, chat_mode=True) + self.cli_interface.print_status( + "๐ŸŽ‰ Chat-based planning pipeline completed successfully!", + "complete", + ) + + return {"status": "success", "result": result, "pipeline_mode": "chat"} + + except Exception as e: + error_msg = f"Chat pipeline execution failed: {str(e)}" + if self.cli_interface: + self.cli_interface.print_status(error_msg, "error") + + return {"status": "error", "error": error_msg, "pipeline_mode": "chat"} + + async def process_input_with_orchestration( + self, input_source: str, input_type: str, enable_indexing: bool = False + ) -> Dict[str, Any]: + """ + Process input using the intelligent agent orchestration engine. + + This is the main CLI interface to the latest agent orchestration capabilities. + Updated to match UI version: default enable_indexing=False. + + Args: + input_source: Input source (file path, URL, or chat input) + input_type: Type of input ('file', 'url', or 'chat') + enable_indexing: Whether to enable advanced intelligence analysis (default: False) + + Returns: + dict: Processing result with status and details + """ + pipeline_result = None + + try: + # Initialize MCP app + init_result = await self.initialize_mcp_app() + if init_result["status"] != "success": + return init_result + + # Process file:// URLs for traditional file/URL inputs + if input_source.startswith("file://"): + file_path = input_source[7:] + if os.name == "nt" and file_path.startswith("/"): + file_path = file_path.lstrip("/") + input_source = file_path + + # Execute appropriate pipeline based on input type + if input_type == "chat": + # Use chat-based planning pipeline for user requirements + # Pass enable_indexing to chat pipeline as well + pipeline_result = await self.execute_chat_pipeline( + input_source, enable_indexing=enable_indexing + ) + else: + # Use traditional multi-agent research pipeline for files/URLs + pipeline_result = await self.execute_full_pipeline( + input_source, enable_indexing=enable_indexing + ) + + return { + "status": pipeline_result["status"], + "analysis_result": "Integrated into agent orchestration pipeline", + "download_result": "Integrated into agent orchestration pipeline", + "repo_result": pipeline_result.get("result", ""), + "pipeline_mode": pipeline_result.get("pipeline_mode", "comprehensive"), + "error": pipeline_result.get("error"), + } + + except Exception as e: + error_msg = f"Error during orchestrated processing: {str(e)}" + if self.cli_interface: + self.cli_interface.print_status(error_msg, "error") + + return { + "status": "error", + "error": error_msg, + "analysis_result": "", + "download_result": "", + "repo_result": "", + "pipeline_mode": "comprehensive" if enable_indexing else "optimized", + } + + finally: + # Clean up resources + await self.cleanup_mcp_app() diff --git a/DeepCode/config/mcp_tool_definitions.py b/DeepCode/config/mcp_tool_definitions.py new file mode 100644 index 00000000..0ae5c31f --- /dev/null +++ b/DeepCode/config/mcp_tool_definitions.py @@ -0,0 +1,375 @@ +""" +MCPๅทฅๅ…ทๅฎšไน‰้…็ฝฎๆจกๅ— +MCP Tool Definitions Configuration Module + +ๅฐ†ๅทฅๅ…ทๅฎšไน‰ไปŽไธป็จ‹ๅบ้€ป่พ‘ไธญๅˆ†็ฆป๏ผŒๆไพ›ๆ ‡ๅ‡†ๅŒ–็š„ๅทฅๅ…ทๅฎšไน‰ๆ ผๅผ +Separate tool definitions from main program logic, providing standardized tool definition format + +ๆ”ฏๆŒ็š„ๅทฅๅ…ท็ฑปๅž‹๏ผš +- ๆ–‡ไปถๆ“ไฝœๅทฅๅ…ท (File Operations) +- ไปฃ็ ๆ‰ง่กŒๅทฅๅ…ท (Code Execution) +- ๆœ็ดขๅทฅๅ…ท (Search Tools) +- ้กน็›ฎ็ป“ๆž„ๅทฅๅ…ท (Project Structure Tools) +""" + +from typing import Dict, List, Any + + +class MCPToolDefinitions: + """MCPๅทฅๅ…ทๅฎšไน‰็ฎก็†ๅ™จ""" + + @staticmethod + def get_code_implementation_tools() -> List[Dict[str, Any]]: + """ + ่Žทๅ–ไปฃ็ ๅฎž็Žฐ็›ธๅ…ณ็š„ๅทฅๅ…ทๅฎšไน‰ + Get tool definitions for code implementation + """ + return [ + # MCPToolDefinitions._get_read_file_tool(), + # MCPToolDefinitions._get_read_multiple_files_tool(), + # MCPToolDefinitions._get_read_code_mem_tool(), + MCPToolDefinitions._get_write_file_tool(), + # MCPToolDefinitions._get_write_multiple_files_tool(), + # MCPToolDefinitions._get_execute_python_tool(), + # MCPToolDefinitions._get_execute_bash_tool(), + ] + + @staticmethod + def _get_read_file_tool() -> Dict[str, Any]: + """่ฏปๅ–ๆ–‡ไปถๅทฅๅ…ทๅฎšไน‰""" + return { + "name": "read_file", + "description": "Read file content, supports specifying line number range", + "input_schema": { + "type": "object", + "properties": { + "file_path": { + "type": "string", + "description": "File path, relative to workspace", + }, + "start_line": { + "type": "integer", + "description": "Start line number (starting from 1, optional)", + }, + "end_line": { + "type": "integer", + "description": "End line number (starting from 1, optional)", + }, + }, + "required": ["file_path"], + }, + } + + @staticmethod + def _get_read_multiple_files_tool() -> Dict[str, Any]: + """ๆ‰น้‡่ฏปๅ–ๅคšไธชๆ–‡ไปถๅทฅๅ…ทๅฎšไน‰""" + return { + "name": "read_multiple_files", + "description": "Read multiple files in a single operation (for batch reading)", + "input_schema": { + "type": "object", + "properties": { + "file_requests": { + "type": "string", + "description": 'JSON string with file requests, e.g., \'{"file1.py": {}, "file2.py": {"start_line": 1, "end_line": 10}}\' or simple array \'["file1.py", "file2.py"]\'', + }, + "max_files": { + "type": "integer", + "description": "Maximum number of files to read in one operation", + "default": 5, + "minimum": 1, + "maximum": 10, + }, + }, + "required": ["file_requests"], + }, + } + + @staticmethod + def _get_read_code_mem_tool() -> Dict[str, Any]: + """Read code memory tool definition - reads from implement_code_summary.md""" + return { + "name": "read_code_mem", + "description": "Check if file summaries exist in implement_code_summary.md for multiple files in a single call. Returns summaries for all requested files if available.", + "input_schema": { + "type": "object", + "properties": { + "file_paths": { + "type": "array", + "items": {"type": "string"}, + "description": "List of file paths to check for summary information in implement_code_summary.md", + } + }, + "required": ["file_paths"], + }, + } + + @staticmethod + def _get_write_file_tool() -> Dict[str, Any]: + """ๅ†™ๅ…ฅๆ–‡ไปถๅทฅๅ…ทๅฎšไน‰""" + return { + "name": "write_file", + "description": "Write content to file", + "input_schema": { + "type": "object", + "properties": { + "file_path": { + "type": "string", + "description": "File path, relative to workspace", + }, + "content": { + "type": "string", + "description": "Content to write to file", + }, + "create_dirs": { + "type": "boolean", + "description": "Whether to create directories if they don't exist", + "default": True, + }, + "create_backup": { + "type": "boolean", + "description": "Whether to create backup file if file already exists", + "default": False, + }, + }, + "required": ["file_path", "content"], + }, + } + + @staticmethod + def _get_write_multiple_files_tool() -> Dict[str, Any]: + """ๆ‰น้‡ๅ†™ๅ…ฅๅคšไธชๆ–‡ไปถๅทฅๅ…ทๅฎšไน‰""" + return { + "name": "write_multiple_files", + "description": "Write multiple files in a single operation (for batch implementation)", + "input_schema": { + "type": "object", + "properties": { + "file_implementations": { + "type": "string", + "description": 'JSON string mapping file paths to content, e.g., \'{"file1.py": "content1", "file2.py": "content2"}\'', + }, + "create_dirs": { + "type": "boolean", + "description": "Whether to create directories if they don't exist", + "default": True, + }, + "create_backup": { + "type": "boolean", + "description": "Whether to create backup files if they already exist", + "default": False, + }, + "max_files": { + "type": "integer", + "description": "Maximum number of files to write in one operation", + "default": 5, + "minimum": 1, + "maximum": 10, + }, + }, + "required": ["file_implementations"], + }, + } + + @staticmethod + def _get_execute_python_tool() -> Dict[str, Any]: + """Pythonๆ‰ง่กŒๅทฅๅ…ทๅฎšไน‰""" + return { + "name": "execute_python", + "description": "Execute Python code and return output", + "input_schema": { + "type": "object", + "properties": { + "code": {"type": "string", "description": "Python code to execute"}, + "timeout": { + "type": "integer", + "description": "Timeout in seconds", + "default": 30, + }, + }, + "required": ["code"], + }, + } + + @staticmethod + def _get_execute_bash_tool() -> Dict[str, Any]: + """Bashๆ‰ง่กŒๅทฅๅ…ทๅฎšไน‰""" + return { + "name": "execute_bash", + "description": "Execute bash command", + "input_schema": { + "type": "object", + "properties": { + "command": { + "type": "string", + "description": "Bash command to execute", + }, + "timeout": { + "type": "integer", + "description": "Timeout in seconds", + "default": 30, + }, + }, + "required": ["command"], + }, + } + + @staticmethod + def _get_file_structure_tool() -> Dict[str, Any]: + """ๆ–‡ไปถ็ป“ๆž„่Žทๅ–ๅทฅๅ…ทๅฎšไน‰""" + return { + "name": "get_file_structure", + "description": "Get directory file structure", + "input_schema": { + "type": "object", + "properties": { + "directory": { + "type": "string", + "description": "Directory path, relative to workspace", + "default": ".", + }, + "max_depth": { + "type": "integer", + "description": "Maximum traversal depth", + "default": 5, + }, + }, + }, + } + + @staticmethod + def _get_search_code_references_tool() -> Dict[str, Any]: + """็ปŸไธ€ไปฃ็ ๅ‚่€ƒๆœ็ดขๅทฅๅ…ทๅฎšไน‰ - ๅˆๅนถไบ†ไธ‰ไธชๆญฅ้ชคไธบไธ€ไธชๅทฅๅ…ท""" + return { + "name": "search_code_references", + "description": "UNIFIED TOOL: Search relevant reference code from index files. Combines directory setup, index loading, and searching in a single call.", + "input_schema": { + "type": "object", + "properties": { + "indexes_path": { + "type": "string", + "description": "Path to the indexes directory containing JSON index files", + }, + "target_file": { + "type": "string", + "description": "Target file path to be implemented", + }, + "keywords": { + "type": "string", + "description": "Search keywords, comma-separated", + "default": "", + }, + "max_results": { + "type": "integer", + "description": "Maximum number of results to return", + "default": 10, + }, + }, + "required": ["indexes_path", "target_file"], + }, + } + + @staticmethod + def _get_get_indexes_overview_tool() -> Dict[str, Any]: + """่Žทๅ–็ดขๅผ•ๆฆ‚่งˆๅทฅๅ…ทๅฎšไน‰""" + return { + "name": "get_indexes_overview", + "description": "Get overview of all available reference code index information from specified directory", + "input_schema": { + "type": "object", + "properties": { + "indexes_path": { + "type": "string", + "description": "Path to the indexes directory containing JSON index files", + } + }, + "required": ["indexes_path"], + }, + } + + @staticmethod + def _get_set_workspace_tool() -> Dict[str, Any]: + """Set workspace directory tool definition""" + return { + "name": "set_workspace", + "description": "Set the workspace directory for file operations", + "input_schema": { + "type": "object", + "properties": { + "workspace_path": { + "type": "string", + "description": "Directory path for the workspace", + } + }, + "required": ["workspace_path"], + }, + } + + # @staticmethod + # def _get_set_indexes_directory_tool() -> Dict[str, Any]: + # """Set indexes directory tool definition - DEPRECATED: Use unified search_code_references instead""" + # return { + # "name": "set_indexes_directory", + # "description": "Set the directory path for code reference indexes", + # "input_schema": { + # "type": "object", + # "properties": { + # "indexes_path": { + # "type": "string", + # "description": "Directory path containing index JSON files" + # } + # }, + # "required": ["indexes_path"] + # } + # } + + @staticmethod + def get_available_tool_sets() -> Dict[str, str]: + """ + ่Žทๅ–ๅฏ็”จ็š„ๅทฅๅ…ท้›†ๅˆ + Get available tool sets + """ + return { + "code_implementation": "ไปฃ็ ๅฎž็Žฐ็›ธๅ…ณๅทฅๅ…ท้›† / Code implementation tool set", + # ๅฏไปฅๅœจ่ฟ™้‡ŒๆทปๅŠ ๆ›ดๅคšๅทฅๅ…ท้›† + # "data_analysis": "ๆ•ฐๆฎๅˆ†ๆžๅทฅๅ…ท้›† / Data analysis tool set", + # "web_scraping": "็ฝ‘้กต็ˆฌๅ–ๅทฅๅ…ท้›† / Web scraping tool set", + } + + @staticmethod + def get_tool_set(tool_set_name: str) -> List[Dict[str, Any]]: + """ + ๆ นๆฎๅ็งฐ่Žทๅ–็‰นๅฎš็š„ๅทฅๅ…ท้›† + Get specific tool set by name + """ + tool_sets = { + "code_implementation": MCPToolDefinitions.get_code_implementation_tools(), + } + + return tool_sets.get(tool_set_name, []) + + @staticmethod + def get_all_tools() -> List[Dict[str, Any]]: + """ + ่Žทๅ–ๆ‰€ๆœ‰ๅฏ็”จๅทฅๅ…ท + Get all available tools + """ + all_tools = [] + for tool_set_name in MCPToolDefinitions.get_available_tool_sets().keys(): + all_tools.extend(MCPToolDefinitions.get_tool_set(tool_set_name)) + return all_tools + + +# ไพฟๆท่ฎฟ้—ฎๅ‡ฝๆ•ฐ +def get_mcp_tools(tool_set: str = "code_implementation") -> List[Dict[str, Any]]: + """ + ไพฟๆทๅ‡ฝๆ•ฐ๏ผš่Žทๅ–MCPๅทฅๅ…ทๅฎšไน‰ + Convenience function: Get MCP tool definitions + + Args: + tool_set: ๅทฅๅ…ท้›†ๅ็งฐ (้ป˜่ฎค: "code_implementation") + + Returns: + ๅทฅๅ…ทๅฎšไน‰ๅˆ—่กจ + """ + return MCPToolDefinitions.get_tool_set(tool_set) diff --git a/DeepCode/config/mcp_tool_definitions_index.py b/DeepCode/config/mcp_tool_definitions_index.py new file mode 100644 index 00000000..b414f85d --- /dev/null +++ b/DeepCode/config/mcp_tool_definitions_index.py @@ -0,0 +1,620 @@ +""" +MCPๅทฅๅ…ทๅฎšไน‰้…็ฝฎๆจกๅ— +MCP Tool Definitions Configuration Module + +ๅฐ†ๅทฅๅ…ทๅฎšไน‰ไปŽไธป็จ‹ๅบ้€ป่พ‘ไธญๅˆ†็ฆป๏ผŒๆไพ›ๆ ‡ๅ‡†ๅŒ–็š„ๅทฅๅ…ทๅฎšไน‰ๆ ผๅผ +Separate tool definitions from main program logic, providing standardized tool definition format + +ๆ”ฏๆŒ็š„ๅทฅๅ…ท็ฑปๅž‹๏ผš +- ๆ–‡ไปถๆ“ไฝœๅทฅๅ…ท (File Operations) +- ไปฃ็ ๆ‰ง่กŒๅทฅๅ…ท (Code Execution) +- ๆœ็ดขๅทฅๅ…ท (Search Tools) +- ้กน็›ฎ็ป“ๆž„ๅทฅๅ…ท (Project Structure Tools) +""" + +from typing import Dict, List, Any + + +class MCPToolDefinitions: + """MCPๅทฅๅ…ทๅฎšไน‰็ฎก็†ๅ™จ""" + + @staticmethod + def get_code_implementation_tools() -> List[Dict[str, Any]]: + """ + ่Žทๅ–ไปฃ็ ๅฎž็Žฐ็›ธๅ…ณ็š„ๅทฅๅ…ทๅฎšไน‰ + Get tool definitions for code implementation + """ + return [ + # MCPToolDefinitions._get_read_file_tool(), + # MCPToolDefinitions._get_read_multiple_files_tool(), + # MCPToolDefinitions._get_read_code_mem_tool(), + MCPToolDefinitions._get_write_file_tool(), + # MCPToolDefinitions._get_write_multiple_files_tool(), + # MCPToolDefinitions._get_execute_python_tool(), + # MCPToolDefinitions._get_execute_bash_tool(), + MCPToolDefinitions._get_search_code_references_tool(), + # MCPToolDefinitions._get_search_code_tool(), + # MCPToolDefinitions._get_file_structure_tool(), + # MCPToolDefinitions._get_set_workspace_tool(), + # MCPToolDefinitions._get_operation_history_tool(), + ] + + @staticmethod + def get_code_evaluation_tools() -> List[Dict[str, Any]]: + """ + ่Žทๅ–ไปฃ็ ่ฏ„ไผฐ็›ธๅ…ณ็š„ๅทฅๅ…ทๅฎšไน‰ + Get tool definitions for code evaluation + """ + return [ + MCPToolDefinitions._get_analyze_repo_structure_tool(), + MCPToolDefinitions._get_detect_dependencies_tool(), + MCPToolDefinitions._get_assess_code_quality_tool(), + MCPToolDefinitions._get_evaluate_documentation_tool(), + MCPToolDefinitions._get_check_reproduction_readiness_tool(), + MCPToolDefinitions._get_generate_evaluation_summary_tool(), + MCPToolDefinitions._get_detect_empty_files_tool(), + MCPToolDefinitions._get_detect_missing_files_tool(), + MCPToolDefinitions._get_generate_code_revision_report_tool(), + ] + + @staticmethod + def _get_read_file_tool() -> Dict[str, Any]: + """่ฏปๅ–ๆ–‡ไปถๅทฅๅ…ทๅฎšไน‰""" + return { + "name": "read_file", + "description": "Read file content, supports specifying line number range", + "input_schema": { + "type": "object", + "properties": { + "file_path": { + "type": "string", + "description": "File path, relative to workspace", + }, + "start_line": { + "type": "integer", + "description": "Start line number (starting from 1, optional)", + }, + "end_line": { + "type": "integer", + "description": "End line number (starting from 1, optional)", + }, + }, + "required": ["file_path"], + }, + } + + @staticmethod + def _get_read_multiple_files_tool() -> Dict[str, Any]: + """ๆ‰น้‡่ฏปๅ–ๅคšไธชๆ–‡ไปถๅทฅๅ…ทๅฎšไน‰""" + return { + "name": "read_multiple_files", + "description": "Read multiple files in a single operation (for batch reading)", + "input_schema": { + "type": "object", + "properties": { + "file_requests": { + "type": "string", + "description": 'JSON string with file requests, e.g., \'{"file1.py": {}, "file2.py": {"start_line": 1, "end_line": 10}}\' or simple array \'["file1.py", "file2.py"]\'', + }, + "max_files": { + "type": "integer", + "description": "Maximum number of files to read in one operation", + "default": 5, + "minimum": 1, + "maximum": 10, + }, + }, + "required": ["file_requests"], + }, + } + + @staticmethod + def _get_read_code_mem_tool() -> Dict[str, Any]: + """Read code memory tool definition - reads from implement_code_summary.md""" + return { + "name": "read_code_mem", + "description": "Check if file summaries exist in implement_code_summary.md for multiple files in a single call. Returns summaries for all requested files if available.", + "input_schema": { + "type": "object", + "properties": { + "file_paths": { + "type": "array", + "items": {"type": "string"}, + "description": "List of file paths to check for summary information in implement_code_summary.md", + } + }, + "required": ["file_paths"], + }, + } + + @staticmethod + def _get_write_file_tool() -> Dict[str, Any]: + """ๅ†™ๅ…ฅๆ–‡ไปถๅทฅๅ…ทๅฎšไน‰""" + return { + "name": "write_file", + "description": "Write content to file", + "input_schema": { + "type": "object", + "properties": { + "file_path": { + "type": "string", + "description": "File path, relative to workspace", + }, + "content": { + "type": "string", + "description": "Content to write to file", + }, + "create_dirs": { + "type": "boolean", + "description": "Whether to create directories if they don't exist", + "default": True, + }, + "create_backup": { + "type": "boolean", + "description": "Whether to create backup file if file already exists", + "default": False, + }, + }, + "required": ["file_path", "content"], + }, + } + + @staticmethod + def _get_write_multiple_files_tool() -> Dict[str, Any]: + """ๆ‰น้‡ๅ†™ๅ…ฅๅคšไธชๆ–‡ไปถๅทฅๅ…ทๅฎšไน‰""" + return { + "name": "write_multiple_files", + "description": "Write multiple files in a single operation (for batch implementation)", + "input_schema": { + "type": "object", + "properties": { + "file_implementations": { + "type": "string", + "description": 'JSON string mapping file paths to content, e.g., \'{"file1.py": "content1", "file2.py": "content2"}\'', + }, + "create_dirs": { + "type": "boolean", + "description": "Whether to create directories if they don't exist", + "default": True, + }, + "create_backup": { + "type": "boolean", + "description": "Whether to create backup files if they already exist", + "default": False, + }, + "max_files": { + "type": "integer", + "description": "Maximum number of files to write in one operation", + "default": 5, + "minimum": 1, + "maximum": 10, + }, + }, + "required": ["file_implementations"], + }, + } + + @staticmethod + def _get_execute_python_tool() -> Dict[str, Any]: + """Pythonๆ‰ง่กŒๅทฅๅ…ทๅฎšไน‰""" + return { + "name": "execute_python", + "description": "Execute Python code and return output", + "input_schema": { + "type": "object", + "properties": { + "code": {"type": "string", "description": "Python code to execute"}, + "timeout": { + "type": "integer", + "description": "Timeout in seconds", + "default": 30, + }, + }, + "required": ["code"], + }, + } + + @staticmethod + def _get_execute_bash_tool() -> Dict[str, Any]: + """Bashๆ‰ง่กŒๅทฅๅ…ทๅฎšไน‰""" + return { + "name": "execute_bash", + "description": "Execute bash command", + "input_schema": { + "type": "object", + "properties": { + "command": { + "type": "string", + "description": "Bash command to execute", + }, + "timeout": { + "type": "integer", + "description": "Timeout in seconds", + "default": 30, + }, + }, + "required": ["command"], + }, + } + + @staticmethod + def _get_file_structure_tool() -> Dict[str, Any]: + """ๆ–‡ไปถ็ป“ๆž„่Žทๅ–ๅทฅๅ…ทๅฎšไน‰""" + return { + "name": "get_file_structure", + "description": "Get directory file structure", + "input_schema": { + "type": "object", + "properties": { + "directory": { + "type": "string", + "description": "Directory path, relative to workspace", + "default": ".", + }, + "max_depth": { + "type": "integer", + "description": "Maximum traversal depth", + "default": 5, + }, + }, + }, + } + + @staticmethod + def _get_search_code_references_tool() -> Dict[str, Any]: + """็ปŸไธ€ไปฃ็ ๅ‚่€ƒๆœ็ดขๅทฅๅ…ทๅฎšไน‰ - ๅˆๅนถไบ†ไธ‰ไธชๆญฅ้ชคไธบไธ€ไธชๅทฅๅ…ท""" + return { + "name": "search_code_references", + "description": "UNIFIED TOOL: Search relevant reference code from index files. Combines directory setup, index loading, and searching in a single call.", + "input_schema": { + "type": "object", + "properties": { + "indexes_path": { + "type": "string", + "description": "Path to the indexes directory containing JSON index files", + }, + "target_file": { + "type": "string", + "description": "Target file path to be implemented", + }, + "keywords": { + "type": "string", + "description": "Search keywords, comma-separated", + "default": "", + }, + "max_results": { + "type": "integer", + "description": "Maximum number of results to return", + "default": 10, + }, + }, + "required": ["indexes_path", "target_file"], + }, + } + + @staticmethod + def _get_search_code_tool() -> Dict[str, Any]: + """ไปฃ็ ๆœ็ดขๅทฅๅ…ทๅฎšไน‰ - ๅœจๅฝ“ๅ‰ไปฃ็ ๅบ“ไธญๆœ็ดขๆจกๅผ""" + return { + "name": "search_code", + "description": "Search patterns in code files within the current repository", + "input_schema": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "Search pattern", + }, + "file_pattern": { + "type": "string", + "description": "File pattern (e.g., '*.py')", + "default": "*.py", + }, + "use_regex": { + "type": "boolean", + "description": "Whether to use regular expressions", + "default": False, + }, + "search_directory": { + "type": "string", + "description": "Specify search directory (optional)", + }, + }, + "required": ["pattern"], + }, + } + + @staticmethod + def _get_operation_history_tool() -> Dict[str, Any]: + """ๆ“ไฝœๅŽ†ๅฒๅทฅๅ…ทๅฎšไน‰""" + return { + "name": "get_operation_history", + "description": "Get operation history", + "input_schema": { + "type": "object", + "properties": { + "last_n": { + "type": "integer", + "description": "Return the last N operations", + "default": 10, + }, + }, + }, + } + + @staticmethod + def _get_get_indexes_overview_tool() -> Dict[str, Any]: + """่Žทๅ–็ดขๅผ•ๆฆ‚่งˆๅทฅๅ…ทๅฎšไน‰""" + return { + "name": "get_indexes_overview", + "description": "Get overview of all available reference code index information from specified directory", + "input_schema": { + "type": "object", + "properties": { + "indexes_path": { + "type": "string", + "description": "Path to the indexes directory containing JSON index files", + } + }, + "required": ["indexes_path"], + }, + } + + @staticmethod + def _get_set_workspace_tool() -> Dict[str, Any]: + """Set workspace directory tool definition""" + return { + "name": "set_workspace", + "description": "Set the workspace directory for file operations", + "input_schema": { + "type": "object", + "properties": { + "workspace_path": { + "type": "string", + "description": "Directory path for the workspace", + } + }, + "required": ["workspace_path"], + }, + } + + # @staticmethod + # def _get_set_indexes_directory_tool() -> Dict[str, Any]: + # """Set indexes directory tool definition - DEPRECATED: Use unified search_code_references instead""" + # return { + # "name": "set_indexes_directory", + # "description": "Set the directory path for code reference indexes", + # "input_schema": { + # "type": "object", + # "properties": { + # "indexes_path": { + # "type": "string", + # "description": "Directory path containing index JSON files" + # } + # }, + # "required": ["indexes_path"] + # } + # } + + # Code evaluation tool definitions + @staticmethod + def _get_analyze_repo_structure_tool() -> Dict[str, Any]: + return { + "name": "analyze_repo_structure", + "description": "Perform comprehensive repository structure analysis", + "input_schema": { + "type": "object", + "properties": { + "repo_path": { + "type": "string", + "description": "Path to the repository to analyze", + } + }, + "required": ["repo_path"], + }, + } + + @staticmethod + def _get_detect_dependencies_tool() -> Dict[str, Any]: + return { + "name": "detect_dependencies", + "description": "Detect and analyze project dependencies across multiple languages", + "input_schema": { + "type": "object", + "properties": { + "repo_path": { + "type": "string", + "description": "Path to the repository", + } + }, + "required": ["repo_path"], + }, + } + + @staticmethod + def _get_assess_code_quality_tool() -> Dict[str, Any]: + return { + "name": "assess_code_quality", + "description": "Assess code quality metrics and identify potential issues", + "input_schema": { + "type": "object", + "properties": { + "repo_path": { + "type": "string", + "description": "Path to the repository", + } + }, + "required": ["repo_path"], + }, + } + + @staticmethod + def _get_evaluate_documentation_tool() -> Dict[str, Any]: + return { + "name": "evaluate_documentation", + "description": "Evaluate documentation completeness and quality", + "input_schema": { + "type": "object", + "properties": { + "repo_path": { + "type": "string", + "description": "Path to the repository", + }, + "docs_path": { + "type": "string", + "description": "Optional path to external documentation", + }, + }, + "required": ["repo_path"], + }, + } + + @staticmethod + def _get_check_reproduction_readiness_tool() -> Dict[str, Any]: + return { + "name": "check_reproduction_readiness", + "description": "Assess repository readiness for reproduction and validation", + "input_schema": { + "type": "object", + "properties": { + "repo_path": { + "type": "string", + "description": "Path to the repository", + }, + "docs_path": { + "type": "string", + "description": "Optional path to reproduction documentation", + }, + }, + "required": ["repo_path"], + }, + } + + @staticmethod + def _get_generate_evaluation_summary_tool() -> Dict[str, Any]: + return { + "name": "generate_evaluation_summary", + "description": "Generate comprehensive evaluation summary combining all analysis results", + "input_schema": { + "type": "object", + "properties": { + "repo_path": { + "type": "string", + "description": "Path to the repository", + }, + "docs_path": { + "type": "string", + "description": "Optional path to reproduction documentation", + }, + }, + "required": ["repo_path"], + }, + } + + @staticmethod + def _get_detect_empty_files_tool() -> Dict[str, Any]: + return { + "name": "detect_empty_files", + "description": "Detect empty files in the repository that may need implementation", + "input_schema": { + "type": "object", + "properties": { + "repo_path": { + "type": "string", + "description": "Path to the repository to analyze", + } + }, + "required": ["repo_path"], + }, + } + + @staticmethod + def _get_detect_missing_files_tool() -> Dict[str, Any]: + return { + "name": "detect_missing_files", + "description": "Detect missing essential files like main programs, tests, requirements, etc.", + "input_schema": { + "type": "object", + "properties": { + "repo_path": { + "type": "string", + "description": "Path to the repository to analyze", + } + }, + "required": ["repo_path"], + }, + } + + @staticmethod + def _get_generate_code_revision_report_tool() -> Dict[str, Any]: + return { + "name": "generate_code_revision_report", + "description": "Generate comprehensive code revision report combining empty files, missing files, and quality analysis", + "input_schema": { + "type": "object", + "properties": { + "repo_path": { + "type": "string", + "description": "Path to the repository to analyze", + }, + "docs_path": { + "type": "string", + "description": "Optional path to documentation", + }, + }, + "required": ["repo_path"], + }, + } + + @staticmethod + def get_available_tool_sets() -> Dict[str, str]: + """ + ่Žทๅ–ๅฏ็”จ็š„ๅทฅๅ…ท้›†ๅˆ + Get available tool sets + """ + return { + "code_implementation": "ไปฃ็ ๅฎž็Žฐ็›ธๅ…ณๅทฅๅ…ท้›† / Code implementation tool set", + "code_evaluation": "ไปฃ็ ่ฏ„ไผฐ็›ธๅ…ณๅทฅๅ…ท้›† / Code evaluation tool set", + # ๅฏไปฅๅœจ่ฟ™้‡ŒๆทปๅŠ ๆ›ดๅคšๅทฅๅ…ท้›† + # "data_analysis": "ๆ•ฐๆฎๅˆ†ๆžๅทฅๅ…ท้›† / Data analysis tool set", + # "web_scraping": "็ฝ‘้กต็ˆฌๅ–ๅทฅๅ…ท้›† / Web scraping tool set", + } + + @staticmethod + def get_tool_set(tool_set_name: str) -> List[Dict[str, Any]]: + """ + ๆ นๆฎๅ็งฐ่Žทๅ–็‰นๅฎš็š„ๅทฅๅ…ท้›† + Get specific tool set by name + """ + tool_sets = { + "code_implementation": MCPToolDefinitions.get_code_implementation_tools(), + "code_evaluation": MCPToolDefinitions.get_code_evaluation_tools(), + } + + return tool_sets.get(tool_set_name, []) + + @staticmethod + def get_all_tools() -> List[Dict[str, Any]]: + """ + ่Žทๅ–ๆ‰€ๆœ‰ๅฏ็”จๅทฅๅ…ท + Get all available tools + """ + all_tools = [] + for tool_set_name in MCPToolDefinitions.get_available_tool_sets().keys(): + all_tools.extend(MCPToolDefinitions.get_tool_set(tool_set_name)) + return all_tools + + +# ไพฟๆท่ฎฟ้—ฎๅ‡ฝๆ•ฐ +def get_mcp_tools(tool_set: str = "code_implementation") -> List[Dict[str, Any]]: + """ + ไพฟๆทๅ‡ฝๆ•ฐ๏ผš่Žทๅ–MCPๅทฅๅ…ทๅฎšไน‰ + Convenience function: Get MCP tool definitions + + Args: + tool_set: ๅทฅๅ…ท้›†ๅ็งฐ (้ป˜่ฎค: "code_implementation") + + Returns: + ๅทฅๅ…ทๅฎšไน‰ๅˆ—่กจ + """ + return MCPToolDefinitions.get_tool_set(tool_set) diff --git a/DeepCode/deepcode.py b/DeepCode/deepcode.py new file mode 100755 index 00000000..9a300c54 --- /dev/null +++ b/DeepCode/deepcode.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +""" +DeepCode - AI Research Engine Launcher + +๐Ÿงฌ Next-Generation AI Research Automation Platform +โšก Transform research papers into working code automatically +""" + +import os +import sys +import subprocess +from pathlib import Path + + +def check_dependencies(): + """Check if necessary dependencies are installed""" + import importlib.util + + print("๐Ÿ” Checking dependencies...") + + missing_deps = [] + missing_system_deps = [] + + # Check Streamlit availability + if importlib.util.find_spec("streamlit") is not None: + print("โœ… Streamlit is installed") + else: + missing_deps.append("streamlit>=1.28.0") + + # Check PyYAML availability + if importlib.util.find_spec("yaml") is not None: + print("โœ… PyYAML is installed") + else: + missing_deps.append("pyyaml") + + # Check asyncio availability + if importlib.util.find_spec("asyncio") is not None: + print("โœ… Asyncio is available") + else: + missing_deps.append("asyncio") + + # Check PDF conversion dependencies + if importlib.util.find_spec("reportlab") is not None: + print("โœ… ReportLab is installed (for text-to-PDF conversion)") + else: + missing_deps.append("reportlab") + print("โš ๏ธ ReportLab not found (text files won't convert to PDF)") + + # Check LibreOffice for Office document conversion + try: + import subprocess + import platform + + subprocess_kwargs = { + "capture_output": True, + "text": True, + "timeout": 5, + } + + if platform.system() == "Windows": + subprocess_kwargs["creationflags"] = 0x08000000 # Hide console window + + # Try different LibreOffice commands + libreoffice_found = False + for cmd in ["libreoffice", "soffice"]: + try: + result = subprocess.run([cmd, "--version"], **subprocess_kwargs) + if result.returncode == 0: + print( + "โœ… LibreOffice is installed (for Office document conversion)" + ) + libreoffice_found = True + break + except ( + subprocess.CalledProcessError, + FileNotFoundError, + subprocess.TimeoutExpired, + ): + continue + + if not libreoffice_found: + missing_system_deps.append("LibreOffice") + print("โš ๏ธ LibreOffice not found (Office documents won't convert to PDF)") + + except Exception: + missing_system_deps.append("LibreOffice") + print("โš ๏ธ Could not check LibreOffice installation") + + # Display missing dependencies + if missing_deps or missing_system_deps: + print("\n๐Ÿ“‹ Dependency Status:") + + if missing_deps: + print("โŒ Missing Python dependencies:") + for dep in missing_deps: + print(f" - {dep}") + print(f"\nInstall with: pip install {' '.join(missing_deps)}") + + if missing_system_deps: + print("\nโš ๏ธ Missing system dependencies (optional for full functionality):") + for dep in missing_system_deps: + print(f" - {dep}") + print("\nInstall LibreOffice:") + print(" - Windows: Download from https://www.libreoffice.org/") + print(" - macOS: brew install --cask libreoffice") + print(" - Ubuntu/Debian: sudo apt-get install libreoffice") + + # Only fail if critical Python dependencies are missing + if missing_deps: + return False + else: + print("\nโœ… Core dependencies satisfied (optional dependencies missing)") + else: + print("โœ… All dependencies satisfied") + + return True + + +def cleanup_cache(): + """Clean up Python cache files""" + try: + print("๐Ÿงน Cleaning up cache files...") + # Clean up __pycache__ directories + os.system('find . -type d -name "__pycache__" -exec rm -r {} + 2>/dev/null') + # Clean up .pyc files + os.system('find . -name "*.pyc" -delete 2>/dev/null') + print("โœ… Cache cleanup completed") + except Exception as e: + print(f"โš ๏ธ Cache cleanup failed: {e}") + + +def print_banner(): + """Display startup banner""" + banner = """ +โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•— +โ•‘ โ•‘ +โ•‘ ๐Ÿงฌ DeepCode - AI Research Engine โ•‘ +โ•‘ โ•‘ +โ•‘ โšก NEURAL โ€ข AUTONOMOUS โ€ข REVOLUTIONARY โšก โ•‘ +โ•‘ โ•‘ +โ•‘ Transform research papers into working code โ•‘ +โ•‘ Next-generation AI automation platform โ•‘ +โ•‘ โ•‘ +โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +""" + print(banner) + + +def launch_paper_test(paper_name: str, fast_mode: bool = False): + """Launch paper testing mode""" + try: + print("\n๐Ÿงช Launching Paper Test Mode") + print(f"๐Ÿ“„ Paper: {paper_name}") + print(f"โšก Fast mode: {'enabled' if fast_mode else 'disabled'}") + print("=" * 60) + + # Run the test setup + setup_cmd = [sys.executable, "test_paper.py", paper_name] + if fast_mode: + setup_cmd.append("--fast") + + result = subprocess.run(setup_cmd, check=True) + + if result.returncode == 0: + print("\nโœ… Paper test setup completed successfully!") + print("๐Ÿ“ Files are ready in deepcode_lab/papers/") + print("\n๐Ÿ’ก Next steps:") + print(" 1. Install MCP dependencies: pip install -r requirements.txt") + print( + f" 2. Run full pipeline: python -m workflows.paper_test_engine --paper {paper_name}" + + (" --fast" if fast_mode else "") + ) + + except subprocess.CalledProcessError as e: + print(f"\nโŒ Paper test setup failed: {e}") + sys.exit(1) + except Exception as e: + print(f"\nโŒ Unexpected error: {e}") + sys.exit(1) + + +def main(): + """Main function""" + # Parse command line arguments + if len(sys.argv) > 1: + if sys.argv[1] == "test" and len(sys.argv) >= 3: + # Paper testing mode: python deepcode.py test rice [--fast] + paper_name = sys.argv[2] + fast_mode = "--fast" in sys.argv or "-f" in sys.argv + + print_banner() + launch_paper_test(paper_name, fast_mode) + return + elif sys.argv[1] in ["--help", "-h", "help"]: + print_banner() + print(""" +๐Ÿ”ง Usage: + python deepcode.py - Launch web interface + python deepcode.py test - Test paper reproduction + python deepcode.py test --fast - Test paper (fast mode) + +๐Ÿ“„ Examples: + python deepcode.py test rice - Test RICE paper reproduction + python deepcode.py test rice --fast - Test RICE paper (fast mode) + +๐Ÿ“ Available papers:""") + + # List available papers + papers_dir = "papers" + if os.path.exists(papers_dir): + for item in os.listdir(papers_dir): + item_path = os.path.join(papers_dir, item) + if os.path.isdir(item_path): + paper_md = os.path.join(item_path, "paper.md") + addendum_md = os.path.join(item_path, "addendum.md") + status = "โœ…" if os.path.exists(paper_md) else "โŒ" + addendum_status = "๐Ÿ“„" if os.path.exists(addendum_md) else "โž–" + print(f" {status} {item} {addendum_status}") + print( + "\n Legend: โœ… = paper.md exists, ๐Ÿ“„ = addendum.md exists, โž– = no addendum" + ) + return + + print_banner() + + # Check dependencies + if not check_dependencies(): + print("\n๐Ÿšจ Please install missing dependencies and try again.") + sys.exit(1) + + # Get current script directory + current_dir = Path(__file__).parent + streamlit_app_path = current_dir / "ui" / "streamlit_app.py" + + # Check if streamlit_app.py exists + if not streamlit_app_path.exists(): + print(f"โŒ UI application file not found: {streamlit_app_path}") + print("Please ensure the ui/streamlit_app.py file exists.") + sys.exit(1) + + print(f"\n๐Ÿ“ UI App location: {streamlit_app_path}") + print("๐ŸŒ Starting DeepCode web interface...") + print("๐Ÿš€ Launching on http://localhost:8501") + print("=" * 70) + print("๐Ÿ’ก Tip: Keep this terminal open while using the application") + print("๐Ÿ›‘ Press Ctrl+C to stop the server") + print("=" * 70) + + # Launch Streamlit application + try: + cmd = [ + sys.executable, + "-m", + "streamlit", + "run", + str(streamlit_app_path), + "--server.port", + "8503", + "--server.address", + "localhost", + "--browser.gatherUsageStats", + "false", + "--theme.base", + "dark", + "--theme.primaryColor", + "#3b82f6", + "--theme.backgroundColor", + "#0f1419", + "--theme.secondaryBackgroundColor", + "#1e293b", + ] + + subprocess.run(cmd, check=True) + + except subprocess.CalledProcessError as e: + print(f"\nโŒ Failed to start DeepCode: {e}") + print("Please check if Streamlit is properly installed.") + sys.exit(1) + except KeyboardInterrupt: + print("\n\n๐Ÿ›‘ DeepCode server stopped by user") + print("Thank you for using DeepCode! ๐Ÿงฌ") + except Exception as e: + print(f"\nโŒ Unexpected error: {e}") + print("Please check your Python environment and try again.") + sys.exit(1) + finally: + # Clean up cache files + cleanup_cache() + + +if __name__ == "__main__": + main() diff --git a/DeepCode/deepcode_lab/papers/1/1.md b/DeepCode/deepcode_lab/papers/1/1.md new file mode 100644 index 00000000..c3db5183 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/1.md @@ -0,0 +1,1283 @@ +# Extracted from 1.pdf + +*Total pages: 23* + +--- + +## Page 1 + +DeepCode: Open Agentic Coding +Zongwei Liโˆ— โˆ—Zhonghang Liโˆ—Zirui Guo Xubin Ren Chao Huangโ€  +The University of Hong Kong +{zongwei9888, bjdwh.zzh, larfii1010, xubinrencs, chaohuang75}@gmail.com +/gtbSource Code:https://github.com/HKUDS/DeepCode +Abstract +Recent advances in large language models (LLMs) have given rise to powerful +coding agents, making it possible for code assistants to evolve into code engineers. +However, existing methods still face significant challenges in achieving high-fidelity +document-to-codebase synthesisโ€”such as scientific papers to codeโ€”primarily due +to a fundamental conflict between information overload and the context bottlenecks +of LLMs. In this work, we introduce DeepCode, a fully autonomous framework +that fundamentally addresses this challenge through principled information-flow +management. By treating repository synthesis as a channel optimization problem, +DeepCode seamlessly orchestrates four information operations to maximize task- +relevant signals under finite context budgets: source compression via blueprint +distillation, structured indexing using stateful code memory, conditional knowl- +edge injection via retrieval-augmented generation, and closed-loop error correction. +Extensive evaluations on the PaperBench benchmark demonstrate that DeepCode +achieves state-of-the-art performance, decisively outperforming leading commer- +cial agents such as Cursor and Claude Code, and crucially, surpassing PhD-level +human experts from top institutes on key reproduction metrics. By systematically +transforming paper specifications into production-grade implementations compara- +ble to human expert quality, this work establishes new foundations for autonomous +scientific reproduction that can accelerate research evaluation and discovery. +โ‘  Human Expert (Top ML PhD) +100 +75 +50 +25 +0 +72.4%75.9% +Human Expert +DeepCodeโ‘ก Commercial Code Agents +100 +75 +50 +25 +0 +40.0%58.7%58.4%84.8% +Codex +Claude Code +Cursor +DeepCodeโ‘ข Scientific Code Agent +100 +75 +50 +25 +0 +51.1%73.5% +Paper Coder +DeepCodeโ‘ฃ LLM-Based Agents +100 +75 +50 +25 +0 +5.0%7.7%9.8%16.4%35.4%43.3%73.5% +Gemini 2-flash +GPT-4o +DeepSeek R1 +o3-mini +Claude 3.5o1 +DeepCode08/12/2025, 22:43 New_UI.html +๏ฌle:///Users/lizongwei/Desktop/new_draw/New_UI.html 1/1 +Figure 1: DeepCode main results. +1 Introduction +The rapid evolution of Large Language Models (LLMs) has initiated a profound shift in how software +is specified, implemented, and maintained [ 1,2]. AI-assisted coding tools such as Cursor and Codex +โˆ—Equal contribution. +โ€ Chao Huang is the Corresponding Author. +Preprint. Under review.arXiv:2512.07921v1 [cs.SE] 8 Dec 2025 + +## Page 2 + +1. The Aspiration: +Document -to-Repository +AI Agents Human Experts~42%~72%Paper +Replication scoreThe Reality: +A Major Performance Gap2. The Core Obstacle: +Info Overload vs. Context Bottleneck +Information +Overloa d +LLM Context +Bottleneck +1. Specification +Preservation2. Global +Consistenc y3. Underspecified +Desig n4. Executable +Faithfulnes sThis leads to four key failures3. The Solution: DeepCode's +Information -Flow Management +Result: Surpasses Human Expert +Cursor Human Experts DeepCode +Four Information Operations +1. Source +Compression2. Structured +Indexing3. Knowledge +Injection4. Error +Correction +Blueprint CodeMem CodeRAG Verification +Maximizing relevant signalsFigure 2: From Challenge to Solution of DeepCode. Left: Current AI agents achieve only a 42% +paper replication score compared to 72% for human experts, highlighting the limitations of existing +agents. Middle: The core challenge stems from information overload conflicting with LLM context +limits, causing four key failure modes. Right: DeepCode addresses this through four information +operations (Blueprint, CodeMem, CodeRAG, Verification), surpassing human expert performance. +have already transformed everyday development practice by automating routine implementation tasks +and offering intelligent inline suggestions [ 3,4]. Yet these systems remain fundamentally assistive: +they operate at the level of code completion, assuming that a human engineer still performs the higher- +level tasks of understanding specifications, planning system architecture, and validating behavior. +Recent advances in agentic LLM frameworks point toward a more ambitious paradigmโ€”what we +termagentic software engineeringโ€”in which LLM-based agents are expected to plan, orchestrate, and +refine entire software projects from high-level natural language or document-level specifications [ 5,6]. +In this emerging regime, programming shifts fromwriting codetowriting specifications, and the +central question becomes:can an artificial coding agent behave as an autonomous engineer that +translates rich, informal specifications into comprehensive, robust systems? +A natural and stringent testbed for this paradigm ishigh-fidelity, document-grounded program +synthesis, where a complex scientific paper serves as the sole specification and the goal is to +produce a fully executable implementation that faithfully reflects it. Such papers are detailed +multimodal specifications, combining informal exposition with equations, pseudo-code, and scattered +hyperparameters. In this work, we tackle the highly challenging task of reproducing machine +learning papers as complete code repositories. Recent efforts have explored this via LLM-based +agents. PaperBench evaluates frontier models on 20 ICML papers, finding the strongest model (o1) +with IterativeAgent achieves only 42.4% replication score, far below 72.4% for human experts [ 7]. +PaperCoder employs a multi-agent pipeline spanning planning, analysis, and generation, reaching +51.14% reproduction rate on PaperBench [ 8]. These modest results reveal that current approaches fall +well short of reliable, end-to-end replication. We identify four key challenges that underlie this gap: +(i) Specification Preservation.Papers describe the target system through scattered, multimodal +constraints. Preserving a faithful mapping from this fragmented specification to implementation is +inherently difficult.(ii) Global Consistency under Partial Views.Repositories comprise interde- +pendent modules, but generation proceeds file-by-file under limited context. Maintaining consistency +across interfaces, types, and invariants under finite context windows easily leads to broken abstrac- +tions.(iii) Completion of Underspecified Designs.Papers specify only algorithmic cores, leaving +implementation details and experimental frameworks implicit. Inferring these consequential but +underspecified choices is non-trivial.(iv) Executable Faithfulness.Faithful reproduction requires +executable systems, not just plausible code. Long-horizon generation often yields repositories with +subtle logic bugs, dependency conflicts, and fragile pipelines that prevent end-to-end execution. +We argue that fundamentally addressing these challenges requiresprincipled information-flow man- +agement. We abstract the synthesis process as the transmission of a high-entropy specificationโ€”the +scientific paperโ€”through a sequence of bandwidth-constrained channels, defined by the LLMโ€™s +2 + +## Page 3 + +context windows. Naive strategies that simply concatenate raw documents with growing code history +induce channel saturation, where redundant tokens mask critical algorithmic constraints, causing +the effective Signal-to-Noise Ratio to collapse. Consequently, valid repository generation requires a +paradigm shift governed bycontextual information maximization: at each generation step, the system +must actively maximize the density of task-relevant signals while suppressing irrelevant noise. +Motivated by this perspective, we introduceDeepCode, an open agentic coding framework that +fundamentally reimagines repository-level synthesis as a problem ofhierarchical information-flow +management. Rather than treating synthesis as a monolithic process, DeepCode systematically ad- +dresses the doc-to-repos challenges by instantiating the proposed paradigm through four orchestrated +information operations: (1)source compression, which distills unstructured multi-modal specifica- +tions into a precise structural blueprint to maximize signal density; (2)structured indexing, which +abstracts the evolving repository state into concise memory entries to maintain global consistency +without context saturation; (3)conditional knowledge injection, which leverages retrieval-augmented +generation to bridge implicit specification gaps with standard implementation patterns; and (4)error +correction, which utilizes closed-loop verification to transform execution feedback into corrective +signals for rectifying transmission errors. Our contributions are threefold: +โ€ขWe characterize the task of high-fidelity document-to-repository synthesis through an information- +theoretic lens, identifying the central conflict as aninformation-overload vs. context-bottleneck +conflict. From this perspective, we propose an information-theoretic design principle: effective +agentic coding systems must explicitly structure, route, and compress information to maximize +task-relevant signal under finite context budgets. +โ€ขWe instantiate this principle in DeepCode, a systematic framework that orchestrates four strategic +information operations: blueprint distillation, stateful memory management, conditional knowledge +injection, and closed-loop verification. By dynamically optimizing the signal-to-noise ratio within +the context window, DeepCode effectively resolves the challenges of long-range specification +preservation, cross-file consistency, and implicit knowledge gaps in complex generation tasks. +โ€ขExtensive evaluations on the PaperBench benchmark demonstrate that DeepCode achieves state-of- +the-art performance, decisivelyoutperforming leading commercial agents(e.g. Cursor, Claude +Code, Codex) and, notably,surpassing human expert performanceon key reproduction metrics. +Furthermore, our analysis reveals that principled information-flow management yields significantly +larger performance gains than merely scaling model size or context length, offering a pivotal +direction for the future design of autonomous software engineers. +2 Preliminary +2.1 Task Definition +The primary objective of this work is to develop a system forhigh-fidelity program synthesis. We +formalize this as the process of learning a mapping function, Fgen, which transforms a specification +document,D, into a complete and executable code repository,P. The core function is defined as: +Fgen:Dโ†’P(1) +where Drepresents the space of specification documents and Prepresents the space of valid code +repositories. Such that for a given input document D โˆˆD , the output is a program repository +P=F gen(D). We address two primary manifestations of this task: +โ€ขScientific Paper Reproduction:Given a scientific paper from domains such as machine learning +or computer sciences as the source document D, the system should generate the full source code P +required to replicate the paperโ€™s key experiments and results. +โ€ขSoftware System Generation:Given a comprehensive technical design document or a concise +natural language requirement for a software application (e.g., specifying UI, backend APIs, and +database schema) as D, the system should generate the corresponding multi-component software +repositoryP, including frontend, backend, and configuration files. +Input: Source Document D.The source document Dis represented as a sequence of multi-modal +elements, D= (d 1, d2, . . . , d L), where each element dican be a block of text, a mathematical +3 + +## Page 4 + +equation, a table, a figure, or a snippet of pseudocode. The length Lof this sequence is typically +large, posing significant challenges for models with finite context windows. +Output: Code Repository P.The target output Pis not a single file but a structured repository. We +define it as a tuple: +P= (T,C,M)(2) +Here,Trepresents the directory structure that organizes the files in C.C={c 1, c2, . . . , c N}is a +set of Nsource code files. The generation of a coherent set Cwhere files correctly interact (e.g., +via imports and function calls) is a non-trivial problem of ensuring cross-file consistency. Mis +the dependency manifest (e.g. requirements.txt ,package.json ,README.md file) specifying all +external libraries required to run the code. +2.2 Objectives +An ideal synthesis function Fgenmust generate a repository Pโˆ—that optimizes a composite scoring +function. Under our paradigm ofprincipled information-flow management, this optimization is +framed as maximizing the effective signal-to-noise ratio across the synthesis channel. The optimal +output is defined as: +Pโˆ—= arg max +PโˆˆPScore(P|D)(3) +To overcome the conflict between information overload and finite context bandwidth, the scoring +function decomposes into four distinct objectives, each corresponding to an information operation: +โ€ขSpecification Preservation:The repository must faithfully implement the rigid algorithmic +constraints hidden within the multimodal source document. The objective is to maximize signal +density by extracting precise blueprints from the unstructured input noise. +โ€ขGlobal Structural Consistency:The generated modules must maintain strict interface compatibil- +ity and type coherence. The objective is to maintain state consistency without context saturation, +achieved by indexing the evolving codebase into compact, retrievable summaries. +โ€ขDomain Knowledge Grounding:The system must bridge the gap between abstract academic +descriptions and concrete engineering implementations. The objective is to resolve underspecified +designs by conditionally injecting standard libraries and patterns from external knowledge bases. +โ€ขFunctional Executability:The final repository must be robust and runnable. The objective is to +minimize transmission errors (bugs) by treating runtime execution feedback as a corrective signal +to iteratively refine the generated code. +Our framework is designed to satisfy these objectives by explicitly routing and compressing informa- +tion, enabling high-fidelity repository generation under strict context window constraints. +3 The DeepCode Framework +We introduce DeepCode, a multi-stage framework designed to instantiate the principle of principled +information-flow management for repository-level synthesis. To solve the optimization problem, +DeepCode decomposes the generation process into three orchestrated phases, each serving a distinct +information-processing role to maximize the effective signal-to-noise ratio. The process initiates +with(1) Blueprint Generation, where a planning agent acts as a source compression mechanism, +distilling the high-entropy source document Dinto a structured, high-signal implementation blueprint +to extract critical constraints while filtering narrative noise. Guided by this blueprint, the subsequent +(2) Code Generationphase synthesizes source files while preventing channel saturation through +two integrated mechanisms: a stateful Code Memory (CodeMem) that performs structured indexing +of the evolving codebase to maintain cross-file consistency, and a CodeRAG system that performs +conditional knowledge injection to bridge implicit domain gaps with standard implementation patterns. +Finally, the framework concludes with(3) Automated Verification, a closed-loop error correction +phase where a validation agent treats runtime execution feedback as corrective signals to identify and +rectify transmission errors, ensuring the functional correctness of the final output. +4 + +## Page 5 + +Phase 3: Automated Verification andRefinement +Static Analysis and Code Quality Refinement +Analysis Agent +Static Analysis of Code Issues +Modification Agent +Line-level Modifications inspired by LSP +Sandbox Execution and Functional Correction +Sandbox + Refinement +Test data +Execution traceAnalyzing trace +LSP-based +refinementOptimal Code +Repository +โ€ฆ +Source +DocumentsHierarchical Content Segmentation +Structural Parsing +Keyword -chunk Pairs +โ€ฆ +Concept Agent Algorithm Agent +Conceptual Analysis Schema Algorithmic Implementation Schema +Planning Agent +Phase 1: Blueprint GenerationCoding Blueprint +Project File Hierarchy Component Specification +Verification Protocol Execution EnvironmentStaged +Dev. +PlanCode Files +Phase 2: Code GenerationLLMs +GenerateIterative Code Generation +CodeRAG CodeMemAdditional +Resources +Retrieved +Knowledge +Target +Code FileMemory +Context +Memory +SummarizationUpdate New memory entry +Next IterationPick from +Blueprint +Paper +Docs +Passage +Figure 3: The overall framework of DeepCode. +3.1 Phase 1: Blueprint Generation +The primary goal of the first phase is to perform source compression: distilling the unstructured, +lengthy content of a source document (e.g. a scientific paper) into a structured, machine-readable +implementation blueprint. This distillation process directly mitigates the challenges of information +overload by transforming the raw input Dinto a high-density signal format. The process begins with +a crucial preprocessing step: hierarchical content segmentation. +3.1.1 Hierarchical Content Segmentation +Instead of feeding the entire document Dinto an LLM, we first parse it into a structured representation +that facilitates targeted information access. We introduce ahierarchical content index, which +leverages the inherent structure of academic papers and technical documents. The process is: +1.Structural Parsing:The source document Dis parsed to identify its hierarchical structure based +on explicit delimiters like section and subsection titles (e.g. "3. Methodology", "3.1. Model +Architecture"). This divides the document into a set of content chunksS={s 1, s2, . . . , s K}. +2.Keyword-Chunk Association:Each chunk skis stored as a key-value pair (hk, ck), where the +heading hkserves as a natural, high-level semantic keyword, and ckis the corresponding raw text +content of that section. +This indexed structure effectively transforms the problem from one of long-context comprehension +to a series of more manageable, on-demand retrievals. An agent no longer needs to process the +entire document at once. Instead, it can query the index using semantic keywords (e.g. requesting the +content associated with "Model Architecture") to fetch only the most relevant context for its current +task. This approach drastically reduces the token load for any single operation and allows the model +to focus its limited context window on the most pertinent information, thereby solving the problem of +context overload and information forgetting. This structured representation serves as the foundational +input for the specialized agents that perform the detailed analysis in the subsequent steps. +3.1.2 Multi-Agent Specification Analysis +Following the hierarchical segmentation, we employ a specialized multi-agent system to conduct +a deep and structured analysis of the documentโ€™s content. This approach decomposes the complex +comprehension task into two parallel tracks, executed by aConcept Agentand anAlgorithm Agent. +Each agent is equipped with a specific prompt and interacts with the indexed document to extract +complementary layers of information, ensuring a comprehensive understanding without processing +the entire document simultaneously. +5 + +## Page 6 + +Concept Agent: High-Level Structural and Conceptual Mapping.The Concept Agent is tasked +with building a holistic, high-level understanding of the document. Its primary objective is to map the +paperโ€™s entire conceptual structure, identify its core scientific contributions, and outline the necessary +components for a successful experimental reproduction. Operating on the indexed document, the +agent is instructed to use a segmented reading strategy, querying the index with semantically broad +keywords (e.g. โ€œintroductionโ€, โ€œmethodโ€). This allows it to assemble a comprehensive overview by +strategically fetching relevant sections. The output of this agent is a structuredConceptual Analysis +Schema. This schema comprises a detailed paper structure map, a method decomposition map +outlining the systemโ€™s core functional components, an implementation map aligning claims with code +requirements, and a reproduction roadmap specifying the criteria for success. Collectively, these +elements translate the paperโ€™s narrative into a structured project plan. +Algorithm Agent: Low-Level Technical Detail Extraction.Complementing the conceptual +overview, the Algorithm Agent is responsible for the meticulous extraction of every low-level +technical detail required for an exact implementation. Itโ€™s designed to perform an exhaustive search +for all algorithms, mathematical formulations, model architectures, training procedures, and hy- +perparameters. Moreover, it can leverage online search capabilities to retrieve relevant algorithm +implementations from the web as references. Like the Concept Agent, it leverages the segmented read- +ing strategy but uses a distinct set of highly specific keywords (e.g. โ€œalgorithmโ€, โ€œhyperparameterโ€) to +perform targeted queries on the most technically dense sections of the document. The agentโ€™s output +is a granularAlgorithmic Implementation Schema. This schema captures verbatim pseudocode from +algorithm boxes, exact mathematical equations and their variables, detailed layer-by-layer network +architectures, and a comprehensive list of all hyperparameters with references to their locations in the +paper. This schema serves as a precise, unambiguous technical specification, designed to leave no +detail to interpretation during the code generation phase. +3.1.3 Synthesizing the Implementation Blueprint +The analytical outputs from the Concept and Algorithm agents are then synthesized by theCode +Planning Agentinto a single, holistic implementation blueprint. This agentโ€™s critical function +is to orchestrate the high-level conceptual framework with the low-level technical specifications, +performing a final disambiguation and grounding step. It reconciles the architectural overview with +the granular implementation details, ensuring that every abstract component is directly linked to a +precise technical specification. Should any inconsistencies arise, the agent is authorized to perform +targeted queries on the indexed document to resolve them. The finalImplementation Blueprint Bis +a structured intermediate representation designed to be a self-contained, unambiguous specification +for code generation. This blueprint is organized into the following canonical sections: +โ€ขProject File Hierarchy:A prioritized project file structure that dictates the logical organization of +the codebase and the implementation order of its modules. +โ€ขComponent Specification:A granular specification for every module, class, and function, explic- +itly mapping each to its corresponding algorithmic pseudocode and mathematical formulation. +โ€ขVerification Protocol:A formal plan for validating the final implementation. It defines the +experimental setup, specifies the target metrics from the source document, and establishes the +success criteria for reproduction. +โ€ขExecution Environment:A complete specification of all software dependencies, library versions, +and requisite hardware configurations needed to compile and run the code. +โ€ขStaged Development Plan:A phased implementation roadmap that defines the build order of +components and integrates staged verification checks to ensure modular correctness. +By consolidating all distilled information into this canonical blueprint, the Code Planning Agent +concludes the specification distillation phase. This artifact serves as the definitive "source of truth" for +the subsequent code generation phase, effectively resolving the long-context challenge by providing +a dense, structured, and actionable input that obviates any need for the coding agents to interact with +the original, lengthy document. +3.2 Phase 2: Code Generation +Upon generating the high-signal blueprint, the second phase synthesizes the code repository. This +phase maximizes the density of relevant context while preventing channel saturation caused by the +6 + +## Page 7 + +accumulation of raw code history. A naive iterative approach, which appends previously generated +code to the prompt, leads to a collapse in the signal-to-noise ratio and induces hallucinations. To +overcome this, we propose a dual-mechanism strategy for efficient information routing: (1) a stateful +CodeMemthat performs structured indexing of the evolving repository to maintain internal structural +cohesion without context bloat, and (2) aCodeRAGsystem that performs conditional knowledge +injection, grounding the implementation in external patterns to bridge implicit knowledge gaps. +3.2.1 Stateful Generation with CodeMem +The core of our generation process is the Code Memory mechanism, a strategy designed to maintain a +compressed, structured representation of the repositoryโ€™s state, thereby ensuring cross-file consistency +without suffering from prohibitive context lengths. Instead of passing the full source code of +previously implemented files to the generative agent, we iteratively build and query a structured +memory bank,M. +Let the set of all files to be implemented, as defined by Sec. 2, be C={c 1, c2, . . . , c N}. The +generation process is an iterative loop over t= 1, . . . , N . At each step t, we maintain the set of +implemented files, Ctโˆ’1, and the set of unimplemented files, Utโˆ’1. The process for generating the +target file for the current step,ห†c t, is as follows: +1.Context Formulation.The generation context for the current step, Xt, is constructed not from +raw source code, but from the static implementation blueprint Band a dynamically selected subset +of the Code Memory, Mtโˆ’1. The agent first identifies which previously implemented files are +relevant to the current target file ห†ct(where ห†ctdenotes the blank code file to be generated, and ct +denotes the resulting generated code file). It then retrieves only their corresponding summaries +from the memory bank: +Xt= (B,SelectRelevantMemory(M tโˆ’1,ห†ct))(4) +where SelectRelevantMemory is a function that queries Mtโˆ’1to fetch only the essential sum- +maries of dependencies. +2.Code Generation.The coding agent, represented by the LLM function L, synthesizes the source +code for the target file based on the curated context: +ct=L(X t)(5) +3.Memory Update.After generating the code ct, the system clears the generation context. A +specialized summarization agent, S, is then invoked. This agent analyzes the newly generated +source code ctto extract its structural essence and create a new memory entry, mt. The Code +Memory is then updated: +Mt=M tโˆ’1โˆช {m t}(6) +The summarization agent Sdistills the code into a structured format that captures all information +necessary for inter-module communication. Each memory entry mtis a structured object containing: +โ€ขCore Purpose ( Pt):A concise, natural language summary of the fileโ€™s primary responsibility and +role within the repository. +โ€ขPublic Interface ( It):A formal description of all externally accessible classes, functions, and +constants, including their signatures and purposes (e.g., Class(params): methods). +โ€ขDependency Edges ( Et):A comprehensive map of the fileโ€™s position within the projectโ€™s depen- +dency graph. This structured entry specifies bothafferent couplings(internal dependencies), +detailing the specific imports from other project modules and external packages, and predictedef- +ferent couplings(external dependencies), identifying which unimplemented modules are expected +to consume this fileโ€™s public interface. +โ€ขNext Implementation Target ( ห†ct+1):A decision on the next file to be implemented, based on the +blueprint, dependency graph and the current state. Note that, to avoid introducing noise into the +memory, this information is separated fromm tand provided independently as part ofLinput. +This mechanism effectively decouples the context size from the repository size. The context provided +to the agent at any step tremains compact, containing only the high-level blueprint and the highly +compressed summaries of relevant, already-implemented files. This stateful, summary-based ap- +proach allows our system to maintain global consistency and logical cohesion across a large number +of files, directly solving the long-context and cross-file consistency challenges. +7 + +## Page 8 + +3.2.2 Knowledge Grounding with CodeRAG +While the Code Memory mechanism ensures internal consistency, it does not address the challenges +of model hallucination or the omission of implicit domain knowledge. To mitigate these issues, we +introduce a retrieval-augmented generation framework,CodeRAG, which grounds the synthesis +process in a pre-indexed corpus of relevant, high-quality code repositories. This process is divided +into two stages: an indexing phase and an adaptive retrieval phase during code generation. +Repository Indexing.The goal of this phase is to analyze a set of relevant source code repositories, +R={R 1, R2, . . . , R K}, and build a structured, queryable index, J. The process, modeled by +Iindex:R ร— B โ†’ J, consists of the following steps: +1.Relevance Filtering:For each repository Rkโˆˆ R, we perform an initial LLM-based filtering +to identify a subset of source files, Cโ€ฒ +kโŠ‚R k, that are most relevant to the target project structure +defined in the implementation blueprint B. In this context, Rcan denote either the corresponding +repository cited in the references of the target paper or other relevant repositories identified through +online search. This focuses computational resources on the most promising assets. +2.Code Understanding:Each relevant source file cโ€ฒ +sโˆˆ Cโ€ฒ +kis independently analyzed to create +a structured summary, analogous to the memory entries described previously. This summary +captures the fileโ€™s purpose, key concepts, and public interfaces. +3.Relationship Mapping:The core of the indexing process is to establish explicit links between +the analyzed source files and the target files in our blueprint. For each source file summary, an +agent maps it to one or more target files inB, generating a set of relationship tuples. +The final output index Jis a structured knowledge base containing a collection of relationship +tuples. Each tuple is defined as (cโ€ฒ +s,ห†ct, ฯ„, ฯƒ, ฮณ) . Here, cโ€ฒ +sis a file in the source repository and ห†ctis +the corresponding target file in the blueprintโ€™s structure. ฯ„denotes the relationship type, indicating +the nature of the potential contribution, while ฯƒis a confidence score representing the strength of +the mapping. ฮณis a set of actionable context, such as helpful code snippets, usage suggestions, and +implementation patterns. +Adaptive Retrieval.During the iterative code generation phase, our framework will optionally query +the CodeRAG index Jto augment its context. At each generation step tfor a target file ห†ct, the agent +makes an adaptive decision on whether to retrieve external knowledge. This decision is modeled by a +binary functionฮด: +rt=ฮด(X t,ห†ct)(7) +where flag rtโˆˆ {0,1} andXtis the standard context containing the blueprint and relevant code +memory. The decision is based on the complexity of the target file and the level of detail available in +the blueprint. If rt= 1, the agent queries the index Jto find the most relevant relationship tuples for +ห†ct. The retrieved context ฮณfrom the highest-confidence relationship is used to create an augmented +context,Xโ€ฒ +t: +Xโ€ฒ +t=Xtโˆช {Retrieve(J,ห†c t)}(8) +The final code is then generated using this enriched context: ct=L(Xโ€ฒ +t). By dynamically incorpo- +rating proven implementation patterns from existing repositories, CodeRAG significantly reduces +the likelihood of generating erroneous or suboptimal code, thus bridging the knowledge gap for the +generative agent. +3.3 Phase 3: Automated Verification and Refinement +The final phase serves as an error correction mechanism to ensure the functional faithfulness of the +synthesized repository P. Recognizing that purely generative processes are prone to transmission +errorsโ€”manifesting as logic bugs, invalid dependencies, or dead codeโ€”this phase establishes a +crucial closed-loop feedback system absent in standard models. By treating execution outcomes as +corrective signals, the framework systematically identifies and rectifies defects through two sequential +stages: (1) a static analysis pass to ensure structural integrity and code quality, and (2) a dynamic +execution pass within a sandboxed environment to enforce functional correctness. +3.3.1 Static Analysis and Code Quality Refinement +The first stage addresses issues that can be detected without executing the code. This process is +orchestrated by a dedicated Analysis Agent and a Modification Agent. +8 + +## Page 9 + +Static Analysis.An Analysis Agent, denoted by the function Astatic, inspects the generated repository +Pagainst the implementation blueprint B. It produces a structured static analysis report, Rstatic, +which identifies a set of issues. This process can be formalized as:R static=A static(P,B). +The identified issues I={i 1, i2, . . . , i K}fall into two categories: i)Structural Discrepancies:This +includes integrity violations such as missing files specified in the blueprint or empty (zero-byte) +source files that were not correctly generated. ii)Code Quality Deficiencies:The agent leverages +an LLM to perform a quality assessment of each source file, assigning a quality score, q(ci), and +flagging sections with poor style, complexity, or maintainability. +Code Refinement.The report Rstaticis then passed to a Modification Agent, Amodify . This agent iter- +ates through each issue ikโˆˆIand applies a targeted fix. To perform precise, line-level modifications +without rewriting entire files, the agent utilizes a programmatic interface inspired by the Language +Server Protocol (LSP). We model this refinement operation as a function ฮฆLSPthat takes a file ciand +a modification instruction from the report, producing a corrected file cโ€ฒ +i. The overall process yields a +statically refined repositoryPโ€ฒas:Pโ€ฒ=A modify(P,R static). +3.3.2 Sandbox Execution and Functional Correction +After static refinement, the repository Pโ€ฒundergoes dynamic testing in a secure, isolated sandbox +environment to ensure it runs as intended. +Environment Verification and Setup.A Sandbox Agent, Asandbox , first validates the environment +setup instructions (e.g., in README.md ) against the dependencies specified in the blueprint B. Any +discrepancies are corrected. The agent then automatically provisions the specified environment and +installs all dependencies. +Iterative Execution and Correction.The agent then attempts to execute the main entry points of +the repository, using automatically generated test data and test files designed to exercise the core +algorithms and functions. The execution process, Esandbox , takes the repository Pโ€ฒ +jat iteration j +(initiallyPโ€ฒ +0=Pโ€ฒ) and produces an execution trace,T j, containing all outputs and error messages. +Tj=E sandbox (Pโ€ฒ +j)(9) +This initiates an iterative refinement loop. If the trace Tjcontains errors ( Terror +jฬธ=โˆ…), the Sandbox +Agent analyzes the error messages to identify the likely faulty files and the nature of the bug. It then +generates a modification instruction and invokes the LSP-based refinement function ฮฆLSPto patch the +code, producing the repository for the next iteration, Pโ€ฒ +j+1. This loop continues until the execution is +successful or a maximum number of iterations is reached. +Pโ€ฒ +j+1= ฮฆ LSP(Pโ€ฒ +j,Terror +j)(10) +The final verified output of our entire framework is the repository Pโˆ—=Pโ€ฒ +J, where Jis the terminal +iteration of the refinement loop. This multi-stage verification and correction process ensures that the +synthesized code is not only structurally sound but also functionally correct and conformant to the +original specification. +4 Experiments +In this section, we evaluate the effectiveness of the proposed DeepCode framework by addressing +the following 3 research questions:RQ1:How does DeepCode perform compared to existing agent +frameworks?RQ2:How does the choice of different LLMs affect the performance of DeepCode? +RQ3:What is the contribution of each module within the DeepCode architecture? +4.1 Experiments Settings +Datasets.To evaluate DeepCodeโ€™s capabilities in code comprehension and generation, particularly +for automated vulnerability detection, we employPaperBench Code-Dev, an innovative benchmark +created by OpenAI [ 7]. PaperBench Code-Dev assesses AI modelsโ€™ ability to independently reproduce +leading ML research from major conferences like ICML 2024, focusing on 20 significant papers. +Models are required to generate all necessary code from scratch, using only the research papers as +references, without accessing existing codebases from the original authors. These tasks are performed +9 + +## Page 10 + +in a virtual machine environment, with the goal of building a functional codebase, replicating +experiments, and creating a reproduce.sh script for execution. Each paper is accompanied by a +detailed evaluation rubric approved by the authors, which breaks down the reproduction task into 8,316 +specific, gradable components, meticulously assessed using a hierarchical weighting scheme and +SimpleJudge, a sophisticated automated judge powered by OpenAIโ€™s o3-mini model. This benchmark +is rigorously crafted to challenge AI with tasks requiring advanced natural language understanding, +algorithmic reasoning, and the ability to generate reliable code from abstract descriptions, all of +which are crucial skills for automating vulnerability detection effectively. +Baselines.In order to evaluate the effectiveness of the proposed framework, we include a range of +baseline methods for comparison. These baselines fall into four distinct categories: +(1) LLM Agents.We compare against results reported in [ 7] for several state-of-the-art language +models using two agent scaffolding approaches: (1)BasicAgent, a simple tool-use loop based on +Inspect AIโ€™s basic agent that allows models to terminate early, and (2)IterativeAgent, which forces +models to use their full allocated time and employs prompts designed to encourage incremental, +piecemeal progress. All agents run in Ubuntu 24.04 Docker containers with access to a single A10 +GPU, the internet, and standard development tools including bash, Python, web browsing, and file +reading capabilities [ 7]. The baseline models include GPT-4o, o1, o3-mini, DeepSeek-R1, Claude +3.5 Sonnet, and Gemini 2.0 Flash, with most experiments using a 12-hour time limit (extended to 36 +hours for select o1 runs). +(2) Scientific Code Agents.PaperCoder[ 8]. PaperCoder (also referred to as Paper2Code) is a multi- +agent LLM framework that transforms machine learning papers into executable code repositories via +a three-stage pipeline: planning, which constructs implementation roadmaps, system architecture +diagrams, and file dependencies; analysis, which extracts file-level implementation details; and +generation, which produces modular code in dependency order. +(3) Commercial Code Agents.We compare against three state-of-the-art commercial code agents +that provide AI-powered development assistance through different interfaces and capabilities: +โ€ขCursor(Version 1.7.52) is an AI-assisted integrated development environment built as a fork of +Visual Studio Code with additional AI features. Cursor allows developers to choose between +cutting-edge LLMs and provides codebase embedding models that give agents deep understanding +and recall [ 9]. In our experiments, Cursor uses Claude Sonnet 4.5-thinking as the underlying model. +โ€ขClaude Code(Version 2.0.22) is Anthropicโ€™s agentic coding tool that lives in the terminal and +helps developers turn ideas into code. Claude Code maintains awareness of the entire project +structure, can find up-to-date information from the web, and with MCP can pull from external +data sources like Google Drive, Figma, and Slack. It can directly edit files, run commands, create +commits, and use MCP to read design docs or update tickets [ 10]. Our evaluation uses Claude +Sonnet 4.5-thinking. +โ€ขCodex(Version codex-cli 0.47.0) is OpenAIโ€™s coding agent that runs locally from the terminal +and can read, modify, and run code on the userโ€™s machine. Codex is optimized for use with +GPT-5-Codex for agentic coding, with configurable reasoning levels from medium to high for +complex tasks. In auto approval mode, Codex can read files, make edits, and run commands in the +working directory automatically [11]. We configure Codex with GPT-5 Codex-high. +(4) Human Experts.The human baseline [ 7] consists of 8 ML PhD students and graduates from top +institutions (e.g. Berkeley, Cambridge, Carnegie Mellon) who worked part-time over a four-week +window on a 3-paper subset (all-in-one, fre, stay-on-topic). Participants had similar computational +resources (A10 GPU) and could use AI coding assistants like ChatGPT and GitHub Copilot. The +best-of-3 human attempts (Best@3) represent expert-level performance on this subset. +Experimental Setup.To evaluate DeepCodeโ€™s efficacy in high-fidelity repository synthesis, we adopt +a rigorous framework under realistic constraints. The setup combines a secure execution environment +and the PaperBench protocol for fair, reproducible, detailed comparisons across baselines. +(1) Implementation Environment.All experiments are conducted within an Ubuntu 22.04 LTS- +based sandboxed environment. This infrastructure is provisioned with a standard Python development +stack and essential dependencies. DeepCode is configured to operate within this isolated space, +retaining privileges for file system manipulation, shell command execution, and internet access, +thereby simulating a standard software research and development workflow. +10 + +## Page 11 + +(2) Task Execution.DeepCode accepts the target paper in both PDF and Markdown formats, along +with any supplementary addenda, as primary inputs. To ensure that generated solutions stem from +algorithmic reasoning rather than retrieval, a source code blacklist is enforced during execution. This +protocol precludes access to the authorsโ€™ original repositories and known third-party implementations +during web browsing. With input parameters defined and the search space constrained, DeepCode +initiates its autonomous workflow for code generation and debugging. +(3) Grading Methodology.Assessment of the generated code follows the PaperBench Code-Dev +protocol, which focuses on structural and functional correctness and does not include post-submission +reproduction. Grading is carried out by SimpleJudge, an automated system based on OpenAIโ€™s +o3-mini, which performs static analysis of the submitted repository against a set of fine-grained, +hierarchical criteria co-developed with the authors of the source paper. The judging logic is restricted +to the โ€œCode Developmentโ€ leaf nodes of this rubric and examines core aspects of software quality, +including static correctness (syntax validity and compliance with language standards), dependency +validity (completeness and correctness of dependency specifications such as requirements.txt ), +project structure (coherent and consistent organization of files and directories), and algorithmic +fidelity (faithful implementation of the algorithms and interfaces described in the original paper). +This procedure is designed to align the evaluation with the central technical contributions of the work. +(4) Evaluation Metrics and Protocol.Our primary evaluation metric is the Replication Score, which +quantifies the proficiency of DeepCode in translating theoretical concepts into a functional codebase. +The score for a single replication trial is derived from the hierarchical rubric through a bottom-up +aggregation process.(i) Leaf node scoring:SimpleJudge first evaluates each leaf node criterion +on a binary basis, assigning a score of 1 for โ€œpassโ€ (compliance) and 0 for โ€œfailโ€ (non-compliance). +(ii) Score aggregation:The score for any parent node is then computed as the weighted average of +the scores of its immediate children. The weights, predetermined during the rubric design, reflect +the relative importance of each sub-task.(iii) Final score derivation:This recursive aggregation +continues up the hierarchy until a single score is obtained for the root node, which serves as the +Replication Score for that trial. +To account for the stochasticity inherent in code generation, we adopt a strict evaluation protocol. For +each target paper, three independent replication trials are performed, and each resulting repository is +scored separately by SimpleJudge using the procedure described above. The final Replication Score +is the average of the three scores, mitigating outliers and providing a more stable and reliable measure +of the modelโ€™s typical performance. +4.2 Main Results +The primary results of our experiments are detailed in Figure 4. We analyze the performance +of DeepCode against the four established categories of baselines: general-purpose LLM agents, +specialized scientific code agents, commercial code agents, and human experts. +โ€ขComparison against LLM Agents.Figure 4 presents average replication scores across all +benchmark papers. Among general-purpose LLM agents, performance varies significantly by model +and scaffolding. With BasicAgent, Claude-3.5-Sonnet achieves the highest score (35.4 ยฑ0.8), while +other frontier models range from 5.0 to 19.5. IterativeAgent scaffolding improves some models, +with o1 reaching the best LLM agent performance of 43.3 ยฑ1.1. DeepCode achieves 73.5 ยฑ2.8, +/uni0000002b/uni00000058/uni00000050/uni00000044/uni00000051/uni00000003/uni00000028/uni0000005b/uni00000053/uni00000048/uni00000055/uni00000057/uni00000027/uni00000048/uni00000048/uni00000053/uni00000026/uni00000052/uni00000047/uni00000048/uni00000013/uni00000015/uni00000018/uni00000018/uni00000013/uni0000001a/uni00000018/uni00000014/uni00000013/uni00000013 +/uni0000001a/uni00000015/uni00000011/uni00000017/uni00000008/uni0000001a/uni00000019/uni00000011/uni0000001a/uni00000008/uni00000014/uni00000011/uni00000003/uni0000002b/uni00000058/uni00000050/uni00000044/uni00000051/uni00000003/uni00000028/uni0000005b/uni00000053/uni00000048/uni00000055/uni00000057/uni00000003/uni0000000b/uni00000037/uni00000052/uni00000053/uni00000003/uni00000030/uni0000002f/uni00000003/uni00000033/uni0000004b/uni00000027/uni0000000c +/uni00000026/uni00000052/uni00000047/uni00000048/uni0000005b +/uni00000026/uni0000004f/uni00000044/uni00000058/uni00000047/uni00000048/uni00000003/uni00000026/uni00000052/uni00000047/uni00000048/uni00000026/uni00000058/uni00000055/uni00000056/uni00000052/uni00000055 +/uni00000027/uni00000048/uni00000048/uni00000053/uni00000026/uni00000052/uni00000047/uni00000048/uni00000013/uni00000015/uni00000018/uni00000018/uni00000013/uni0000001a/uni00000018/uni00000014/uni00000013/uni00000013 +/uni00000017/uni00000013/uni00000011/uni00000013/uni00000008/uni00000018/uni0000001b/uni00000011/uni0000001a/uni00000008 /uni00000018/uni0000001b/uni00000011/uni00000017/uni00000008/uni0000001b/uni00000018/uni00000011/uni00000017/uni00000008/uni00000015/uni00000011/uni00000003/uni00000026/uni00000052/uni00000050/uni00000050/uni00000048/uni00000055/uni00000046/uni0000004c/uni00000044/uni0000004f/uni00000003/uni00000026/uni00000052/uni00000047/uni00000048/uni00000003/uni00000024/uni0000004a/uni00000048/uni00000051/uni00000057/uni00000056 +/uni00000033/uni00000044/uni00000053/uni00000048/uni00000055/uni00000003/uni00000026/uni00000052/uni00000047/uni00000048/uni00000055/uni00000027/uni00000048/uni00000048/uni00000053/uni00000026/uni00000052/uni00000047/uni00000048/uni00000013/uni00000015/uni00000018/uni00000018/uni00000013/uni0000001a/uni00000018/uni00000014/uni00000013/uni00000013 +/uni00000018/uni00000014/uni00000011/uni00000014/uni00000008/uni0000001a/uni00000016/uni00000011/uni00000019/uni00000008/uni00000016/uni00000011/uni00000003/uni00000036/uni00000046/uni0000004c/uni00000048/uni00000051/uni00000057/uni0000004c/uni00000049/uni0000004c/uni00000046/uni00000003/uni00000026/uni00000052/uni00000047/uni00000048/uni00000003/uni00000024/uni0000004a/uni00000048/uni00000051/uni00000057 +/uni0000002a/uni00000048/uni00000050/uni0000004c/uni00000051/uni0000004c/uni00000010/uni00000015/uni00000011/uni00000013/uni00000010/uni00000049/uni0000004f/uni00000044/uni00000056/uni0000004b/uni0000002a/uni00000033/uni00000037/uni00000010/uni00000017/uni00000052 +/uni00000027/uni00000048/uni00000048/uni00000053/uni00000036/uni00000048/uni00000048/uni0000004e/uni00000003/uni00000035/uni00000014/uni00000052/uni00000016/uni00000010/uni00000050/uni0000004c/uni00000051/uni0000004c +/uni00000026/uni0000004f/uni00000044/uni00000058/uni00000047/uni00000048/uni00000003/uni00000016/uni00000011/uni00000018/uni00000003/uni00000036/uni00000052/uni00000051/uni00000051/uni00000048/uni00000057/uni00000052/uni00000014 +/uni00000027/uni00000048/uni00000048/uni00000053/uni00000026/uni00000052/uni00000047/uni00000048/uni00000013/uni00000015/uni00000018/uni00000018/uni00000013/uni0000001a/uni00000018/uni00000014/uni00000013/uni00000013 +/uni00000018/uni00000011/uni00000013/uni00000008/uni0000001a/uni00000011/uni0000001a/uni00000008/uni0000001c/uni00000011/uni0000001b/uni00000008/uni00000014/uni00000019/uni00000011/uni00000017/uni00000008/uni00000016/uni00000018/uni00000011/uni00000017/uni00000008/uni00000017/uni00000016/uni00000011/uni00000016/uni00000008/uni0000001a/uni00000016/uni00000011/uni00000019/uni00000008/uni00000017/uni00000011/uni00000003/uni0000002f/uni0000002f/uni00000030/uni00000010/uni00000025/uni00000044/uni00000056/uni00000048/uni00000047/uni00000003/uni00000024/uni0000004a/uni00000048/uni00000051/uni00000057/uni00000056 +Figure 4: Comparison of DeepCode with four baseline categories: (1) human experts, (2) state-of- +the-art commercial code agents, (3) scientific code agents, and (4) LLM-based agents +11 + +## Page 12 + +representing a 70% relative improvement over the best LLM agent baseline. This substantial gap +demonstrates that our frameworkโ€™s specialized design, which incorporates systematic planning, +structured code generation and automated verification, provides significant advantages over general- +purpose agent scaffolding. +โ€ขComparison against Scientific Code Agents.PaperCoder, a specialized multi-agent framework +designed for transforming machine learning papers into executable code, achieves a score of +51.1ยฑ1.4, outperforming all LLM agents baselines. However, DeepCode achieves a significantly +higher score of 73.5 ยฑ2.8โ€”an improvement of over 22 points. This substantial gain suggests that +our approach to task decomposition, code generation, and repository-level integration is markedly +more effective than existing specialized methods. +โ€ขComparison against Commercial Code Agents.Table 1 details a direct comparison with +leading commercial agents on a 5-paper subset. DeepCode achieves an average score of 0.8482, +decisively outperforming Codex (0.3997), Cursor (0.5841), and Claude Code (0.5871). This result is +particularly noteworthy: DeepCode uses the same base model as both Cursor and Claude Code. The +dramatic performance difference provides strong evidence that our frameworkโ€™s performance gains +are not merely a product of a powerful base model. Rather, the advantage is directly attributable to +the superior agentic architecture, planning, and execution strategies of DeepCode. +โ€ขComparison against Human Experts.The most compelling finding is the comparison to human +expert performance. As shown in the final rows of Figure 4, we benchmarked performance on +the 3-paper subset. The human baseline, which represents the best-of-3 attempts from ML PhD +students, achieved a score of 72.4. Our DeepCodeโ€™s average performance on this same subset was +75.9ยฑ4.5, meaning it not only competes with but exceeds the score of the best attempt from a +human expert. This result strongly validates our approach, demonstrating its capability to automate +and even surpass expert-level performance on this highly challenging task. +Table 1: Reproduction scores of DeepCode and commercial code agents on 5-paper subset +Model fre rice bam pinn mech-u Avg. +Codex (GPT 5 Codex-high) 0.4095 0.3645 0.1937 0.5382 0.4926 0.3997 +Claude Code (Claude Sonnet 4.5-think) 0.6286 0.3787 0.3829 0.7233 0.8222 0.5871 +Cursor (Claude Sonnet 4.5-think) 0.6344 0.4186 0.3779 0.7748 0.7148 0.5841 +DeepCode(Claude Sonnet 4.5-think)0.8435 0.7380 0.8530 0.9474 0.8888 0.8541 +4.3 Analysis on Different LLMs +We evaluate DeepCode with five LLM backbones (Claude-4.5-Sonnet, GPT-5, Claude-3.5-Sonnet, +Gemini-2.5-Pro, DeepSeek-R1) on three PaperBench tasks (fre, all-in-one, stay-on-topic). The +tasks vary in specification complexity: fre and all-in-one contain long, interdependent setups with +overlapping constraints, while stay-on-topic provides more structured descriptions. Agent architecture +and tooling remain constant to isolate model capability effects. +As shown in Figure 5, reproduction scores exhibit consistent stratification across all three tasks. +Claude-4.5-Sonnet achieves the best or near-best performance (0.72-0.82), demonstrating particular +strength on fre and all-in-one where it more reliably reconstructs implementation details and multi- +stage pipelines implied by complex, underspecified descriptions. GPT-5 tracks Claude-4.5-Sonnet +closely on most metrics (0.69-0.81) and shows marginal advantages on stay-on-topic (0.81 vs. +0.72), suggesting additional robustness in maintaining alignment with fixed experimental framings, +though this does not overturn Claude-4.5-Sonnetโ€™s overall dominance. Mid-tier models occupy an +intermediate performance range: Claude-3.5-Sonnet (0.48-0.57) and Gemini-2.5-Pro (0.44-0.73) +successfully recover main experimental skeletons but leave notable gaps in finer-grained procedural +steps. DeepSeek-R1 consistently underperforms ( โ‰ˆ0.29), reproducing only fragments of target +workflows across all tasks. This stable ranking pattern across heterogeneous specifications indicates +that under fixed agent architecture, the underlying language model becomes the primary factor +determining the ceiling and reliability of automatic paper-level reproduction. +12 + +## Page 13 + +0.0 0.2 0.4 0.6 0.8 1.0DeepSeek-R1Gemini-2.5-proClaude-3.5-sonnetGPT-5Claude-4.5-sonnet +0.2930.7250.5200.7730.823fre +0.0 0.2 0.4 0.6 0.8 1.0 +Replication Score0.2870.4400.5700.6940.758all-in-one +0.0 0.2 0.4 0.6 0.8 1.00.2930.5250.4800.8120.720stay-on-topicDeepSeek-R1 Gemini-2.5-pro Claude-3.5-sonnet GPT-5 Claude-4.5-sonnetFigure 5: DeepCode reproduction results on the 3-paper subset across LLM backbones +4.4 Ablation Studies +In this section, we conduct ablation studies on three core components of DeepCode: CodeRAG, +CodeMem, and Automated Verification. Specifically, we evaluate CodeRAG and Automated Verifica- +tion on a 3-paper subset (all-in-one, fre, stay-on-topic), while CodeMem is assessed on 5 randomly +selected tasks (test-time-model-adaptation, rice, mechanistic-understanding, fre, all-in-one). Our key +findings are summarized as follows. +(1) Impact of CodeRAG.To decouple the impact of CodeRAG, we conducted an ablation study using +Gemini-2.5-Flash. As visualized in Figure 6a, the integration of CodeRAG delivers a substantial +performance leap (up to 70% relative gain), effectively breaking the base modelโ€™s performance ceiling +(0.35โ€“0.38). Notably, we observed negligible gains when applying CodeRAG to frontier models +like Claude 4.5 Sonnet. This contrast yields a critical insight: while reasoning giants likely encode +sufficient implementation patterns within their parameters, cost-efficient models like Flash suffer +from inherentknowledge gaps. Consequently, CodeRAG proves indispensable for these architectures, +acting as a vital bridge to fill implicit domain voids with standard practicesโ€”confirming that external +knowledge injection is essential for democratizing high-fidelity replication on lightweight models. +(2) Impact of CodeMem.We ablate CodeMemโ€™s contribution on five PaperBench tasks using +Claude-4.5-Sonnet, comparing DeepCodeโ€™s structured memory against a "Simple" baseline that +naively evicts historical messages via sliding windows when approaching context limits. +Results demonstrate that unstructured eviction causes context saturation with signal loss: the Simple +protocol achieves only 0.33-0.43 in rice, fre, and mechanistic-understanding tasks due to dependency +truncation, where foundational class definitions are discarded before dependent code generation. +CodeMemโ€™s structured indexing maintains task-relevant signal density, restoring scores to 0.70-0.92 +by preserving critical dependencies without exhausting context budgets. Even in scenarios with strong +baseline performance (test-time-model-adaptation: 0.62 โ†’0.72; all-in-one: 0.66 โ†’0.76), Structured +memory delivers consistent gains, confirming our core thesis: effective agentic coding requires +explicit information flow management to maximize signal-to-noise ratio under context constraints. +(3) Impact of Automated Verification.Across 3 test papers, Automated Verification yields consistent +gains of 3.7โ€“6.5%, elevating scores from 0.69โ€“0.81 to 0.73โ€“0.84. The layer primarily corrects three +types of residual errors: typos in variable names, missing dependencies, and wrong command-line +arguments. These errors prevent otherwise sound implementations from executing reliably. The +0.30 0.35 0.40 0.45 0.50 0.55 0.60 0.65 0.70 +Replication Score (CodeRAG)fre +all-in-one +stay-on-topic+68.8%0.380 0.642 ++41.0%0.354 0.499 ++71.3%0.360 0.616- CodeRAG +CodeRAG +- Verification +Verification0.650 0.675 0.700 0.725 0.750 0.775 0.800 0.825 0.850Replication Score (Verification) ++3.7%0.8136 0.8435 ++5.4%0.7193 0.7585 ++6.5%0.6895 0.7342 +(a) Ablation of CodeRAG and Verification +test-time-model +adaptationrice +mechanistic +understanding +fre +all-in-one0.20.40.60.81.0Simple +Code Memory (b) Ablation of CodeMem +Figure 6: Ablation studies of key components in DeepCode on PaperBench +13 + +## Page 14 + +modest improvement reflects an important fact: the earlier phases have already achieved technical cor- +rectness. Verification is a final pass to ensure reliable execution. It eliminates small but consequential +deviations that cause borderline implementations to fail, transforming them into faithful replications. +5 Related Work +5.1 General Coding Agents +The field of software engineering is being rapidly transformed by agentic systems that have evolved +from passive code assistants into autonomous entities capable of planning, executing multi-step +tasks, and self-correction [ 4,2]. Research has explored several key architectures for these agents. +One prominent trend involves multi-agent frameworks that emulate human development teams. +This includes systems like ChatDev [ 12], MetaGPT [ 13], and CodePoRi [ 14], which simulate entire +software company organizational structures to manage development tasks from scratch. For repo-level +code generation, CodeS [ 15] proposed to decompose repository generation into specialized agents +for structure planning and content filling. AgentCoder [ 16] employs atest-driven refinement loop +involving programmer, test designer, and test executor agents, while MapCoder [ 17] mirrors human +program synthesis with four agents handling example retrieval, planning, generation, and debugging. +A second major trend focuses on enhancing agents with specialized tools and interfaces. For instance, +CodeAgent [ 18] integrates five domain-specific tools to support repository-level analysis, while SWE- +agent [ 19] introduces a high-level Agent-Computer Interface (ACI) to enable robust agent interaction +with file systems and development environments. In addition, ToolGen [ 20] proposes representing +each tool as a unique token and directly integrating tool-specific knowledge into the parameters of the +LLM, thereby enabling a paradigm shift toward seamless unification of tool invocation and natural +language generation. +Recent advancements in academic research are increasingly being translated into practical, produc- +tized tools. Commercial code agents emerging from this trend can be broadly categorized into two +distinct paradigms: (1) AI-native integrated development environments (IDEs) such as Cursor [ 9] +and Trae [ 21] that embed AI capabilities directly into the editor interface, and (2) terminal-based +or extension-based agents including Claude Code [ 10], Gemini CLI [ 22], Github Copilot [ 23], and +Cline [ 24] that operate through command-line interfaces or editor extensions. These coding agents +leverage a holistic understanding of the codebase to perform complex tasks such as multi-file refac- +toring and autonomous edits. They support flexible, composable workflows and integrate seamlessly +into diverse development pipelines. Commercial deployments indicate significant improvements in +both function implementation and overall programming productivity. Despite their effectiveness, +these agents suffer from context window limitations that impair their ability to process lengthy +technical documents such as academic papers, and struggle to maintain coherence and correctness +when synthesizing repository-level codebases. +5.2 Scientific Coding Agents +In contrast to general-purpose coding agents, this class of agents targets more complex code generation +scenarios, including the implementation and reproduction of entire codebases from high-level ideas +and academic papers. For example, Paper2Code [ 8] addresses the research reproducibility crisis by +transforming machine learning papers into executable repositories. Its code generation framework +follows a structured three-stage process that includes system architecture design, implementation +detail extraction, and modular code generation. CodeScientist [ 25] generates experimental code +from literature, employing an iterative generate-execute-reflect cycle to write, run, and debug Python +experiments. In addition, AlphaEvolve [ 26] utilize code generation for algorithmic discovery, using +an LLM as an evolutionary mutator to propose variations to entire codebases, which are then +rigorously evaluated. Besides, the automation code in AI Scientist [ 27] and AI-Researcher [ 6] +enables agents to iteratively plan and execute experiments, handle errors, and refine future runs +based on results. AI Scientist focuses on experimental automation, maintaining execution history +and generating plots and notes to support scientific write-ups. AI-Researcher extends this with a +multi-stage refinement framework, where a code agent implements modular solutions and an advisor +agent provides structured feedback for iterative validation, revision, and scaling. These agents +have advanced the pace of scientific research, yet achieving higher generation efficiency without +compromising code quality remains an open challenge. +14 + +## Page 15 + +6 Discussion: Challenges and Future Directions +While DeepCode demonstrates the efficacy of principled information-flow management in high- +fidelity repository synthesis, the transition from episodic coding tasks to autonomous, cost-effective, +and self-evolving engineering remains fraught with challenges. We identify three critical frontiers +that define the future trajectory of agentic software engineering. +(1) Agentic Capability and Computational Efficiency.SOTA performance in agentic coding +currently relies on massive, proprietary LLMs (e.g. GPT-5, Claude 4.5), which incur prohibitive +deployment costs and high latency. Conversely, smaller, open-weight models offer efficiency but +lack the complex reasoning capabilities required for autonomous decision-making in open-ended +engineering tasks. Bridging this gap presents a dichotomy of challenges.(i) Fine-tuning limits: +Enhancing small models via supervised fine-tuning (SFT) is constrained by a data bottleneckโ€”while +raw code is abundant, high-quality agentic trajectories are scarce and expensive to curate.(ii) +Knowledge injection limits:Merely augmenting small models with external knowledge is often +insufficient; retrieved contexts may lack direct relevance to the specific coding task, and small models +struggle to integrate complex inputs without suffering from attention dilution. +We envision a shift toward hybrid agentic architectures that synergize models of varying scales, em- +ploying large models for high-level reasoning and efficient small models for routine implementation. +Besides, distilling knowledge from large models helps reduce the data bottleneck. +(2) From Episodic to Evolving Agents.Current coding agents typically operate in an episodic +manner: they reset after each project, failing to carry over experience or tacit knowledge to subsequent +tasks. Enabling agents to self-evolve and accumulate expertise mirrors human professional growth +but faces significant hurdles.(i) Reinforcement Learning constraints:While RL-based optimization +theoretically allows agents to learn from feedback, it requires well-defined reward functions, which +are difficult to formulate for complex, multi-objective software engineering tasks. Moreover, this +approach is inapplicable to closed-source LLMs where parameter updates are impossible.(ii) +Memory scalability issues:The alternative approachโ€”stacking historical experiences into a long- +term memoryโ€”introduces severe noise. Simply accumulating raw interaction logs leads to context +bloat, where retrieving relevant past experiences becomes a โ€œneedle in a haystackโ€ problem. +Beyond relying on extensive manual annotation and training, a scalable solution involves automating +the abstraction of past experiences. Future agents can implement post-task reflection to condense +execution traces into reusable skills or heuristics. Storing these refined insights allows agents to +retrieve corresponding high-level guidance, enabling self-evolution while avoiding context explosion. +(3) Dynamic Planning and Adaptability.Most existing frameworks utilize a linear Plan-then-Code +workflow, assuming that all constraints are knowable a priori. In real-world engineering, specifications +often evolve, and critical implementation constraints are frequently discovered only during the coding +process. Separation between planning and execution leads to fragility: if the initial blueprint is flawed, +the coding agent is often constrained by a stale plan, leading to suboptimal workarounds or failure. +Future researches advance toward dynamic, bidirectional planning frameworks. Agents are able +to adapt their initial blueprints when encountering unforeseen constraints during implementation. +Establishing a feedback mechanism where execution insights directly inform and update the high-level +plan is crucial for handling the complex realities of large-scale software development. +7 Conclusion +In this work, we presented DeepCode, an autonomous framework that advances the frontier of agentic +code engineering by reimagining document-to-repository synthesis as a challenge ofinformation-flow +management. Addressing the fundamental conflict between information overload and finite context +bottlenecks, we demonstrated that treating synthesis as a channel optimization problemโ€”solved +through the orchestration of blueprint distillation, stateful memory, conditional knowledge injection, +and closed-loop verificationโ€”effectively maximizes the signal-to-noise ratio for long-horizon tasks. +Empirical evaluations on PaperBench confirm that DeepCode establishes a new SOTA, decisively +outperforming leading commercial agents and surpassing PhD-level human experts in reproduction +accuracy. These findings validate that hierarchical information orchestration, rather than indiscrimi- +nate context scaling, provides the decisive path toward robust autonomous systems, laying a critical +foundation for the future of automated scientific discovery and rigorous research reproduction. +15 + +## Page 16 + +References +[1]Juyong Jiang, Fan Wang, Jiasi Shen, Sungju Kim, and Sunghun Kim. A survey on large +language models for code generation.arXiv preprint arXiv:2406.00515, 2024. +[2]Yuyao Ge, Lingrui Mei, Zenghao Duan, Tianhao Li, Yujia Zheng, Yiwei Wang, Lexin Wang, +Jiayu Yao, Tianyu Liu, Yujun Cai, Baolong Bi, Fangda Guo, Jiafeng Guo, Shenghua Liu, +and Xueqi Cheng. A survey of vibe coding with large language models, 2025. URL https: +//arxiv.org/abs/2510.12399. +[3]Sida Peng, Eirini Kalliamvakou, Peter Cihon, and Mert Demirer. The impact of ai on developer +productivity: Evidence from github copilot.arXiv preprint arXiv:2302.06590, 2023. +[4]Yihong Dong, Xue Jiang, Jiaru Qian, Tian Wang, Kechi Zhang, Zhi Jin, and Ge Li. A survey on +code generation with llm-based agents, 2025. URL https://arxiv.org/abs/2508.00083 . +[5]Huanting Wang, Jingzhi Gong, Huawei Zhang, Jie Xu, and Zheng Wang. Ai agentic program- +ming: A survey of techniques, challenges, and opportunities.arXiv preprint arXiv:2508.11126, +2025. +[6]Jiabin Tang, Lianghao Xia, Zhonghang Li, and Chao Huang. AI-Researcher: Autonomous +Scientific Innovation. InNeurIPS, 2025. +[7]Giulio Starace, Oliver Jaffe, Dane Sherburn, James Aung, Jun Shern Chan, Leon Maksin, +Rachel Dias, Evan Mays, Benjamin Kinsella, Wyatt Thompson, Johannes Heidecke, Amelia +Glaese, and Tejal Patwardhan. Paperbench: Evaluating aiโ€™s ability to replicate ai research, 2025. +URLhttps://arxiv.org/abs/2504.01848. +[8]Minju Seo, Jinheon Baek, Seongyun Lee, and Sung Ju Hwang. Paper2code: Automating code +generation from scientific papers in machine learning, 2025. URL https://arxiv.org/abs/ +2504.17192. +[9] Anysphere. Cursor: The best way to code with ai.https://cursor.com, 2025. +[10] Anthropic. Claude code: Agentic coding tool for your terminal. https://docs.claude.com/ +en/docs/claude-code/overview, 2025. +[11] OpenAI. Codex cli: Pair with codex in your terminal. https://developers.openai.com/ +codex/cli, 2025. +[12] Chen Qian, Wei Liu, Hongzhang Liu, Nuo Chen, Yufan Dang, Jiahao Li, Cheng Yang, Weize +Chen, Yusheng Su, Xin Cong, Juyuan Xu, Dahai Li, Zhiyuan Liu, and Maosong Sun. Chatdev: +Communicative agents for software development, 2024. URL https://arxiv.org/abs/ +2307.07924. +[13] Sirui Hong, Mingchen Zhuge, Jiaqi Chen, Xiawu Zheng, Yuheng Cheng, Ceyao Zhang, Jinlin +Wang, Zili Wang, Steven Ka Shing Yau, Zijuan Lin, Liyang Zhou, Chenyu Ran, Lingfeng +Xiao, Chenglin Wu, and Jรผrgen Schmidhuber. Metagpt: Meta programming for a multi-agent +collaborative framework, 2024. URLhttps://arxiv.org/abs/2308.00352. +[14] Zeeshan Rasheed, Malik Abdul Sami, Kai-Kristian Kemell, Muhammad Waseem, Mika Saari, +Kari Systรค, and Pekka Abrahamsson. Codepori: Large-scale system for autonomous software +development using multi-agent technology, 2024. URL https://arxiv.org/abs/2402. +01411. +[15] Daoguang Zan, Ailun Yu, Wei Liu, Dong Chen, Bo Shen, Yafen Yao, Wei Li, Xiaolin Chen, +Yongshun Gong, Bei Guan, et al. Codes: Natural language to code repository via multi-layer +sketch.ACM Transactions on Software Engineering and Methodology, 2024. +[16] Dong Huang, Jie M. Zhang, Michael Luck, Qingwen Bu, Yuhao Qing, and Heming Cui. +Agentcoder: Multi-agent-based code generation with iterative testing and optimisation, 2024. +URLhttps://arxiv.org/abs/2312.13010. +[17] Md. Ashraful Islam, Mohammed Eunus Ali, and Md Rizwan Parvez. MapCoder: Multi-agent +code generation for competitive problem solving. InACL, pages 4912โ€“4944, 2024. +16 + +## Page 17 + +[18] Kechi Zhang, Jia Li, Ge Li, Xianjie Shi, and Zhi Jin. CodeAgent: Enhancing code generation +with tool-integrated agent systems for real-world repo-level coding challenges. InACL, pages +13643โ€“13658, 2024. +[19] John Yang, Carlos E. Jimenez, Alexander Wettig, Kilian Lieret, Shunyu Yao, Karthik +Narasimhan, and Ofir Press. Swe-agent: agent-computer interfaces enable automated software +engineering. InNeurIPS, pages 50528โ€“50652, 2025. +[20] Renxi Wang, Xudong Han, Lei Ji, Shu Wang, Timothy Baldwin, and Haonan Li. Toolgen: +Unified tool retrieval and calling via generation. InThe Thirteenth International Conference on +Learning Representations, 2025. URL https://openreview.net/forum?id=XLMAMmowdY . +[21] ByteDance. Trae: The real ai engineer.https://www.trae.ai, 2025. +[22] Google. Gemini cli: An open-source ai agent that brings the power of gemini directly into your +terminal.https://github.com/google-gemini/gemini-cli, 2025. +[23] GitHub and OpenAI. Github copilot: Your ai pair programmer. https://github.com/ +features/copilot, 2025. +[24] Saoud Rizwan and others. Cline: Autonomous coding agent for vs code. https://github. +com/cline/cline, 2024. +[25] Peter Jansen, Oyvind Tafjord, Marissa Radensky, Pao Siangliulue, Tom Hope, Bhavana Dalvi +Mishra, Bodhisattwa Prasad Majumder, Daniel S. Weld, and Peter Clark. Codescientist: End- +to-end semi-automated scientific discovery with code-based experimentation, 2025. URL +https://arxiv.org/abs/2503.22708. +[26] Alexander Novikov, Ngรขn V หœu, Marvin Eisenberger, Emilien Dupont, Po-Sen Huang, et al. +Alphaevolve: A coding agent for scientific and algorithmic discovery, 2025. URL https: +//arxiv.org/abs/2506.13131. +[27] Chris Lu, Cong Lu, Robert Tjarko Lange, Jakob Foerster, Jeff Clune, and David Ha. The +ai scientist: Towards fully automated open-ended scientific discovery, 2024. URL https: +//arxiv.org/abs/2408.06292. +17 + +## Page 18 + +A Appendix +This appendix supplements the main text by providing four categories of supplementary materials. +First, theComplete Resultssubsection reports an extensive quantitative evaluation of DeepCode, +including comparative analysis against multiple benchmark models and reproducibility analysis across +different papers and operational scenarios. Second, theDeepCode Application Casessubsection +showcases representative visualizations demonstrating DeepCodeโ€™s end-to-end capabilities, covering +backend systems, web user interfaces, and the Paper2Code research reproduction workflow. Third, the +DeepCode Sub-Agent Detailssubsection elucidates the internal multi-agent architecture, clarifying +the roles, responsibilities, and coordination patterns for implementing specific specialized sub-agents. +Finally, theMCP Toolkit in DeepCodesubsection documents the Model Context Protocol (MCP) +tools integrated into the system, defining the external interfaces through which DeepCode interacts +with code repositories, documentation, and execution environments. +A.1 Full Results +This appendix reports quantitative results that complement the main text and provide a more systematic +evaluation of DeepCodeโ€™s overall capability and stability on research code reproduction tasks. Table 2 +first compares, under a unified evaluation protocol, a range of general-purpose code execution agents +(including both BasicAgent and IterativeAgent configurations), existing specialized reproduction +systems such as PaperCoder, and human experts on the same benchmark. DeepCode achieves +an average reproduction score of 73.5ยฑ2.8 on the full benchmark, substantially outperforming +PaperCoder ( 51.1ยฑ1.4 ) as well as all configurations derived from commercial models. On the +3-paper subset, DeepCode attains an average score of 75.9ยฑ4.5 , exceeding the human โ€œBest@3โ€ +score of 72.4, indicating that, on representative deep learning papers, the system delivers reproduction +quality comparable to or better than that of strong human practitioners. +Table 1 further selects a 5-paper subset (fre, rice, bam, pinn, mech-u) for a head-to-head comparison +against several widely used commercial code assistants (Codex, Claude Code, Cursor, etc.). Across all +papers, DeepCode achieves the highest reproduction score, with an average of 0.8482, corresponding +to an absolute improvement of more than 0.26 over the strongest competing system. The advantage +is consistent across all individual papers, suggesting that the gains arise from architectural and +procedural design choices rather than from favorable alignment with a narrow subset of tasks. +Finally, Table 3 provides per-paper details for the Claude 4.5 Sonnetโ€“based configuration, includ- +ing three independent runs, their mean and standard error, as well as the associated average cost. +Across a diverse set of targetsโ€”such as FRE, PINN, MECHANISTIC-UNDERSTANDING, and +SEQUENTIAL-NEURAL-SCORE-ESTIMATIONโ€”DeepCodeโ€™s reproduction scores typically lie +in the 0.7โ€“0.9 range with relatively small standard errors, while the distribution of average cost +across papers remains tight. This indicates strong cross-task generalization, stable behavior across +repeated runs, and reasonable resource usage. Taken together, these appendix results reinforce the +main conclusions of the paper: on realistic research code reproduction benchmarks, DeepCode not +only achieves significantly higher average performance than existing automated reproduction and +code assistance systems, but also demonstrates robust and consistent advantages in fine-grained, +multi-paper, multi-run analyses. +A.2 Use Cases for DeepCode +This appendix provides a series of visual artifacts generated by DeepCode, offering concrete evidence +of its capabilities across different software development and research domains. These examples +are intended to supplement the main paper by illustrating the practical utility and versatility of our +system. +The initial set of examples, depicted in Figure 7, focuses on DeepCodeโ€™s proficiency in generating +sophisticated backend systems. The figures showcase automatically constructed administrative +dashboards, which likely include functionalities for data monitoring, user management, and content +moderation. Such pages are critical for the operational management of modern web applications but +are often tedious and repetitive to build. DeepCodeโ€™s ability to scaffold these complex, data-driven +interfaces from high-level specifications demonstrates its potential to significantly reduce boilerplate +engineering and accelerate the deployment of robust server-side infrastructure. +18 + +## Page 19 + +Table 2: Average reproduction scores: DeepCode vs. LLMs and human experts +Model Average Replication Scores +GEMINI-2.0-FLASH (BasicAgent)5.0ยฑ0.0 +4o (BasicAgent)7.7ยฑ0.0 +o3-mini (BasicAgent)5.1ยฑ0.8 +o1 (BasicAgent)19.5ยฑ1.2 +R1 (BasicAgent)9.8ยฑ0.0 +CLAUDE-3-5-SONNET (BasicAgent)35.4ยฑ0.8 +o3-mini (IterativeAgent)16.4ยฑ1.4 +o1 (IterativeAgent)43.3ยฑ1.1 +CLAUDE-3-5-SONNET (IterativeAgent)27.5ยฑ1.6 +o1 [36 hours] (IterativeAgent)42.4ยฑ1.0 +PaperCoder51.1ยฑ1.4 +DeepCode 73.6ยฑ5.3 +Human[3 paper subset, Best@3] 72.4 +DeepCode[3 paper subset, Average]76.7ยฑ3.9 +Table 3: DeepCode with Claude 4.5 Sonnet results. +Paper Run 1 Run 2 Run 3 Mean Std. Error Avg. Cost +FRE 0.844 0.823 0.803 0.814 0.020 9.14 +RICE 0.738 0.609 0.761 0.702 0.082 8.22 +BAM 0.853 0.673 0.719 0.748 0.094 8.45 +WILL-MODEL-FORGET 0.776 0.793 0.857 0.808 0.042 9.20 +PINN 0.947 0.800 0.983 0.910 0.097 7.84 +ALL-IN-ONE 0.769 0.747 0.759 0.759 0.011 9.43 +ADAPTIVE-PRUNING 0.547 0.570 0.516 0.544 0.027 9.13 +LBCS 0.689 0.732 0.820 0.747 0.066 10.01 +MECHANISTIC-UNDERSTANDING 0.889 0.944 0.941 0.925 0.031 10.20 +TEST-TIME-MODEL-ADAPTATION 0.717 0.578 0.652 0.649 0.069 7.90 +SAMPLE-SPECIFIC-MASKS 0.690 0.740 0.583 0.671 0.080 8.30 +BRIDGING-DATA-GAPS 0.552 0.566 0.626 0.581 0.039 7.98 +STAY-ON-TOPIC-WITH-CLASSIFIER-FREE-GUIDANCE 0.734 0.800 0.626 0.705 0.088 9.12 +STOCHASTIC-INTERPOLANTS 0.851 0.792 0.801 0.815 0.031 8.89 +LCA-ON-THE-LINE 0.665 0.844 0.739 0.749 0.090 7.73 +SEQUENTIAL-NEURAL-SCORE-ESTIMATION 0.930 0.862 0.817 0.870 0.057 10.01 +SAPG 0.702 0.755 0.757 0.738 0.031 9.19 +FTRL 0.558 0.606 0.631 0.598 0.037 7.06 +ROBUST-CLIP 0.772 0.742 0.685 0.733 0.044 7.83 +BBOX 0.620 0.681 0.631 0.644 0.033 11.90 +Building upon the backend logic, a systemโ€™s utility is often defined by its user-facing presentation. +Figure 8 illustrates DeepCodeโ€™s capacity for generating intuitive and functional Web UIs. The +generated interfaces, featuring elements such as data visualization charts and interactive forms, +translate abstract user requirements into tangible, interactive components. This capability not only +complements the backend generation by providing a corresponding frontend, but also empowers +developers and designers to rapidly prototype and iterate on user experiences, thereby shortening the +path from concept to a functional product. +Perhaps DeepCodeโ€™s most ambitious application, however, lies in its potential to bridge the chasm +between academic research and practical implementation. The Paper2Code functionality, illustrated +in Figure 9, exemplifies this capability. The figure is twofold: on the left, it presents the high-level +code structure that DeepCode inferred from a research paper, discerning the architectural blueprint +of the proposed algorithm, including its modular components and file organization. On the right, it +provides a concrete code sample, instantiating a specific function or class with precise logic. This +powerful feature moves beyond conventional code generation by interpreting unstructured scientific +19 + +## Page 20 + +Figure 7: DeepCode-generated backend system pages. +Figure 8: DeepCode-generated Web UI. +language to produce structured, executable artifacts, thereby holding immense promise for enhancing +research reproducibility and accelerating the adoption of novel scientific discoveries. +A.3 Sub-Agents Details of DeepCode +DeepCode decomposes the software engineering pipeline into a set of specialized agents with narrow, +well-specified responsibilities and standardized communication interfaces, rather than relying on a +single monolithic generative model. The individual agents and their responsibilities are summarized +in Table 4. This modular design allows different stages of the lifecycleโ€”ranging from requirement un- +derstanding to architectural planning and code synthesisโ€”to be implemented as transformations over +shared intermediate representations, while preserving global architectural and semantic consistency. +During the planning stage, DeepCode relies on explicit coordination between conceptual and +algorithmic analysis agents to derive a coherent development blueprint from high-level spec- +ifications.The Central Orchestrating Agent first routes each input through the Document Parsing +and/or Intent Understanding agents to obtain a structured specification, which then serves as the +input to the Code Planning agent. Within this planning module, two internal analysis pipelines +operate in parallel over the same intermediate representation. The conceptual analysis sub-agent is +responsible for system-level decomposition: it identifies major subsystems, their responsibilities, +and inter-module interfaces, and it constructs an architecture-level call topology. The algorithmic +analysis sub-agent is responsible for computational aspects: it abstracts key algorithmic ideas, selects +candidate data structures, reasons about time and space complexity constraints, and enumerates +feasible implementation patterns. The partial plans produced by these two sub-agents are reconciled +by a planning aggregation component (Code Analysis agent), which resolves inconsistencies and +materializes a project-level development roadmap, including module boundaries, interface signatures, +dependency relations, implementation priorities, and testing hooks. This roadmap serves as the design +baseline that constrains all downstream code generation and refinement steps. +20 + +## Page 21 + +Figure 9: Paper2Code Samples of DeepCode. Left: Code Structure, Right: Code Sample +During the code synthesis stage, DeepCode couples retrieval-augmented reference mining with +a global code memory, forming a closed-loop process that enforces repository-level consistency +during incremental generation.On the retrieval side, the Code Reference Mining and Code Indexing +agents implement a Retrieval-Augmented Generation (RAG) layer: they maintain multi-granularity +indices over a corpus of prior implementations and expose to the Code Generation agent semantically +relevant and structurally compatible code patterns, ranging from individual functions to reusable +design idioms. In parallel, the Code Memory agent maintains a structured representation of the +current repository state, including cross-file symbol tables, dependency graphs, and project-wide +conventions such as naming schemes, error-handling strategies, and configuration mechanisms. +Before emitting new code, the Code Generation agent issues queries to the Code Memory agent to +obtain the up-to-date repository context and applicable constraints; after generation, it writes back +the newly introduced symbols and dependencies, triggering an update of the global repository model. +This queryโ€“constraintโ€“update loop allows DeepCode to align local synthesis decisions with global +architectural intent, reducing interface mismatches, naming drift, and latent coupling across the +codebase. +A.4 MCP Tool Stack in DeepCode +Table 5 summarizes the Model Context Protocol (MCP) tools integrated into DeepCode. The tools are +grouped into three functional categories:Perception & Retrieval,Cognitive Processing, andAction & +Execution. This organization makes the main stages of the system explicit. Perception & Retrieval +tools give the model access to up-to-date web search results, web pages, and binary documents such +as research papers and technical manuals, which helps mitigate the effects of the modelโ€™s knowledge +cut-off. Cognitive Processing tools then convert large codebases and long documents into semantic +indexes and context-window-compatible segments, so that the model can issue natural language +queries over existing artifacts and work with long technical materials. Action & Execution tools +finally operate on the local development environment by reading and writing project files, executing +shell commands, and interacting with the version control system. +Taken together, the tools in Table 5 form an end-to-end loop for assisted software development. The +system can retrieve external and local information, reorganize it into internal structures that fit within +the modelโ€™s context window, and then apply code changes while observing their effects through +commands such as tests or package installations. The table also shows that operations with side +effects on the environment (file I/O, command execution, and Git operations) are confined to the +Action & Executionlayer and are described as sandboxed and path-validated. This separation between +information access, semantic processing, and environment manipulation makes the extension of the +base language model through MCP tools transparent and easier to reason about. +21 + +## Page 22 + +Table 4: Functional Specifications of Specialized Sub-Agents in the DeepCode Framework +Agent Role Functional Description +Central Orchestrating +AgentFunctions as the central control unit, responsible for task decomposi- +tion, resource allocation, and the strategic coordination of sub-agents +based on the complexity of the input requirements. +Intent Understanding +AgentConducts semantic parsing of natural language inputs to extract +functional requirements, converting ambiguous user descriptions +into formal technical specifications. +Document Parsing Agent Processes unstructured technical documents (e.g., research papers). +It extracts multimodal information, including text, mathematical +formulas, and diagrams, to establish a ground truth for implementa- +tion. +Concept Analysis Agent Abstracts core theoretical concepts and logical flows from the parsed +specifications, ensuring the computational model aligns with the +theoretical underpinnings of the source material. +Algorithm Analysis Agent Evaluates and selects appropriate algorithmic strategies and data +structures. It focuses on optimizing computational complexity and +feasibility before code synthesis begins. +Code Planning Agent Formulates the software architecture and development roadmap. +This agent determines the technology stack, designs modular file +structures, and enforces design patterns to ensure scalability. +Code Reference Mining +AgentRetrieves external knowledge by identifying relevant open-source +repositories. It analyzes dependency graphs to recommend integra- +tion patterns and library usages. +Code Memory Agent Manages the state and context throughout the generation lifecycle. +It utilizes hierarchical data structures to retain historical decisions +and maintain semantic consistency across long-context interactions. +Code Generation Agent Synthesizes executable source code based on the architectural plan +and retrieved references. It implements functional interfaces and +integrates distinct modules into a cohesive codebase. +Automated Validation +AgentExecutes a rigorous quality assurance loop. It performs static analy- +sis, generates unit tests, and iteratively debugs the codebase to verify +functional correctness and adherence to specifications. +22 + +## Page 23 + +Table 5: Specification of Model Context Protocol (MCP) Tools Integrated into DeepCode. These +tools extend the Large Language Modelโ€™s capabilities across perception, cognitive processing, and +environment manipulation domains +Category MCP Server Name Functional Description & Academic Specifi- +cation +Perception & Retrievalbrave_search A real-time information retrieval interface lever- +aging the Brave Search API. It provides the +agent with temporal-aware access to web in- +dices, enabling the retrieval of up-to-date doc- +umentation and resolving knowledge cut-off +limitations. +bocha_mcp A specialized search module delivering struc- +tured "modal cards" and semantic summaries. +It serves as a secondary knowledge source, opti- +mizing token efficiency by returning structured +entities rather than raw HTML. +fetch A web content ingestion engine that retrieves +URL endpoints and normalizes heterogeneous +HTML structures into clean Markdown. It acts +as the agentโ€™s primary reading interface for ex- +ternal documentation. +pdf_downloader Binary resource acquisition tool designed for +academic papers and technical manuals. It han- +dles HTTP streams to ingest non-textual doc- +ument formats (PDF/DOCX) for downstream +processing. +Cognitive Processingcode_reference_indexer A Retrieval-Augmented Generation (RAG) +module for local codebases. It constructs a +vector or semantic index of the project files, +allowing the agent to perform natural language +queries over the existing code structure. +document_segmentation A pre-processing utility implementing semantic +chunking algorithms. It partitions large techni- +cal documents into context-window-compliant +segments, facilitating the "Paper2Code" work- +flow for complex algorithm implementation. +Action & Executionfilesystem A sandboxed file I/O interface allowing con- +trolled read/write operations within the project +directory. It enforces path validation security +policies to prevent unauthorized system access +during code generation. +code_implementation The core generative engine encapsulated as an +MCP tool. It orchestrates the synthesis of func- +tional code blocks, integrating logic planning +with atomic file writing operations to ensure +code coherence. +command_executor A runtime environment interface permitting the +execution of shell commands (e.g., pytest , +pip install ). It establishes a feedback loop +by capturing stdout /stderr for iterative de- +bugging and self-correction. +git_command Version control management interface. It ab- +stracts Git plumbing commands, enabling the +agent to manage repository state, branch for ex- +perimental features, and maintain a clean com- +mit history. +23 + diff --git a/DeepCode/deepcode_lab/papers/1/1.pdf b/DeepCode/deepcode_lab/papers/1/1.pdf new file mode 100644 index 00000000..bef7fc33 Binary files /dev/null and b/DeepCode/deepcode_lab/papers/1/1.pdf differ diff --git a/DeepCode/deepcode_lab/papers/1/code_implementation_report.txt b/DeepCode/deepcode_lab/papers/1/code_implementation_report.txt new file mode 100644 index 00000000..f349665c --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/code_implementation_report.txt @@ -0,0 +1 @@ +{'status': 'success', 'plan_file': '/Users/DanielKim_1/DeepCode/deepcode_lab/papers/1/initial_plan.txt', 'target_directory': '/Users/DanielKim_1/DeepCode/deepcode_lab/papers/1', 'code_directory': '/Users/DanielKim_1/DeepCode/deepcode_lab/papers/1/generate_code', 'results': {'file_tree': 'The file tree structure has been successfully created based on the implementation plan.\n\n```bash\nmkdir -p deepcode_repro/docker\nmkdir -p deepcode_repro/src/core\nmkdir -p deepcode_repro/src/agents\nmkdir -p deepcode_repro/src/utils\nmkdir -p deepcode_repro/experiments\nmkdir -p deepcode_repro/data/input_papers\nmkdir -p deepcode_repro/data/output_repos\ntouch deepcode_repro/main.py\ntouch deepcode_repro/config.yaml\ntouch deepcode_repro/requirements.txt\ntouch deepcode_repro/README.md\ntouch deepcode_repro/docker/Dockerfile\ntouch deepcode_repro/docker/entrypoint.sh\ntouch deepcode_repro/src/__init__.py\ntouch deepcode_repro/src/core/__init__.py\ntouch deepcode_repro/src/core/document_parser.py\ntouch deepcode_repro/src/core/memory.py\ntouch deepcode_repro/src/core/rag_engine.py\ntouch deepcode_repro/src/core/blueprint.py\ntouch deepcode_repro/src/agents/__init__.py\ntouch deepcode_repro/src/agents/base.py\ntouch deepcode_repro/src/agents/planning.py\ntouch deepcode_repro/src/agents/coding.py\ntouch deepcode_repro/src/agents/verification.py\ntouch deepcode_repro/src/utils/mcp_tools.py\ntouch deepcode_repro/src/utils/prompts.py\ntouch deepcode_repro/src/utils/logger.py\ntouch deepcode_repro/experiments/run_paperbench.py\ntouch deepcode_repro/experiments/validate_repro.py\n```', 'code_implementation': "Failed to generate final report: 'CallToolResult' object has no attribute 'get'"}, 'mcp_architecture': 'standard'} \ No newline at end of file diff --git a/DeepCode/deepcode_lab/papers/1/codebase_index_report.txt b/DeepCode/deepcode_lab/papers/1/codebase_index_report.txt new file mode 100644 index 00000000..3d6e66c9 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/codebase_index_report.txt @@ -0,0 +1 @@ +{'status': 'skipped', 'reason': 'fast_mode_enabled', 'message': 'Codebase intelligence orchestration skipped for optimized processing'} \ No newline at end of file diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/README.md b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/README.md new file mode 100644 index 00000000..324adbaa --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/README.md @@ -0,0 +1,125 @@ +# DeepCode: Open Agentic Coding + +A fully autonomous multi-agent framework for high-fidelity document-to-repository synthesis. DeepCode solves the context bottleneck in LLM-based coding via principled information-flow management (CodeMem, CodeRAG, and Blueprinting). + +This repository is a reproduction of the framework described in the paper "DeepCode: Open Agentic Coding". + +## Core Features + +* **Hierarchical Content Segmentation**: Transforms raw research papers (PDF/Markdown) into structured semantic chunks. +* **Planning Swarm**: Synthesizes a detailed implementation Blueprint (JSON) using Concept, Algorithm, and Planning agents. +* **CodeMem**: A graph-based memory system that maintains cross-file consistency without saturating the context window. +* **CodeRAG**: Retrieval-Augmented Generation engine for injecting external knowledge or specific paper details on demand. +* **Verification Swarm**: A two-stage verification loop (Static Analysis & Dynamic Sandbox Execution) to ensure functional correctness. +* **Docker Sandbox**: Secure execution environment for running generated code and tests. + +## Prerequisites + +* **OS**: Linux or macOS (required for Docker compatibility) +* **Python**: >= 3.10 +* **Docker Engine**: Installed and running +* **API Keys**: + * Anthropic (Claude-3.5-Sonnet) OR OpenAI (GPT-4o) + * Brave Search (Optional, for external web search) + +## Installation + +1. **Clone the repository**: + ```bash + git clone https://github.com/yourusername/deepcode_repro.git + cd deepcode_repro + ``` + +2. **Install Python dependencies**: + ```bash + pip install -r requirements.txt + ``` + +3. **Build the Docker Sandbox**: + The sandbox environment is required for the Verification Agent to run tests safely. + ```bash + cd docker + docker build -t deepcode_sandbox:latest . + cd .. + ``` + +## Configuration + +1. **Edit `config.yaml`**: + Open `config.yaml` and configure your LLM provider and API keys. + + ```yaml + llm: + provider: "anthropic" # or "openai" + model: "claude-3-5-sonnet-20240620" # or "gpt-4o" + api_key: "YOUR_API_KEY_HERE" + temperature: 0.0 + max_tokens: 4096 + + sandbox: + image: "deepcode_sandbox:latest" + workspace_mount: "./data/output_repos" + ``` + + *Note: You can also set API keys via environment variables `ANTHROPIC_API_KEY` or `OPENAI_API_KEY`.* + +## Usage + +### 1. Generate a Repository from a Paper + +To generate a codebase from a research paper (PDF or Markdown): + +```bash +python main.py --paper data/input_papers/attention_is_all_you_need.pdf --output_dir data/output_repos/transformer +``` + +**Workflow:** +1. **Parsing**: The paper is chunked into segments. +2. **Planning**: The Planning Swarm creates a `blueprint.json`. +3. **Coding**: The Coding Swarm generates files topologically, updating `memory.json`. +4. **Verification**: The Verification Swarm runs static analysis and executes tests in Docker, applying fixes as needed. + +### 2. Run Validation Checks + +To verify that the environment, Docker sandbox, and agents are set up correctly: + +```bash +python experiments/validate_repro.py +``` + +### 3. Run PaperBench Evaluation + +To run the framework on a dataset of papers (PaperBench) and measure Pass@1 rates: + +```bash +python experiments/run_paperbench.py --papers_dir data/input_papers --output_dir data/bench_results --limit 5 +``` + +## Project Structure + +```text +deepcode_repro/ +โ”œโ”€โ”€ main.py # CLI Entry point +โ”œโ”€โ”€ config.yaml # Configuration +โ”œโ”€โ”€ docker/ # Sandbox environment +โ”œโ”€โ”€ src/ +โ”‚ โ”œโ”€โ”€ core/ # Core engines +โ”‚ โ”‚ โ”œโ”€โ”€ document_parser.py # PDF/MD Parsing +โ”‚ โ”‚ โ”œโ”€โ”€ blueprint.py # Schema definitions +โ”‚ โ”‚ โ”œโ”€โ”€ memory.py # CodeMem graph +โ”‚ โ”‚ โ””โ”€โ”€ rag_engine.py # CodeRAG vector store +โ”‚ โ”œโ”€โ”€ agents/ # Agent Swarms +โ”‚ โ”‚ โ”œโ”€โ”€ planning.py # Concept/Algo/Planner agents +โ”‚ โ”‚ โ”œโ”€โ”€ coding.py # Generator/Summarizer agents +โ”‚ โ”‚ โ””โ”€โ”€ verification.py # Analysis/Sandbox/Fix agents +โ”‚ โ””โ”€โ”€ utils/ # Utilities +โ”‚ โ”œโ”€โ”€ mcp_tools.py # Docker & FileSystem tools +โ”‚ โ””โ”€โ”€ prompts.py # System prompts +โ””โ”€โ”€ experiments/ # Validation & Benchmarking scripts +``` + +## Troubleshooting + +* **Docker Connection Error**: Ensure Docker Desktop or the Docker daemon is running. Try `docker ps` to verify. +* **PDF Parsing Issues**: If `marker-pdf` fails, the system falls back to `pypdf`. Ensure the PDF is text-selectable. +* **API Rate Limits**: If you encounter rate limits, increase the retry count in `src/agents/base.py` or check your provider's quota. diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/config.yaml b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/config.yaml new file mode 100644 index 00000000..33fb8803 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/config.yaml @@ -0,0 +1,40 @@ +llm: + anthropic_api_key: "${ANTHROPIC_API_KEY}" + openai_api_key: "${OPENAI_API_KEY}" + brave_search_api_key: "${BRAVE_SEARCH_API_KEY}" + + # Models + planning_model: "claude-3-5-sonnet-20240620" + coding_model: "claude-3-5-sonnet-20240620" + verification_model: "gpt-4o" + + # Parameters + temperature: 0.0 + max_tokens: 4096 + retry_attempts: 3 + +sandbox: + docker_image: "deepcode-sandbox:latest" + container_name_prefix: "deepcode_worker" + timeout_seconds: 300 + work_dir: "/workspace" + host_mount_dir: "./data/sandbox_mount" + +rag: + vector_store_path: "./data/chromadb" + collection_name: "deepcode_knowledge" + embedding_model: "text-embedding-3-small" # Using OpenAI for embeddings + similarity_threshold: 0.7 + max_results: 5 + chunk_size: 1000 + chunk_overlap: 200 + +paths: + input_papers: "./data/input_papers" + output_repos: "./data/output_repos" + logs: "./logs" + blueprints: "./data/blueprints" + +memory: + max_context_window: 128000 + summary_token_limit: 500 diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/docker/Dockerfile b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/docker/Dockerfile new file mode 100644 index 00000000..0ec87e6e --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/docker/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:22.04 + +# Prevent interactive prompts during package installation +ENV DEBIAN_FRONTEND=noninteractive + +# Install system dependencies +# python3-pip: for installing python packages +# git: for cloning repositories if needed +# build-essential: for compiling c extensions +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-dev \ + git \ + build-essential \ + curl \ + wget \ + && rm -rf /var/lib/apt/lists/* + +# Ensure python points to python3 +RUN ln -s /usr/bin/python3 /usr/bin/python + +# Upgrade pip +RUN pip3 install --no-cache-dir --upgrade pip + +# Pre-install common ML libraries to speed up execution and reduce network usage during sandbox runs +# As suggested in the Environment Setup section +RUN pip3 install --no-cache-dir \ + numpy \ + pandas \ + scipy \ + scikit-learn \ + torch \ + pytest \ + matplotlib \ + seaborn + +# Create workspace directory +WORKDIR /workspace + +# Copy entrypoint script +# Assumes the build context is the parent directory or docker directory +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +# Set PYTHONPATH to include workspace +ENV PYTHONPATH="${PYTHONPATH}:/workspace" + +# Define entrypoint +ENTRYPOINT ["/entrypoint.sh"] diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/docker/Dockerfile.backup b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/docker/Dockerfile.backup new file mode 100644 index 00000000..e69de29b diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/docker/entrypoint.sh b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/docker/entrypoint.sh new file mode 100644 index 00000000..3d519ea4 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/docker/entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e + +# DeepCode Sandbox Entrypoint + +# Ensure we are in the workspace directory +mkdir -p /workspace +cd /workspace + +# Set PYTHONPATH to include the workspace +export PYTHONPATH="${PYTHONPATH}:/workspace" + +# If arguments are passed, execute them +if [ "$#" -gt 0 ]; then + exec "$@" +else + # Default behavior: keep container running for MCP exec_run commands + echo "DeepCode Sandbox Ready. Waiting for commands..." + # Use tail -f /dev/null to keep the container alive indefinitely + exec tail -f /dev/null +fi diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/experiments/run_paperbench.py b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/experiments/run_paperbench.py new file mode 100644 index 00000000..810008f9 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/experiments/run_paperbench.py @@ -0,0 +1,217 @@ +import os +import sys +import yaml +import glob +import time +import shutil +import argparse +from pathlib import Path +from typing import List, Dict + +# Add project root to path to allow imports from src +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +if project_root not in sys.path: + sys.path.append(project_root) + +from src.utils.logger import setup_logger +from src.utils.mcp_tools import MCPToolkit, DockerSandbox +from src.core.document_parser import DocumentSegmenter +from src.core.memory import CodeMem +from src.core.rag_engine import CodeRAG +from src.agents.planning import PlanningSwarm +from src.agents.coding import CodingSwarm +from src.agents.verification import VerificationSwarm + +logger = setup_logger("PaperBench") + +def load_config(config_path: str = "config.yaml") -> Dict: + """Load configuration from YAML file.""" + path = os.path.join(project_root, config_path) + if not os.path.exists(path): + logger.error(f"Config file not found at {path}") + raise FileNotFoundError(f"Config file not found at {path}") + + with open(path, "r") as f: + return yaml.safe_load(f) + +def run_evaluation(paper_path: str, output_base_dir: str, config: Dict, run_id: str): + """Run the full DeepCode pipeline on a single paper.""" + paper_name = Path(paper_path).stem + repo_name = f"{paper_name}_{run_id}" + output_dir = os.path.join(output_base_dir, repo_name) + + logger.info(f"=== Starting Evaluation for {paper_name} ===") + logger.info(f"Output Directory: {output_dir}") + + # Ensure output directory exists + os.makedirs(output_dir, exist_ok=True) + + # Initialize Tools and Engines + # 1. Document Parser + logger.info("Step 1: Parsing Document...") + segmenter = DocumentSegmenter() + segments = segmenter.parse_file(paper_path) + paper_text = "\n\n".join([s.to_text() for s in segments]) + logger.info(f"Parsed {len(segments)} segments from {paper_name}") + + # 2. Planning Phase + logger.info("Step 2: Planning Phase (Blueprint Synthesis)...") + planning_swarm = PlanningSwarm(config) + blueprint = planning_swarm.plan(paper_text) + + # Save Blueprint + blueprint_path = os.path.join(output_dir, "blueprint.json") + blueprint.save(blueprint_path) + logger.info(f"Blueprint saved to {blueprint_path}") + + # 3. Coding Phase + logger.info("Step 3: Coding Phase (Implementation)...") + # Initialize Memory and RAG + memory = CodeMem() + + # Initialize RAG (Optional based on config, but we'll init it) + rag_persist_dir = os.path.join(output_dir, ".rag_store") + rag_engine = CodeRAG(persist_directory=rag_persist_dir) + # Index the paper segments for RAG + rag_engine.index_segments(segments, source_id=paper_name) + + coding_swarm = CodingSwarm( + config=config, + memory=memory, + rag_engine=rag_engine, + output_dir=output_dir + ) + + coding_swarm.generate_codebase(blueprint) + + # Save Memory State + memory.save(os.path.join(output_dir, "codemem.json")) + logger.info("Coding Phase Complete.") + + # 4. Verification Phase + logger.info("Step 4: Verification Phase (Analysis & Sandbox)...") + + # Setup Sandbox for Verification + # We map the output_dir on host to /workspace in container + sandbox_config = config.get("sandbox", {}) + sandbox = DockerSandbox( + image=sandbox_config.get("image", "deepcode_sandbox:latest"), + host_workspace_path=output_dir, + container_workspace_path="/workspace" + ) + + mcp_toolkit = MCPToolkit(sandbox_config={"sandbox": sandbox}) + + verification_swarm = VerificationSwarm( + config=config, + mcp_toolkit=mcp_toolkit, + output_dir=output_dir + ) + + # Run Verification Loop + verification_swarm.verify_codebase(blueprint) + logger.info("Verification Phase Complete.") + + # 5. Final Evaluation (Pass/Fail) + logger.info("Step 5: Final Evaluation...") + + # We assume the generated repo has tests (e.g., pytest) + # The blueprint should have included test files if the paper implied them, + # or we rely on the verification agent having added them. + # We'll try to run 'pytest' or 'python -m unittest' + + test_passed = False + with sandbox as sb: + # Try pytest first + exit_code, stdout, stderr = sb.execute_command("pytest", timeout=300) + if exit_code == 0: + logger.info(f"Tests PASSED for {paper_name} (pytest)") + test_passed = True + else: + logger.warning(f"pytest failed or not found. Output: {stdout[:200]}...") + # Try unittest + exit_code, stdout, stderr = sb.execute_command("python -m unittest discover .", timeout=300) + if exit_code == 0: + logger.info(f"Tests PASSED for {paper_name} (unittest)") + test_passed = True + else: + logger.error(f"Tests FAILED for {paper_name}") + logger.error(f"Stdout: {stdout}") + logger.error(f"Stderr: {stderr}") + + return { + "paper": paper_name, + "repo_path": output_dir, + "passed": test_passed + } + +def main(): + parser = argparse.ArgumentParser(description="Run PaperBench Evaluation") + parser.add_argument("--papers_dir", type=str, default="data/input_papers", help="Directory containing input papers") + parser.add_argument("--output_dir", type=str, default="data/output_repos", help="Directory for output repositories") + parser.add_argument("--limit", type=int, default=None, help="Limit number of papers to process") + args = parser.parse_args() + + # Setup paths + papers_dir = os.path.join(project_root, args.papers_dir) + output_dir = os.path.join(project_root, args.output_dir) + + if not os.path.exists(papers_dir): + logger.error(f"Papers directory not found: {papers_dir}") + # Create it for user convenience + os.makedirs(papers_dir, exist_ok=True) + logger.info(f"Created {papers_dir}. Please add PDF/MD papers there.") + return + + # Load Config + try: + config = load_config() + except Exception as e: + logger.error(f"Failed to load config: {e}") + return + + # Find papers + paper_files = glob.glob(os.path.join(papers_dir, "*.pdf")) + glob.glob(os.path.join(papers_dir, "*.md")) + paper_files.sort() + + if args.limit: + paper_files = paper_files[:args.limit] + + if not paper_files: + logger.warning("No papers found to process.") + return + + logger.info(f"Found {len(paper_files)} papers to process.") + + results = [] + run_id = time.strftime("%Y%m%d_%H%M%S") + + for paper_path in paper_files: + try: + result = run_evaluation(paper_path, output_dir, config, run_id) + results.append(result) + except Exception as e: + logger.exception(f"Critical failure processing {paper_path}: {e}") + results.append({ + "paper": Path(paper_path).stem, + "repo_path": "FAILED", + "passed": False, + "error": str(e) + }) + + # Report Summary + logger.info("=== Evaluation Summary ===") + total = len(results) + passed = sum(1 for r in results if r["passed"]) + pass_rate = (passed / total) * 100 if total > 0 else 0 + + print(f"\nTotal Papers: {total}") + print(f"Passed: {passed}") + print(f"Pass Rate: {pass_rate:.2f}%") + + for r in results: + status = "PASS" if r["passed"] else "FAIL" + print(f"[{status}] {r['paper']} -> {r['repo_path']}") + +if __name__ == "__main__": + main() diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/experiments/run_paperbench.py.backup b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/experiments/run_paperbench.py.backup new file mode 100644 index 00000000..e69de29b diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/experiments/validate_repro.py b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/experiments/validate_repro.py new file mode 100644 index 00000000..ae854fab --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/experiments/validate_repro.py @@ -0,0 +1,101 @@ +import os +import sys +import logging +import yaml +from pathlib import Path + +# Add project root to path +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from src.utils.logger import setup_logger +from src.utils.mcp_tools import DockerSandbox +from src.core.document_parser import DocumentSegmenter +from src.core.blueprint import Blueprint +from src.core.memory import CodeMem +from src.agents.planning import PlanningSwarm +from src.agents.coding import CodingSwarm +from src.agents.verification import VerificationSwarm + +logger = setup_logger("ValidateRepro") + +def load_config(): + config_path = os.path.join(os.path.dirname(__file__), "..", "config.yaml") + if not os.path.exists(config_path): + logger.error(f"Config file not found at {config_path}") + return {} + with open(config_path, "r") as f: + return yaml.safe_load(f) + +def check_docker(): + logger.info("Checking Docker Sandbox...") + try: + # We assume the image is built or will be built. + # If not built, this might take time or fail if docker is not running. + # We use a short timeout for the command. + with DockerSandbox() as sandbox: + exit_code, stdout, stderr = sandbox.execute_command("echo 'Hello from Sandbox'", timeout=10) + if exit_code == 0 and "Hello from Sandbox" in stdout: + logger.info("โœ… Docker Sandbox is working.") + else: + logger.error(f"โŒ Docker Sandbox failed. Output: {stdout}, Error: {stderr}") + except Exception as e: + logger.error(f"โŒ Docker Sandbox check failed: {e}") + logger.warning("Ensure Docker is running and the image 'deepcode_sandbox:latest' is built (or can be built).") + +def check_document_parser(): + logger.info("Checking Document Parser...") + try: + segmenter = DocumentSegmenter() + text = "# Header 1\nContent 1\n## Header 2\nContent 2" + segments = segmenter.parse_markdown(text) + if len(segments) == 2: + logger.info("โœ… Document Parser (Markdown) is working.") + else: + logger.error(f"โŒ Document Parser failed. Expected 2 segments, got {len(segments)}") + except Exception as e: + logger.error(f"โŒ Document Parser check failed: {e}") + +def check_agents_init(config): + logger.info("Checking Agent Initialization...") + try: + # Planning + PlanningSwarm(config) + logger.info("โœ… PlanningSwarm initialized.") + + # Coding + mem = CodeMem() + # We pass None for rag_engine and a dummy output dir + CodingSwarm(config, mem, None, "data/output_repos") + logger.info("โœ… CodingSwarm initialized.") + + # Verification + VerificationSwarm(config, None, "data/output_repos") + logger.info("โœ… VerificationSwarm initialized.") + + except Exception as e: + logger.error(f"โŒ Agent initialization failed: {e}") + +def main(): + logger.info("Starting DeepCode Reproduction Validation...") + + config = load_config() + if not config: + return + + # Check API Keys + llm_config = config.get("llm", {}) + if not llm_config.get("api_key"): + logger.warning("โš ๏ธ LLM API Key not found in config. Agents will fail if run.") + else: + logger.info(f"โœ… API Key present for provider: {llm_config.get('provider')}") + + check_document_parser() + check_agents_init(config) + + # Docker check might require the daemon, so we wrap it carefully + check_docker() + + logger.info("Validation Complete.") + +if __name__ == "__main__": + main() diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/main.py b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/main.py new file mode 100644 index 00000000..4921b36d --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/main.py @@ -0,0 +1,183 @@ +import os +import sys +import argparse +import yaml +import time +from pathlib import Path +from typing import Dict, Any, Optional + +# Ensure src is in python path +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + +from src.utils.logger import setup_logger, logger +from src.utils.mcp_tools import MCPToolkit, DockerSandbox +from src.core.document_parser import DocumentSegmenter +from src.core.blueprint import Blueprint +from src.core.memory import CodeMem +from src.core.rag_engine import CodeRAG +from src.agents.planning import PlanningSwarm +from src.agents.coding import CodingSwarm +from src.agents.verification import VerificationSwarm + +def load_config(config_path: str = "config.yaml") -> Dict[str, Any]: + """Load configuration from YAML file.""" + if not os.path.exists(config_path): + logger.error(f"Config file not found: {config_path}") + sys.exit(1) + + with open(config_path, "r") as f: + try: + config = yaml.safe_load(f) + return config + except yaml.YAMLError as e: + logger.error(f"Error parsing config file: {e}") + sys.exit(1) + +def run_planning_phase(paper_path: str, config: Dict[str, Any], output_dir: str) -> Blueprint: + """ + Phase 1: Planning + Parse document and generate implementation blueprint. + """ + logger.info("=== Starting Phase 1: Planning ===") + + # 1. Parse Document + logger.info(f"Parsing document: {paper_path}") + segmenter = DocumentSegmenter() + segments = segmenter.parse_file(paper_path) + + # Convert segments to full text for the planning agent + # In a real scenario, we might want to be smarter about this, + # but for now we concatenate relevant sections or pass the whole text + # depending on token limits. The PlanningSwarm expects text. + full_text = "\n\n".join([s.to_text() for s in segments]) + logger.info(f"Document parsed. Total length: {len(full_text)} characters.") + + # 2. Run Planning Swarm + planning_swarm = PlanningSwarm(config) + blueprint = planning_swarm.plan(full_text) + + # 3. Save Blueprint + blueprint_path = os.path.join(output_dir, "blueprint.json") + blueprint.save(blueprint_path) + logger.info(f"Blueprint saved to {blueprint_path}") + + return blueprint + +def run_coding_phase(blueprint: Blueprint, config: Dict[str, Any], output_dir: str, rag_engine: Optional[CodeRAG] = None): + """ + Phase 2: Coding + Generate code based on blueprint, using CodeMem and CodeRAG. + """ + logger.info("=== Starting Phase 2: Coding ===") + + # 1. Initialize Memory + memory = CodeMem() + + # 2. Initialize Coding Swarm + coding_swarm = CodingSwarm( + config=config, + memory=memory, + rag_engine=rag_engine, + output_dir=output_dir + ) + + # 3. Generate Codebase + coding_swarm.generate_codebase(blueprint) + + # 4. Save Memory State + memory_path = os.path.join(output_dir, "codemem_state.json") + memory.save(memory_path) + logger.info(f"CodeMem state saved to {memory_path}") + +def run_verification_phase(blueprint: Blueprint, config: Dict[str, Any], output_dir: str, sandbox: Optional[DockerSandbox] = None): + """ + Phase 3: Verification + Verify generated code using static analysis and sandbox execution. + """ + logger.info("=== Starting Phase 3: Verification ===") + + # 1. Initialize Toolkit with Sandbox + mcp_toolkit = None + if sandbox: + # Create toolkit with existing sandbox configuration + # The MCPToolkit constructor takes a config dict for the sandbox + # But here we might want to pass the actual sandbox instance or just the config + # Looking at MCPToolkit implementation, it creates its own DockerSandbox. + # We should pass the sandbox config. + sandbox_config = config.get("sandbox", {}) + # Ensure workspace paths match + sandbox_config["host_workspace_path"] = output_dir + mcp_toolkit = MCPToolkit(sandbox_config=sandbox_config) + + # 2. Initialize Verification Swarm + verification_swarm = VerificationSwarm( + config=config, + mcp_toolkit=mcp_toolkit, + output_dir=output_dir + ) + + # 3. Verify Codebase + verification_swarm.verify_codebase(blueprint) + +def main(): + parser = argparse.ArgumentParser(description="DeepCode: Open Agentic Coding Framework") + parser.add_argument("--paper", type=str, help="Path to input research paper (PDF/MD)") + parser.add_argument("--output", type=str, default="./data/output_repos/generated_project", help="Output directory for generated code") + parser.add_argument("--config", type=str, default="config.yaml", help="Path to configuration file") + parser.add_argument("--blueprint", type=str, help="Path to existing blueprint.json (skip planning)") + parser.add_argument("--skip-verify", action="store_true", help="Skip verification phase") + parser.add_argument("--use-rag", action="store_true", help="Enable CodeRAG (requires vector store setup)") + + args = parser.parse_args() + + # Setup Logger + setup_logger() + + # Load Config + config = load_config(args.config) + + # Prepare Output Directory + output_dir = os.path.abspath(args.output) + os.makedirs(output_dir, exist_ok=True) + + # Initialize RAG Engine if requested + rag_engine = None + if args.use_rag: + rag_config = config.get("rag", {}) + persist_dir = rag_config.get("persist_directory", "./data/chroma_db") + collection_name = rag_config.get("collection_name", "deepcode_knowledge") + rag_engine = CodeRAG(persist_directory=persist_dir, collection_name=collection_name) + # Note: In a real run, we would index documents here or assume they are indexed. + + # --- Phase 1: Planning --- + blueprint = None + if args.blueprint: + logger.info(f"Loading existing blueprint from {args.blueprint}") + blueprint = Blueprint.load(args.blueprint) + elif args.paper: + blueprint = run_planning_phase(args.paper, config, output_dir) + else: + logger.error("Either --paper or --blueprint must be provided.") + sys.exit(1) + + # --- Phase 2: Coding --- + run_coding_phase(blueprint, config, output_dir, rag_engine) + + # --- Phase 3: Verification --- + if not args.skip_verify: + # Setup Sandbox + sandbox_config = config.get("sandbox", {}) + # Override host path to point to our output directory + sandbox_config["host_workspace_path"] = output_dir + + # We use the context manager in the swarm or toolkit, but here we might want to + # ensure the docker environment is ready. + # The VerificationSwarm uses MCPToolkit which manages the sandbox. + # We just pass the config. + + run_verification_phase(blueprint, config, output_dir, sandbox=True) # Pass True to indicate we want sandbox + + logger.info("=== DeepCode Execution Complete ===") + +if __name__ == "__main__": + main() diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/requirements.txt b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/requirements.txt new file mode 100644 index 00000000..73a931f5 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/requirements.txt @@ -0,0 +1,13 @@ +pypdf>=3.0.0 +marker-pdf>=0.2.0 +chromadb>=0.4.0 +docker>=6.1.0 +pydantic>=2.0.0 +tenacity>=8.2.0 +anthropic>=0.7.0 +openai>=1.0.0 +pyyaml>=6.0 +numpy>=1.24.0 +pandas>=2.0.0 +scikit-learn>=1.2.0 +pytest>=7.0.0 diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/__init__.py b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/__init__.py new file mode 100644 index 00000000..93eaef8a --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/__init__.py @@ -0,0 +1,3 @@ +""" +DeepCode: Open Agentic Coding Framework +""" diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/agents/__init__.py b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/agents/__init__.py new file mode 100644 index 00000000..13109999 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/agents/__init__.py @@ -0,0 +1,20 @@ +from .base import BaseAgent, AgentConfig +from .planning import PlanningSwarm, ConceptAgent, AlgorithmAgent, PlannerAgent +from .coding import CodingSwarm, CodingAgent, SummarizationAgent +from .verification import VerificationSwarm, AnalysisAgent, ModificationAgent, SandboxAgent + +__all__ = [ + "BaseAgent", + "AgentConfig", + "PlanningSwarm", + "ConceptAgent", + "AlgorithmAgent", + "PlannerAgent", + "CodingSwarm", + "CodingAgent", + "SummarizationAgent", + "VerificationSwarm", + "AnalysisAgent", + "ModificationAgent", + "SandboxAgent", +] diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/agents/base.py b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/agents/base.py new file mode 100644 index 00000000..76ad634c --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/agents/base.py @@ -0,0 +1,193 @@ +import os +import yaml +import json +import time +from typing import List, Dict, Any, Optional, Union +from abc import ABC, abstractmethod +from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type + +import openai +import anthropic +from pydantic import BaseModel + +from ..utils.logger import logger +from ..utils.mcp_tools import MCPToolkit + +class AgentConfig(BaseModel): + model_provider: str # "anthropic" or "openai" + model_name: str + temperature: float = 0.0 + max_tokens: int = 4096 + api_key: Optional[str] = None + +class BaseAgent(ABC): + """ + Base class for all DeepCode agents. + Handles LLM interaction, tool execution, and retry logic. + """ + + def __init__(self, name: str, config: Dict[str, Any], mcp_toolkit: Optional[MCPToolkit] = None): + """ + Initialize the agent. + + Args: + name: Name of the agent (for logging). + config: Configuration dictionary (usually from config.yaml). + mcp_toolkit: Optional toolkit for tool execution. + """ + self.name = name + self.config = config + self.mcp_toolkit = mcp_toolkit + + # Load LLM settings + llm_cfg = config.get("llm", {}) + self.provider = llm_cfg.get("provider", "anthropic") + self.model_name = llm_cfg.get("model", "claude-3-5-sonnet-20240620") + self.temperature = llm_cfg.get("temperature", 0.0) + self.max_tokens = llm_cfg.get("max_tokens", 4096) + + # Initialize clients + self._init_client() + + logger.info(f"Agent '{self.name}' initialized with {self.provider}/{self.model_name}") + + def _init_client(self): + """Initialize the appropriate LLM client based on configuration.""" + if self.provider == "anthropic": + api_key = os.environ.get("ANTHROPIC_API_KEY") + if not api_key: + logger.warning(f"ANTHROPIC_API_KEY not found in environment for agent {self.name}") + self.client = anthropic.Anthropic(api_key=api_key) + elif self.provider == "openai": + api_key = os.environ.get("OPENAI_API_KEY") + if not api_key: + logger.warning(f"OPENAI_API_KEY not found in environment for agent {self.name}") + self.client = openai.OpenAI(api_key=api_key) + else: + raise ValueError(f"Unsupported model provider: {self.provider}") + + @retry( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=4, max=10), + retry=retry_if_exception_type(( + anthropic.APIConnectionError, + anthropic.RateLimitError, + openai.APIConnectionError, + openai.RateLimitError + )) + ) + def call_llm(self, + system_prompt: str, + user_message: str, + tools: Optional[List[Dict]] = None) -> str: + """ + Execute a call to the LLM with retry logic. + + Args: + system_prompt: The system instruction. + user_message: The user query or context. + tools: Optional list of tool definitions (if supported). + + Returns: + The text response from the LLM. + """ + logger.debug(f"Agent '{self.name}' calling LLM...") + + try: + if self.provider == "anthropic": + return self._call_anthropic(system_prompt, user_message, tools) + elif self.provider == "openai": + return self._call_openai(system_prompt, user_message, tools) + else: + raise ValueError(f"Unknown provider {self.provider}") + except Exception as e: + logger.error(f"Error calling LLM in agent '{self.name}': {str(e)}") + raise + + def _call_anthropic(self, system_prompt: str, user_message: str, tools: Optional[List[Dict]] = None) -> str: + """Handle Anthropic API call.""" + messages = [{"role": "user", "content": user_message}] + + # Note: Anthropic tool use is slightly different, but for this reproduction + # we might stick to text-based tool use or standard API if needed. + # For simplicity in this base class, we'll focus on text generation. + # If tools are provided, we would format them into the system prompt or use the tool API. + + # If tools are strictly required via API: + kwargs = { + "model": self.model_name, + "max_tokens": self.max_tokens, + "temperature": self.temperature, + "system": system_prompt, + "messages": messages + } + + # Basic tool integration if needed (Anthropic beta tools) + # For now, we assume the prompt handles tool instructions or we use the official tool param if available + # In this repro, we will rely on the prompt to guide the model to use tools if we aren't using the native tool calling API fully yet, + # OR we can implement native tool calling if the library version supports it. + # Given requirements.txt has anthropic>=0.7.0, it supports messages API. + + response = self.client.messages.create(**kwargs) + + # Extract text content + if response.content and len(response.content) > 0: + return response.content[0].text + return "" + + def _call_openai(self, system_prompt: str, user_message: str, tools: Optional[List[Dict]] = None) -> str: + """Handle OpenAI API call.""" + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_message} + ] + + kwargs = { + "model": self.model_name, + "messages": messages, + "temperature": self.temperature, + "max_tokens": self.max_tokens + } + + if tools: + kwargs["tools"] = tools + kwargs["tool_choice"] = "auto" + + response = self.client.chat.completions.create(**kwargs) + return response.choices[0].message.content + + def run_tool(self, tool_name: str, **kwargs) -> Any: + """ + Execute a tool via the MCP Toolkit. + + Args: + tool_name: Name of the tool to run (e.g., 'read_file', 'execute_command'). + **kwargs: Arguments for the tool. + + Returns: + Tool execution result. + """ + if not self.mcp_toolkit: + raise ValueError("MCP Toolkit not initialized for this agent.") + + logger.info(f"Agent '{self.name}' executing tool: {tool_name}") + + # Map string names to toolkit methods + # This is a simple dispatcher. In a full MCP implementation, this might be more dynamic. + if tool_name == "read_file": + return self.mcp_toolkit.fs.read_file(kwargs.get("path")) + elif tool_name == "write_file": + return self.mcp_toolkit.fs.write_file(kwargs.get("path"), kwargs.get("content")) + elif tool_name == "list_files": + return self.mcp_toolkit.fs.list_files(kwargs.get("path", ".")) + elif tool_name == "execute_command": + return self.mcp_toolkit.sandbox.execute_command(kwargs.get("command"), kwargs.get("timeout", 30)) + elif tool_name == "search": + return self.mcp_toolkit.search.search(kwargs.get("query"), kwargs.get("count", 5)) + else: + return f"Error: Tool '{tool_name}' not found." + + @abstractmethod + def run(self, *args, **kwargs) -> Any: + """Main execution method for the agent.""" + pass diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/agents/coding.py b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/agents/coding.py new file mode 100644 index 00000000..41a0fba3 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/agents/coding.py @@ -0,0 +1,211 @@ +import os +from typing import Dict, Any, List, Optional +from deepcode_repro.src.agents.base import BaseAgent +from deepcode_repro.src.core.blueprint import Blueprint, ComponentSpecification +from deepcode_repro.src.core.memory import CodeMem, MemoryEntry, DependencyEdges +from deepcode_repro.src.core.rag_engine import CodeRAG +from deepcode_repro.src.utils.prompts import AgentPrompts +from deepcode_repro.src.utils.logger import logger +from deepcode_repro.src.utils.mcp_tools import MCPToolkit + +class CodingAgent(BaseAgent): + """ + Agent responsible for generating implementation code for a specific file + based on the blueprint specification, memory context, and RAG knowledge. + """ + def __init__(self, config: Dict[str, Any], mcp_toolkit: Optional[MCPToolkit] = None): + super().__init__(name="CodingAgent", config=config, mcp_toolkit=mcp_toolkit) + + def run(self, + file_name: str, + spec: ComponentSpecification, + memory_context: str, + rag_context: str = "") -> str: + """ + Generates code for the target file. + """ + logger.info(f"Generating code for {file_name}...") + + # Construct the prompt + spec_str = spec.model_dump_json(indent=2) + user_message = AgentPrompts.get_coding_task( + file_name=file_name, + spec=spec_str, + context=memory_context, + rag_data=rag_context + ) + + # Call LLM + response = self.call_llm( + system_prompt=AgentPrompts.CODING_AGENT_SYSTEM, + user_message=user_message + ) + + # Extract code from markdown blocks if present + code = self._extract_code(response) + return code + + def _extract_code(self, text: str) -> str: + """Extracts code from markdown code blocks.""" + if "```python" in text: + start = text.find("```python") + 9 + end = text.find("```", start) + return text[start:end].strip() + elif "```" in text: + start = text.find("```") + 3 + end = text.find("```", start) + return text[start:end].strip() + return text.strip() + + +class SummarizationAgent(BaseAgent): + """ + Agent responsible for analyzing generated code and creating a MemoryEntry + summary for the CodeMem graph. + """ + def __init__(self, config: Dict[str, Any]): + super().__init__(name="SummarizationAgent", config=config) + + def run(self, file_path: str, code: str) -> MemoryEntry: + """ + Summarizes the code into a MemoryEntry. + """ + logger.info(f"Summarizing {file_path} for CodeMem...") + + user_message = AgentPrompts.get_summarization_task(code=code) + + response = self.call_llm( + system_prompt=AgentPrompts.SUMMARIZATION_AGENT_SYSTEM, + user_message=user_message + ) + + # Parse the response into a MemoryEntry + # The prompt asks for a JSON structure. We need to parse it. + try: + # Clean up potential markdown wrapping + json_str = response + if "```json" in response: + start = response.find("```json") + 7 + end = response.find("```", start) + json_str = response[start:end].strip() + elif "```" in response: + start = response.find("```") + 3 + end = response.find("```", start) + json_str = response[start:end].strip() + + # We expect the LLM to return a structure compatible with MemoryEntry + # However, MemoryEntry has specific fields. + # Let's assume the prompt guides the LLM to output the correct JSON structure. + # We might need to map it manually if the LLM output is slightly off, + # but for now we'll try direct parsing or construction. + + import json + data = json.loads(json_str) + + # Construct MemoryEntry + # Ensure dependency_edges is properly formatted + deps = data.get("dependency_edges", {}) + edges = DependencyEdges( + afferent=deps.get("afferent", []), + efferent=deps.get("efferent", []) + ) + + entry = MemoryEntry( + file_path=file_path, + core_purpose=data.get("core_purpose", "No purpose provided"), + public_interface=data.get("public_interface", []), # This might need more parsing if it's complex objects + dependency_edges=edges + ) + return entry + + except Exception as e: + logger.error(f"Failed to parse summary for {file_path}: {e}") + # Return a fallback entry + return MemoryEntry( + file_path=file_path, + core_purpose="Summary generation failed", + public_interface=[], + dependency_edges=DependencyEdges(afferent=[], efferent=[]) + ) + + +class CodingSwarm: + """ + Orchestrates the stateful generation loop. + """ + def __init__(self, + config: Dict[str, Any], + memory: CodeMem, + rag_engine: Optional[CodeRAG] = None, + output_dir: str = "data/output_repos"): + self.config = config + self.memory = memory + self.rag_engine = rag_engine + self.output_dir = output_dir + + self.coding_agent = CodingAgent(config) + self.summarization_agent = SummarizationAgent(config) + + # Ensure output directory exists + os.makedirs(self.output_dir, exist_ok=True) + + def generate_codebase(self, blueprint: Blueprint) -> None: + """ + Executes the generation loop for the entire blueprint. + """ + logger.info(f"Starting codebase generation for project: {blueprint.project_name}") + + # Iterate through the development plan (topological sort order) + for file_path in blueprint.dev_plan: + self._generate_file(file_path, blueprint) + + logger.info("Codebase generation complete.") + + def _generate_file(self, file_path: str, blueprint: Blueprint) -> None: + """ + Generates a single file, saves it, and updates memory. + """ + logger.info(f"Processing target file: {file_path}") + + # 1. Get Component Specification + spec = blueprint.get_spec(file_path) + if not spec: + logger.warning(f"No specification found for {file_path}. Skipping.") + return + + # 2. Select Relevant Memory + # We use the dependencies declared in the blueprint spec to filter memory + relevant_entries = self.memory.select_relevant(file_path, spec.dependencies) + memory_context = self.memory.to_context_string(relevant_entries) + + # 3. Retrieve RAG Context (if enabled) + rag_context = "" + if self.rag_engine: + # Simple heuristic: always check, or use should_retrieve + # The prompt context + target file is used to decide + if self.rag_engine.should_retrieve(spec.core_purpose, file_path): + logger.info(f"Retrieving RAG context for {file_path}") + # Query based on the spec description and purpose + query_text = f"{file_path}: {spec.core_purpose}. {spec.algorithmic_details}" + results = self.rag_engine.query(query_text, n_results=3) + # Format results + rag_context = "\n".join([f"Snippet from {r['metadata'].get('source_id', 'unknown')}:\n{r['content']}" for r in results]) + + # 4. Generate Code + code = self.coding_agent.run( + file_name=file_path, + spec=spec, + memory_context=memory_context, + rag_context=rag_context + ) + + # 5. Save Code to Disk + full_path = os.path.join(self.output_dir, file_path) + os.makedirs(os.path.dirname(full_path), exist_ok=True) + with open(full_path, "w") as f: + f.write(code) + logger.info(f"Saved {file_path}") + + # 6. Summarize and Update Memory + memory_entry = self.summarization_agent.run(file_path, code) + self.memory.add_entry(memory_entry) diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/agents/planning.py b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/agents/planning.py new file mode 100644 index 00000000..ea4e9694 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/agents/planning.py @@ -0,0 +1,114 @@ +from typing import Dict, Any, Optional +import json +import re + +from deepcode_repro.src.agents.base import BaseAgent +from deepcode_repro.src.core.blueprint import Blueprint +from deepcode_repro.src.utils.prompts import AgentPrompts +from deepcode_repro.src.utils.logger import logger + +class ConceptAgent(BaseAgent): + """ + Sub-agent responsible for extracting high-level logic and conceptual architecture + from the research paper. + """ + def run(self, paper_text: str) -> str: + logger.info(f"[{self.name}] Extracting conceptual schema...") + system_prompt = AgentPrompts.CONCEPT_AGENT_SYSTEM + # We limit paper text if it's too huge, but usually context windows are large enough now. + # For safety, we might truncate if needed, but let's assume it fits or is handled by the LLM provider. + user_message = f"Analyze the following research paper and describe the high-level conceptual architecture, core components, and data flow. Focus on the 'what' and 'why'.\n\nPaper Content:\n{paper_text}" + return self.call_llm(system_prompt, user_message) + +class AlgorithmAgent(BaseAgent): + """ + Sub-agent responsible for extracting mathematical formulas, algorithms, and pseudocode. + """ + def run(self, paper_text: str) -> str: + logger.info(f"[{self.name}] Extracting algorithmic schema...") + system_prompt = AgentPrompts.ALGORITHM_AGENT_SYSTEM + user_message = f"Analyze the following research paper and extract all mathematical formulations, algorithms, and pseudocode. Provide step-by-step logic and use verbatim LaTeX for equations. Focus on the 'how'.\n\nPaper Content:\n{paper_text}" + return self.call_llm(system_prompt, user_message) + +class PlannerAgent(BaseAgent): + """ + Sub-agent responsible for merging schemas and synthesizing the Blueprint JSON. + """ + def run(self, paper_text: str, concept_schema: str, algo_schema: str) -> Blueprint: + logger.info(f"[{self.name}] Synthesizing Blueprint...") + system_prompt = AgentPrompts.PLANNING_AGENT_SYSTEM + + # Construct the synthesis task + user_message = f""" + Synthesize the Implementation Blueprint based on the extracted schemas. + + --- ORIGINAL PAPER (Excerpt) --- + {paper_text[:4000]}... [Truncated for context] + + --- CONCEPTUAL SCHEMA --- + {concept_schema} + + --- ALGORITHMIC SCHEMA --- + {algo_schema} + + Generate the full JSON Blueprint containing 'project_name', 'file_hierarchy', 'component_specs', and 'dev_plan'. + Ensure the JSON is valid and matches the schema definitions provided in the system prompt. + Return ONLY the JSON object. + """ + + response = self.call_llm(system_prompt, user_message) + + # Clean and parse JSON + try: + # Strip markdown code blocks if present + clean_json = response.strip() + if "```json" in clean_json: + clean_json = clean_json.split("```json")[1].split("```")[0].strip() + elif "```" in clean_json: + clean_json = clean_json.split("```")[1].split("```")[0].strip() + + # Attempt to parse + blueprint_data = json.loads(clean_json) + + # Validate and convert to Blueprint object + # We use from_json which expects a string, or we can construct directly if we have the dict + # Blueprint.from_json expects a JSON string. + return Blueprint.from_json(json.dumps(blueprint_data)) + + except json.JSONDecodeError as e: + logger.error(f"Failed to parse Blueprint JSON: {e}") + logger.debug(f"Raw response: {response}") + raise ValueError("LLM failed to generate valid JSON for Blueprint") from e + except Exception as e: + logger.error(f"Error creating Blueprint object: {e}") + raise e + +class PlanningSwarm: + """ + Orchestrates the Concept, Algorithm, and Planning agents to produce a repository blueprint. + """ + def __init__(self, config: Dict[str, Any]): + self.config = config + self.concept_agent = ConceptAgent("ConceptAgent", config) + self.algorithm_agent = AlgorithmAgent("AlgorithmAgent", config) + self.planner_agent = PlannerAgent("PlanningAgent", config) + + def plan(self, paper_text: str) -> Blueprint: + """ + Executes the planning pipeline: + 1. Extract Conceptual Schema + 2. Extract Algorithmic Schema + 3. Synthesize Blueprint + """ + logger.info("Starting Planning Swarm...") + + # Phase 1: Parallel Extraction (Sequential here for simplicity) + # In a real async system, these could run in parallel + concept_schema = self.concept_agent.run(paper_text) + algo_schema = self.algorithm_agent.run(paper_text) + + # Phase 2: Synthesis + blueprint = self.planner_agent.run(paper_text, concept_schema, algo_schema) + + logger.info(f"Blueprint synthesis complete for project: {blueprint.project_name}") + return blueprint diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/agents/planning.py.backup b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/agents/planning.py.backup new file mode 100644 index 00000000..e69de29b diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/agents/verification.py b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/agents/verification.py new file mode 100644 index 00000000..0be24972 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/agents/verification.py @@ -0,0 +1,256 @@ +import os +import re +from typing import Dict, Any, Optional, List, Tuple +from deepcode_repro.src.agents.base import BaseAgent +from deepcode_repro.src.core.blueprint import Blueprint, ComponentSpecification +from deepcode_repro.src.utils.prompts import AgentPrompts +from deepcode_repro.src.utils.logger import logger +from deepcode_repro.src.utils.mcp_tools import MCPToolkit + +class AnalysisAgent(BaseAgent): + """ + Agent responsible for static analysis of generated code against the blueprint specification. + """ + def __init__(self, config: Dict[str, Any]): + super().__init__(name="AnalysisAgent", config=config) + self.system_prompt = AgentPrompts.ANALYSIS_AGENT_SYSTEM + + def run(self, code: str, spec: ComponentSpecification) -> Optional[str]: + """ + Analyzes the code and returns a report if issues are found, or None if clean. + """ + spec_str = f"Purpose: {spec.core_purpose}\nInterface: {spec.public_interface}" + task = AgentPrompts.get_analysis_task(code, spec_str) + + response = self.call_llm(self.system_prompt, task) + + # Heuristic: If response contains "NO ISSUES FOUND" (case insensitive), return None + if "NO ISSUES FOUND" in response.upper(): + return None + + return response + +class ModificationAgent(BaseAgent): + """ + Agent responsible for applying fixes to code based on feedback (static analysis or runtime errors). + """ + def __init__(self, config: Dict[str, Any]): + super().__init__(name="ModificationAgent", config=config) + self.system_prompt = AgentPrompts.MODIFICATION_AGENT_SYSTEM + + def run(self, code: str, feedback: str) -> str: + """ + Generates corrected code based on the provided feedback. + """ + task = AgentPrompts.get_modification_task(code, feedback) + response = self.call_llm(self.system_prompt, task) + + # Extract code block + code_match = re.search(r"```python\n(.*?)```", response, re.DOTALL) + if code_match: + return code_match.group(1) + + # Fallback: if no block, assume full response is code (risky but necessary) + # or try to find just ``` ... ``` + code_match_generic = re.search(r"```\n(.*?)```", response, re.DOTALL) + if code_match_generic: + return code_match_generic.group(1) + + return response + +class SandboxAgent(BaseAgent): + """ + Agent responsible for analyzing runtime traces and diagnosing errors. + Note: The actual execution happens via the VerificationSwarm using MCP tools. + This agent acts as the 'Brain' analyzing the 'Trace'. + """ + def __init__(self, config: Dict[str, Any]): + super().__init__(name="SandboxAgent", config=config) + self.system_prompt = AgentPrompts.SANDBOX_AGENT_SYSTEM + + def analyze_trace(self, code: str, trace: str) -> str: + """ + Analyzes the execution trace and returns fix instructions. + """ + # We reuse the modification task prompt structure or create a specific one. + # The prompt template for modification takes 'feedback', which can be the trace analysis. + # But here we want to produce the analysis/instructions for the ModificationAgent. + + # Let's construct a specific prompt for diagnosis + task = f""" + Analyze the following execution trace for the provided code. + Identify the root cause of the error and provide specific instructions to fix it. + + CODE: + ```python + {code} + ``` + + EXECUTION TRACE: + {trace} + + OUTPUT: + Provide a concise set of fix instructions. + """ + + response = self.call_llm(self.system_prompt, task) + return response + +class VerificationSwarm: + """ + Orchestrates the verification loop: Static Analysis -> Dynamic Execution -> Self-Correction. + """ + def __init__(self, config: Dict[str, Any], mcp_toolkit: Optional[MCPToolkit], output_dir: str): + self.config = config + self.mcp_toolkit = mcp_toolkit + self.output_dir = output_dir + + self.analysis_agent = AnalysisAgent(config) + self.modification_agent = ModificationAgent(config) + self.sandbox_agent = SandboxAgent(config) + + self.max_retries = 3 + + def verify_file_static(self, file_path: str, spec: ComponentSpecification) -> bool: + """ + Performs static analysis and correction loop. + Returns True if verified (or fixed), False if issues persist. + """ + full_path = os.path.join(self.output_dir, file_path) + if not os.path.exists(full_path): + logger.error(f"File not found for verification: {full_path}") + return False + + logger.info(f"Starting Static Verification for {file_path}") + + for i in range(self.max_retries + 1): + # Read current code + try: + with open(full_path, 'r') as f: + code = f.read() + except Exception as e: + logger.error(f"Failed to read file {full_path}: {e}") + return False + + # Analyze + report = self.analysis_agent.run(code, spec) + + if report is None: + logger.info(f"Static Analysis passed for {file_path}") + return True + + logger.warning(f"Static Analysis issues found in {file_path} (Attempt {i+1}/{self.max_retries + 1})") + logger.debug(f"Report: {report}") + + if i == self.max_retries: + logger.error(f"Max retries reached for static verification of {file_path}") + return False + + # Fix + logger.info(f"Applying fixes to {file_path}...") + new_code = self.modification_agent.run(code, report) + + # Save fixed code + try: + with open(full_path, 'w') as f: + f.write(new_code) + except Exception as e: + logger.error(f"Failed to write fixed code to {full_path}: {e}") + return False + + return False + + def verify_file_dynamic(self, file_path: str, test_command: str = None) -> bool: + """ + Performs dynamic execution and correction loop in the sandbox. + If test_command is not provided, tries to run the file directly (python file.py). + """ + if not self.mcp_toolkit: + logger.warning("No MCP Toolkit provided. Skipping dynamic verification.") + return True + + logger.info(f"Starting Dynamic Verification for {file_path}") + + # Determine command + if test_command: + cmd = test_command + else: + # Default to running the file + cmd = f"python {file_path}" + + for i in range(self.max_retries + 1): + # Execute in Sandbox + logger.info(f"Executing: {cmd} (Attempt {i+1})") + + # We use the sandbox tool directly + # The sandbox expects paths relative to the container workspace + # Assuming the output_dir is mounted to /workspace + + exit_code, stdout, stderr = self.mcp_toolkit.sandbox.execute_command(cmd) + + if exit_code == 0: + logger.info(f"Dynamic Verification passed for {file_path}") + return True + + trace = f"STDOUT:\n{stdout}\n\nSTDERR:\n{stderr}" + logger.warning(f"Execution failed for {file_path}. Exit code: {exit_code}") + + if i == self.max_retries: + logger.error(f"Max retries reached for dynamic verification of {file_path}") + return False + + # Read current code for context + full_path = os.path.join(self.output_dir, file_path) + try: + with open(full_path, 'r') as f: + code = f.read() + except Exception: + logger.error(f"Could not read file {full_path} for dynamic fix.") + return False + + # Analyze Trace + fix_instructions = self.sandbox_agent.analyze_trace(code, trace) + + # Apply Fix + logger.info(f"Applying dynamic fixes to {file_path}...") + new_code = self.modification_agent.run(code, fix_instructions) + + # Save fixed code + try: + with open(full_path, 'w') as f: + f.write(new_code) + except Exception as e: + logger.error(f"Failed to write fixed code to {full_path}: {e}") + return False + + return False + + def verify_codebase(self, blueprint: Blueprint): + """ + Runs verification on all files in the blueprint. + """ + logger.info("Starting Codebase Verification Phase") + + # 1. Static Pass on all files + for filename in blueprint.dev_plan: + spec = blueprint.get_spec(filename) + if spec: + self.verify_file_static(filename, spec) + + # 2. Dynamic Pass + # Ideally, we run a test suite. If no test suite, we try to run main files. + # For this reproduction, we'll try to run files that look like scripts or tests. + + # Check for explicit test files in the plan + test_files = [f for f in blueprint.dev_plan if "test" in f.lower() or "bench" in f.lower()] + + if test_files: + logger.info(f"Found test files: {test_files}. Running dynamic verification on them.") + for tf in test_files: + self.verify_file_dynamic(tf) + else: + logger.info("No explicit test files found. Attempting to run main entry points.") + # Try to find main.py or similar + main_files = [f for f in blueprint.dev_plan if "main" in f.lower()] + for mf in main_files: + self.verify_file_dynamic(mf) diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/core/__init__.py b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/core/__init__.py new file mode 100644 index 00000000..f490ceba --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/core/__init__.py @@ -0,0 +1,4 @@ +from .document_parser import DocumentSegmenter, Segment +from .blueprint import Blueprint, ComponentSpecification, FunctionSignature, ClassSignature +from .memory import CodeMem, MemoryEntry +from .rag_engine import CodeRAG diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/core/blueprint.py b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/core/blueprint.py new file mode 100644 index 00000000..b0d243f8 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/core/blueprint.py @@ -0,0 +1,70 @@ +from typing import List, Dict, Optional, Union, Any +from pydantic import BaseModel, Field +import json + +class FunctionSignature(BaseModel): + """Represents a function signature in the public interface.""" + name: str + args: List[str] + return_type: str + description: Optional[str] = None + +class ClassSignature(BaseModel): + """Represents a class structure in the public interface.""" + name: str + methods: List[FunctionSignature] = [] + description: Optional[str] = None + +class ComponentSpecification(BaseModel): + """ + Detailed specification for a single file/component. + Merged result of Concept Agent and Algorithm Agent. + """ + filename: str + core_purpose: str = Field(..., description="High-level logic and responsibility of this component") + algorithmic_details: Optional[str] = Field(None, description="Verbatim LaTeX equations, pseudocode, or step-by-step logic") + dependencies: List[str] = Field(default_factory=list, description="List of other files or external libraries this component depends on") + public_interface: List[Union[FunctionSignature, ClassSignature]] = Field(default_factory=list, description="Expected classes and functions") + implementation_notes: Optional[str] = Field(None, description="Specific instructions for the coding agent") + +class FileNode(BaseModel): + """Represents a node in the file hierarchy tree.""" + name: str + type: str = Field(..., pattern="^(file|directory)$") + children: Optional[List['FileNode']] = None + description: Optional[str] = None + +class Blueprint(BaseModel): + """ + The Master Blueprint (B) synthesized by the Planning Agent. + Contains the architecture, detailed specs, and execution plan. + """ + project_name: str + file_hierarchy: List[FileNode] = Field(..., description="Tree structure of the repository") + component_specs: Dict[str, ComponentSpecification] = Field(..., description="Map of filename to detailed specification") + dev_plan: List[str] = Field(..., description="Ordered list of filenames to generate (topological sort)") + + def to_json(self) -> str: + """Serialize blueprint to JSON string.""" + return self.model_dump_json(indent=2) + + @classmethod + def from_json(cls, json_str: str) -> 'Blueprint': + """Load blueprint from JSON string.""" + return cls.model_validate_json(json_str) + + def get_spec(self, filename: str) -> Optional[ComponentSpecification]: + """Retrieve specification for a specific file.""" + return self.component_specs.get(filename) + + def save(self, path: str): + """Save blueprint to a file.""" + with open(path, 'w') as f: + f.write(self.to_json()) + + @classmethod + def load(cls, path: str) -> 'Blueprint': + """Load blueprint from a file.""" + with open(path, 'r') as f: + content = f.read() + return cls.from_json(content) diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/core/document_parser.py b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/core/document_parser.py new file mode 100644 index 00000000..d335cfc5 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/core/document_parser.py @@ -0,0 +1,192 @@ +import re +import os +from typing import List, Dict, Optional, Tuple, Any +from dataclasses import dataclass, field +from pathlib import Path + +from ..utils.logger import logger + +# Optional imports for PDF handling +try: + import pypdf +except ImportError: + pypdf = None + logger.warning("pypdf not installed. PDF parsing will be limited.") + +@dataclass +class Segment: + """ + Represents a semantic segment of a document. + """ + header: str + content: str + level: int + metadata: Dict[str, Any] = field(default_factory=dict) + + def to_text(self) -> str: + """Returns the full text representation of the segment.""" + return f"{'#' * self.level} {self.header}\n{self.content}" + +class DocumentSegmenter: + """ + Implements Hierarchical Content Segmentation (Algo 1). + Parses documents into structured chunks (Header, Content) to preserve semantic context. + """ + + def __init__(self, use_marker: bool = True): + self.use_marker = use_marker + + def parse_file(self, file_path: str) -> List[Segment]: + """ + Entry point to parse a file (PDF or MD) into segments. + """ + path = Path(file_path) + if not path.exists(): + logger.error(f"File not found: {file_path}") + raise FileNotFoundError(f"File not found: {file_path}") + + if path.suffix.lower() == '.pdf': + return self.parse_pdf(str(path)) + elif path.suffix.lower() in ['.md', '.markdown', '.txt']: + with open(path, 'r', encoding='utf-8') as f: + text = f.read() + return self.parse_markdown(text) + else: + logger.warning(f"Unsupported file format: {path.suffix}. Treating as text.") + with open(path, 'r', encoding='utf-8') as f: + text = f.read() + return self.parse_markdown(text) + + def parse_pdf(self, pdf_path: str) -> List[Segment]: + """ + Converts PDF to Markdown and segments it. + Prioritizes 'marker-pdf' for high-fidelity conversion if configured. + Falls back to pypdf text extraction. + """ + markdown_text = "" + + # Strategy 1: Try marker-pdf (simulated integration as it's a heavy ML dependency) + # In a real deployment, we would import marker or subprocess it. + # For this reproduction, we check if we can actually run it, otherwise fallback. + if self.use_marker: + try: + # Placeholder for actual marker-pdf call + # from marker.convert import convert_single_pdf + # full_text, images, out_meta = convert_single_pdf(pdf_path, model, metadata) + # markdown_text = full_text + + # Since we can't easily install heavy ML libs in this environment, + # we will fallback to pypdf but structure the text to look like markdown headers + # if possible, or just treat it as raw text. + logger.info("marker-pdf integration is a placeholder. Falling back to pypdf.") + pass + except Exception as e: + logger.warning(f"Failed to use marker-pdf: {e}") + + # Strategy 2: pypdf Fallback + if not markdown_text and pypdf: + try: + reader = pypdf.PdfReader(pdf_path) + text_parts = [] + for page in reader.pages: + text_parts.append(page.extract_text()) + + # Simple heuristic to create structure from raw text + # This is much worse than marker-pdf but allows the code to run without ML deps + raw_text = "\n".join(text_parts) + markdown_text = self._heuristic_pdf_to_markdown(raw_text) + except Exception as e: + logger.error(f"Failed to parse PDF with pypdf: {e}") + return [] + + return self.parse_markdown(markdown_text) + + def _heuristic_pdf_to_markdown(self, text: str) -> str: + """ + A simple heuristic to convert raw PDF text to pseudo-markdown. + It assumes lines that are short and all caps or title case might be headers. + """ + lines = text.split('\n') + md_lines = [] + for line in lines: + stripped = line.strip() + if not stripped: + md_lines.append("") + continue + + # Heuristic: Short lines (<60 chars) that don't end in punctuation might be headers + # This is very rough and intended as a fallback. + if len(stripped) < 60 and not stripped[-1] in ".,:;": + # Check if it looks like a title + if stripped.isupper() or stripped.istitle(): + md_lines.append(f"## {stripped}") + else: + md_lines.append(stripped) + else: + md_lines.append(stripped) + + return "\n".join(md_lines) + + def parse_markdown(self, text: str) -> List[Segment]: + """ + Parses Markdown text into hierarchical segments. + Algorithm: + 1. Identify headers (#, ##, etc.) + 2. Group content under the preceding header. + 3. Maintain a stack or list of segments. + """ + segments: List[Segment] = [] + lines = text.split('\n') + + current_header = "Root" + current_level = 0 + current_content = [] + + # Regex to match markdown headers + header_pattern = re.compile(r'^(#{1,6})\s+(.*)') + + for line in lines: + match = header_pattern.match(line) + if match: + # If we have accumulated content for the previous header, save it + if current_content or current_header != "Root": + segments.append(Segment( + header=current_header, + content="\n".join(current_content).strip(), + level=current_level + )) + + # Start new segment + hashes, title = match.groups() + current_level = len(hashes) + current_header = title.strip() + current_content = [] + else: + current_content.append(line) + + # Append the last segment + if current_content or current_header != "Root": + segments.append(Segment( + header=current_header, + content="\n".join(current_content).strip(), + level=current_level + )) + + return segments + +if __name__ == "__main__": + # Simple test + parser = DocumentSegmenter() + sample_md = """ +# Introduction +This is the intro. + +## Background +Some background info. + +# Methodology +We used X and Y. + """ + segs = parser.parse_markdown(sample_md) + for s in segs: + print(f"[{s.level}] {s.header}: {s.content[:20]}...") diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/core/document_parser.py.backup b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/core/document_parser.py.backup new file mode 100644 index 00000000..e69de29b diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/core/memory.py b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/core/memory.py new file mode 100644 index 00000000..8d2d5731 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/core/memory.py @@ -0,0 +1,124 @@ +import json +from typing import List, Dict, Set, Optional, Any, Union +from pydantic import BaseModel, Field +from deepcode_repro.src.core.blueprint import FunctionSignature, ClassSignature +from deepcode_repro.src.utils.logger import logger + +class DependencyEdges(BaseModel): + """ + Represents the dependency graph edges for a specific file. + """ + afferent: List[str] = Field(default_factory=list, description="Incoming dependencies: Files that import this file") + efferent: List[str] = Field(default_factory=list, description="Outgoing dependencies: Files that this file imports") + +class MemoryEntry(BaseModel): + """ + Represents the summarized state of a generated file (m_t). + """ + file_path: str + core_purpose: str + public_interface: List[Union[FunctionSignature, ClassSignature]] = Field(default_factory=list) + dependency_edges: DependencyEdges = Field(default_factory=DependencyEdges) + implementation_summary: str = Field(default="", description="High-level summary of the implementation details") + + def to_text(self) -> str: + """ + Converts the memory entry to a text representation for LLM context. + """ + sigs = [] + for sig in self.public_interface: + if isinstance(sig, FunctionSignature): + args_str = ", ".join([f"{arg['name']}: {arg['type']}" for arg in sig.args]) + sigs.append(f"Function: {sig.name}({args_str}) -> {sig.return_type}") + elif isinstance(sig, ClassSignature): + methods_str = ", ".join([m.name for m in sig.methods]) + sigs.append(f"Class: {sig.name} [Methods: {methods_str}]") + + interface_str = "\n ".join(sigs) if sigs else "None" + + return ( + f"File: {self.file_path}\n" + f"Purpose: {self.core_purpose}\n" + f"Interface:\n {interface_str}\n" + f"Dependencies (Imports): {', '.join(self.dependency_edges.efferent)}\n" + ) + +class CodeMem(BaseModel): + """ + Manages the global state of the generated codebase (CodeMem). + """ + entries: Dict[str, MemoryEntry] = Field(default_factory=dict) + + def add_entry(self, entry: MemoryEntry): + """ + Update M_t = M_{t-1} U {entry} + """ + self.entries[entry.file_path] = entry + logger.info(f"Updated CodeMem with entry for: {entry.file_path}") + + # Update afferent edges for files that this file imports + for dep in entry.dependency_edges.efferent: + if dep in self.entries: + if entry.file_path not in self.entries[dep].dependency_edges.afferent: + self.entries[dep].dependency_edges.afferent.append(entry.file_path) + + def get_entry(self, file_path: str) -> Optional[MemoryEntry]: + return self.entries.get(file_path) + + def select_relevant(self, target_file: str, declared_dependencies: List[str]) -> List[MemoryEntry]: + """ + SelectRelevantMemory(M, target_file): Return subset of M where dependency exists. + + Args: + target_file: The file currently being generated. + declared_dependencies: List of files the target_file is known to depend on (from Blueprint). + + Returns: + List of MemoryEntry objects relevant to the target_file. + """ + relevant_memory = [] + + # 1. Direct dependencies (Explicitly declared in Blueprint) + for dep_path in declared_dependencies: + if dep_path in self.entries: + relevant_memory.append(self.entries[dep_path]) + else: + logger.warning(f"Dependency {dep_path} for {target_file} not found in CodeMem.") + + # 2. Implicit dependencies (Files that might depend on what we are building - context awareness) + # Usually we care about what we import (efferent), which is covered above. + # Sometimes we might care about files that will import us (afferent) to match their expectations, + # but usually those files haven't been generated yet if we follow topological sort. + + # 3. Global context (optional: add core utils if they exist and are not explicitly listed) + # For now, we stick to the explicit dependencies to minimize context window usage. + + return relevant_memory + + def to_context_string(self, relevant_entries: List[MemoryEntry]) -> str: + """ + Formats a list of memory entries into a context string for the LLM. + """ + if not relevant_entries: + return "No existing code context available." + + context_parts = ["--- EXISTING CODE CONTEXT ---"] + for entry in relevant_entries: + context_parts.append(entry.to_text()) + + return "\n\n".join(context_parts) + + def save(self, path: str): + """Save CodeMem state to JSON file.""" + with open(path, 'w') as f: + f.write(self.model_dump_json(indent=2)) + + @classmethod + def load(cls, path: str) -> 'CodeMem': + """Load CodeMem state from JSON file.""" + try: + with open(path, 'r') as f: + data = json.load(f) + return cls(**data) + except FileNotFoundError: + return cls() diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/core/rag_engine.py b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/core/rag_engine.py new file mode 100644 index 00000000..2388dae7 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/core/rag_engine.py @@ -0,0 +1,160 @@ +import os +import chromadb +from chromadb.config import Settings +from typing import List, Dict, Optional, Any, Tuple +from deepcode_repro.src.utils.logger import logger +from deepcode_repro.src.core.document_parser import Segment + +class CodeRAG: + """ + CodeRAG Engine: Handles indexing and retrieval of semantic context. + + Implements Component 4: CodeRAG Engine. + Purpose: Conditional knowledge injection for underspecified designs. + """ + + def __init__(self, persist_directory: str = "./data/chroma_db", collection_name: str = "deepcode_context"): + """ + Initialize the CodeRAG engine with a ChromaDB client. + + Args: + persist_directory: Path to store the vector database. + collection_name: Name of the collection to use. + """ + self.persist_directory = persist_directory + self.collection_name = collection_name + + # Ensure directory exists + os.makedirs(persist_directory, exist_ok=True) + + try: + self.client = chromadb.PersistentClient(path=persist_directory) + self.collection = self.client.get_or_create_collection(name=collection_name) + logger.info(f"CodeRAG initialized with collection '{collection_name}' at '{persist_directory}'") + except Exception as e: + logger.error(f"Failed to initialize ChromaDB: {e}") + self.client = None + self.collection = None + + def index_segments(self, segments: List[Segment], source_id: str = "doc"): + """ + Index document segments into the vector store. + + Args: + segments: List of Segment objects from DocumentSegmenter. + source_id: Identifier for the source document (e.g., filename). + """ + if not self.collection: + logger.warning("CodeRAG collection not initialized. Skipping indexing.") + return + + ids = [] + documents = [] + metadatas = [] + + for i, seg in enumerate(segments): + # Create a unique ID for each segment + seg_id = f"{source_id}_seg_{i}" + + # Prepare metadata + meta = { + "source": source_id, + "header": seg.header, + "level": seg.level, + "index": i + } + # Add any existing metadata from the segment + if seg.metadata: + for k, v in seg.metadata.items(): + if isinstance(v, (str, int, float, bool)): + meta[k] = v + + ids.append(seg_id) + documents.append(seg.to_text()) + metadatas.append(meta) + + if ids: + try: + self.collection.add( + ids=ids, + documents=documents, + metadatas=metadatas + ) + logger.info(f"Indexed {len(ids)} segments from {source_id}") + except Exception as e: + logger.error(f"Failed to index segments: {e}") + + def query(self, query_text: str, n_results: int = 3, filter_metadata: Optional[Dict] = None) -> List[Dict[str, Any]]: + """ + Retrieve relevant segments for a given query. + + Args: + query_text: The search query (e.g., code snippet or description). + n_results: Number of results to return. + filter_metadata: Optional dictionary to filter results by metadata. + + Returns: + List of dictionaries containing 'content', 'metadata', and 'distance'. + """ + if not self.collection: + logger.warning("CodeRAG collection not initialized. Returning empty results.") + return [] + + try: + results = self.collection.query( + query_texts=[query_text], + n_results=n_results, + where=filter_metadata + ) + + # Parse results into a cleaner format + parsed_results = [] + if results['ids']: + # Chroma returns lists of lists (one list per query) + ids = results['ids'][0] + docs = results['documents'][0] + metas = results['metadatas'][0] + distances = results['distances'][0] if results['distances'] else [0.0] * len(ids) + + for id_, doc, meta, dist in zip(ids, docs, metas, distances): + parsed_results.append({ + "id": id_, + "content": doc, + "metadata": meta, + "distance": dist + }) + + logger.info(f"Retrieved {len(parsed_results)} results for query: '{query_text[:50]}...'") + return parsed_results + + except Exception as e: + logger.error(f"Failed to query CodeRAG: {e}") + return [] + + def clear_index(self): + """Reset the collection.""" + if self.client: + try: + self.client.delete_collection(self.collection_name) + self.collection = self.client.get_or_create_collection(self.collection_name) + logger.info("CodeRAG index cleared.") + except Exception as e: + logger.error(f"Failed to clear index: {e}") + + def should_retrieve(self, context_text: str, target_file: str) -> bool: + """ + Determine if retrieval is necessary (Adaptive Retrieval Step 1). + + Algorithm: r_t = delta(Context, target_file) + + Note: In a full implementation, this might use an LLM to decide. + Here we implement a heuristic: if the target file description or context + contains keywords indicating external dependencies or complex logic + that might be in the knowledge base, return True. + + For now, we default to True if the query is substantial, + or we can make it always True for this reproduction to ensure RAG is used. + """ + # Simple heuristic: always retrieve if we have a target file + # In a real agentic loop, the agent might decide this. + return bool(target_file) diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/utils/logger.py b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/utils/logger.py new file mode 100644 index 00000000..87908a47 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/utils/logger.py @@ -0,0 +1,59 @@ +import logging +import sys +import os +from datetime import datetime + +def setup_logger(name: str = "DeepCode", log_level: int = logging.INFO, log_dir: str = "logs") -> logging.Logger: + """ + Sets up a logger with both console and file handlers. + + Args: + name: Name of the logger + log_level: Logging level (default: logging.INFO) + log_dir: Directory to store log files + + Returns: + Configured logger instance + """ + # Create logger + logger = logging.getLogger(name) + logger.setLevel(log_level) + + # Prevent adding handlers multiple times if logger is already configured + if logger.handlers: + return logger + + # Create formatters + console_formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s', + datefmt='%H:%M:%S' + ) + + file_formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s' + ) + + # Create console handler + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(log_level) + console_handler.setFormatter(console_formatter) + logger.addHandler(console_handler) + + # Create file handler if log_dir is provided + if log_dir: + try: + os.makedirs(log_dir, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = os.path.join(log_dir, f"{name.lower()}_{timestamp}.log") + + file_handler = logging.FileHandler(log_file) + file_handler.setLevel(log_level) + file_handler.setFormatter(file_formatter) + logger.addHandler(file_handler) + except Exception as e: + print(f"Failed to setup file logging: {e}") + + return logger + +# Default logger instance +logger = setup_logger() diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/utils/mcp_tools.py b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/utils/mcp_tools.py new file mode 100644 index 00000000..06837df5 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/utils/mcp_tools.py @@ -0,0 +1,254 @@ +import os +import logging +import docker +import requests +import shutil +from typing import Optional, Dict, Any, List, Tuple, Union +from pathlib import Path +import json +import time + +# Try to import logger, fallback if not available +try: + from src.utils.logger import logger +except ImportError: + import logging + logger = logging.getLogger("DeepCode") + logging.basicConfig(level=logging.INFO) + +class DockerSandbox: + """ + Manages a Docker container for secure execution of generated code. + Ensures that code runs in an isolated environment with access only to the workspace. + """ + def __init__(self, + image: str = "deepcode_sandbox:latest", + host_workspace_path: str = "./data/output_repos", + container_workspace_path: str = "/workspace", + auto_remove: bool = True): + """ + Initialize the Docker Sandbox. + + Args: + image: Docker image to use. + host_workspace_path: Path on host to mount. + container_workspace_path: Path inside container where host path is mounted. + auto_remove: Whether to remove the container when stopped. + """ + self.image = image + self.host_workspace_path = os.path.abspath(host_workspace_path) + self.container_workspace_path = container_workspace_path + self.auto_remove = auto_remove + self.client = docker.from_env() + self.container = None + + # Ensure host workspace exists + os.makedirs(self.host_workspace_path, exist_ok=True) + + def start(self): + """Start the sandbox container.""" + try: + # Check if image exists, if not try to pull or build (simplified to check) + try: + self.client.images.get(self.image) + except docker.errors.ImageNotFound: + logger.warning(f"Image {self.image} not found. Attempting to build from ./docker context if available or pull.") + # Logic to build or pull could go here. For now, we assume it exists or will be built by setup. + # In a real scenario, we might trigger a build here. + pass + + logger.info(f"Starting sandbox container with image {self.image}...") + self.container = self.client.containers.run( + self.image, + command="tail -f /dev/null", # Keep alive + detach=True, + volumes={self.host_workspace_path: {'bind': self.container_workspace_path, 'mode': 'rw'}}, + working_dir=self.container_workspace_path, + auto_remove=self.auto_remove, + network_mode="bridge" # Allow some network for pip install if needed, or 'none' for strict isolation + ) + logger.info(f"Sandbox container started: {self.container.id[:10]}") + except Exception as e: + logger.error(f"Failed to start sandbox: {e}") + raise + + def stop(self): + """Stop the sandbox container.""" + if self.container: + try: + logger.info("Stopping sandbox container...") + self.container.stop() + self.container = None + except Exception as e: + logger.error(f"Error stopping sandbox: {e}") + + def execute_command(self, command: str, timeout: int = 30) -> Tuple[int, str, str]: + """ + Execute a command inside the sandbox. + + Args: + command: Shell command to execute. + timeout: Execution timeout in seconds. + + Returns: + Tuple of (exit_code, stdout, stderr) + """ + if not self.container: + raise RuntimeError("Sandbox not started. Call start() first.") + + logger.debug(f"Executing in sandbox: {command}") + try: + # docker exec_run doesn't support timeout natively in the same way as subprocess + # but we can implement it if needed. For now, simple exec. + exec_result = self.container.exec_run( + cmd=f"bash -c '{command}'", + workdir=self.container_workspace_path + ) + + exit_code = exec_result.exit_code + output = exec_result.output.decode('utf-8', errors='replace') + + # Split stdout/stderr roughly (Docker API combines them usually unless tty=False and stream=True) + # For simple usage, we return output as stdout and empty stderr unless we parse it. + # To get separate streams, we'd need to use socket attachment. + # For this reproduction, combined output is often sufficient or we assume stdout. + + return exit_code, output, "" + except Exception as e: + logger.error(f"Command execution failed: {e}") + return -1, "", str(e) + + def __enter__(self): + self.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.stop() + + +class FileSystemTool: + """ + Provides safe filesystem access restricted to a specific root directory. + Used by agents to read/write code files on the host (which are then mounted to sandbox). + """ + def __init__(self, root_path: str = "./data/output_repos"): + self.root_path = os.path.abspath(root_path) + os.makedirs(self.root_path, exist_ok=True) + + def _validate_path(self, path: str) -> Path: + """Ensure path is within root_path.""" + full_path = (Path(self.root_path) / path).resolve() + if not str(full_path).startswith(self.root_path): + raise ValueError(f"Access denied: Path {path} is outside sandbox root {self.root_path}") + return full_path + + def list_files(self, path: str = ".") -> List[str]: + """List files in a directory relative to root.""" + target_path = self._validate_path(path) + if not target_path.exists(): + return [] + + files = [] + for p in target_path.rglob("*"): + if p.is_file(): + files.append(str(p.relative_to(self.root_path))) + return files + + def read_file(self, path: str) -> str: + """Read content of a file.""" + target_path = self._validate_path(path) + if not target_path.exists(): + raise FileNotFoundError(f"File not found: {path}") + + try: + return target_path.read_text(encoding='utf-8') + except Exception as e: + logger.error(f"Error reading file {path}: {e}") + raise + + def write_file(self, path: str, content: str) -> str: + """Write content to a file. Creates directories if needed.""" + target_path = self._validate_path(path) + + try: + target_path.parent.mkdir(parents=True, exist_ok=True) + target_path.write_text(content, encoding='utf-8') + logger.info(f"Written file: {path}") + return f"Successfully wrote to {path}" + except Exception as e: + logger.error(f"Error writing file {path}: {e}") + raise + + +class BraveSearchTool: + """ + Wrapper for Brave Search API to allow agents to retrieve external information. + """ + def __init__(self, api_key: Optional[str] = None): + self.api_key = api_key or os.environ.get("BRAVE_API_KEY") + self.base_url = "https://api.search.brave.com/res/v1/web/search" + + def search(self, query: str, count: int = 5) -> List[Dict[str, Any]]: + """ + Execute a search query. + """ + if not self.api_key: + logger.warning("Brave Search API key not provided. Returning empty results.") + return [] + + headers = { + "Accept": "application/json", + "Accept-Encoding": "gzip", + "X-Subscription-Token": self.api_key + } + + try: + response = requests.get( + self.base_url, + params={"q": query, "count": count}, + headers=headers, + timeout=10 + ) + response.raise_for_status() + data = response.json() + + results = [] + if "web" in data and "results" in data["web"]: + for item in data["web"]["results"]: + results.append({ + "title": item.get("title"), + "url": item.get("url"), + "description": item.get("description") + }) + return results + except Exception as e: + logger.error(f"Brave Search failed: {e}") + return [] + + +class MCPToolkit: + """ + Aggregates all tools into a single interface for Agents. + """ + def __init__(self, sandbox_config: Dict[str, Any] = None): + self.config = sandbox_config or {} + + # Initialize tools + self.fs = FileSystemTool(root_path=self.config.get("host_workspace", "./data/output_repos")) + self.sandbox = DockerSandbox( + image=self.config.get("image", "deepcode_sandbox:latest"), + host_workspace_path=self.config.get("host_workspace", "./data/output_repos"), + container_workspace_path=self.config.get("container_workspace", "/workspace") + ) + self.search = BraveSearchTool(api_key=self.config.get("brave_api_key")) + + def get_tools_description(self) -> str: + """Returns a text description of available tools for the LLM system prompt.""" + return """ + Available Tools: + 1. read_file(path): Read content of a file. + 2. write_file(path, content): Write content to a file. + 3. list_files(path): List all files in directory. + 4. execute_command(command): Execute a shell command in the sandbox environment. + 5. search_web(query): Search the web for information. + """ diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/utils/prompts.py b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/utils/prompts.py new file mode 100644 index 00000000..07eedbc9 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/deepcode_repro/src/utils/prompts.py @@ -0,0 +1,203 @@ +""" +DeepCode Prompts Module + +This module contains the system prompts and instruction templates for the multi-agent framework. +It defines the persona and specific instructions for each agent in the pipeline: +1. Planning Swarm (Concept, Algorithm, Planning) +2. Coding Swarm (Coding, Summarization) +3. Verification Swarm (Analysis, Modification, Sandbox) +""" + +class AgentPrompts: + """ + Central repository for Agent System Prompts and Task Instructions. + """ + + # ========================================================================= + # 1. PLANNING AGENT SWARM + # ========================================================================= + + CONCEPT_AGENT_SYSTEM = """You are the Concept Agent, an expert software architect specializing in high-level system design. +Your goal is to analyze research papers and extract the core conceptual architecture. +Focus on: +- Identifying the main modules and their responsibilities. +- Determining the data flow between components. +- Abstracting away implementation details to focus on the "what" and "why". + +Output your analysis as a structured conceptual schema describing the system's components and their relationships.""" + + ALGORITHM_AGENT_SYSTEM = """You are the Algorithm Agent, a specialist in mathematical modeling and algorithmic implementation. +Your goal is to extract precise algorithmic details from research papers. +CRITICAL INSTRUCTION: Extract verbatim LaTeX for equations and step-by-step logic. +Focus on: +- Mathematical formulations and loss functions. +- Pseudocode or step-by-step procedural logic. +- Hyperparameters and constant values mentioned in the text. +- Tensor shapes and dimensionality transformations. + +Do not summarize; provide the exact mathematical specifications required for implementation.""" + + PLANNING_AGENT_SYSTEM = """You are the Planning Agent, the Lead Architect responsible for synthesizing a complete Implementation Blueprint. +You will receive input from the Concept Agent (high-level architecture) and the Algorithm Agent (math/logic details). +Your goal is to merge these into a concrete file-level development plan. + +You must produce a JSON Blueprint containing: +1. `file_hierarchy`: A tree structure of directories and files. +2. `component_specs`: Detailed specifications for each file (classes, functions, dependencies). +3. `dev_plan`: A topological sort of the files representing the implementation order. + +Ensure the file structure is standard for a Python repository (e.g., src/, tests/, utils/). +Ensure circular dependencies are minimized. +""" + + # ========================================================================= + # 2. CODING AGENT SWARM + # ========================================================================= + + CODING_AGENT_SYSTEM = """You are the Coding Agent, an expert Python developer. +Your task is to implement a single file based on a provided Component Specification and Context. + +Context provided: +1. Blueprint Specification for the target file. +2. Relevant Memory: Summaries of dependencies (interfaces of files you import). +3. RAG Context: Relevant snippets from the paper or external knowledge (if applicable). + +Guidelines: +- Write production-grade, typed Python code (Python 3.10+). +- Follow the specifications exactly. +- Use the provided dependency interfaces; do not hallucinate methods that don't exist in the memory summaries. +- Include docstrings and comments explaining complex logic (especially math). +- Do not implement placeholders (e.g., `pass`) unless explicitly told to. +- Ensure all imports are valid based on the project structure. + +Output ONLY the code for the file. Do not wrap in markdown blocks if possible, or ensure it is easily extractable.""" + + SUMMARIZATION_AGENT_SYSTEM = """You are the Summarization Agent. +Your task is to analyze the source code of a newly implemented file and generate a compressed memory entry. +This summary will be used by other agents that depend on this file. + +Output a JSON object with: +1. `core_purpose`: A one-sentence description of what the file does. +2. `public_interface`: A list of classes and functions with their signatures (name, args, return type). +3. `dependency_edges`: A list of files this file imports (efferent dependencies). + +Be concise. The goal is to save context window space for future agents.""" + + # ========================================================================= + # 3. VERIFICATION AGENT SWARM + # ========================================================================= + + ANALYSIS_AGENT_SYSTEM = """You are the Analysis Agent, a QA Engineer and Static Analysis expert. +Your task is to review a repository or a specific file against the original Blueprint and Python best practices. + +Check for: +1. Structural Correctness: Does the code match the Blueprint specs (classes, methods)? +2. Syntax/Linting: Are there obvious syntax errors or undefined variables? +3. Import Logic: Are imports correct based on the file hierarchy? +4. Completeness: Are any methods left unimplemented? + +Output a structured Report listing issues found, classified by severity (Critical, Warning, Info).""" + + MODIFICATION_AGENT_SYSTEM = """You are the Modification Agent, a Senior Developer tasked with fixing code. +You will receive: +1. The current source code of a file. +2. An Error Report (from Analysis) or an Execution Trace (from Sandbox). +3. Specific Fix Instructions. + +Your task is to apply the fixes to the code. +- Maintain the original logic where it is correct. +- Only modify the parts necessary to fix the reported issues. +- Return the fully corrected file content.""" + + SANDBOX_AGENT_SYSTEM = """You are the Sandbox Agent, responsible for dynamic verification. +Your task is to analyze execution traces from the runtime environment. + +Input: +- Command executed (e.g., `pytest tests/test_model.py`). +- Stdout/Stderr output. +- Exit code. + +Task: +- Determine if the execution was successful. +- If failed, diagnose the root cause (e.g., ImportError, AssertionError, SyntaxError). +- Formulate specific instructions to fix the error. + +Output a JSON object containing: +- `success`: boolean +- `error_type`: string (or null) +- `fix_instructions`: string (detailed steps to resolve the error)""" + + # ========================================================================= + # TEMPLATES + # ========================================================================= + + @staticmethod + def get_planning_task(paper_text: str) -> str: + return f""" +Analyze the following research paper content and generate a comprehensive Implementation Blueprint. + +PAPER CONTENT: +{paper_text[:50000]}... (truncated if too long) + +Step 1: Concept Agent -> Extract Architecture. +Step 2: Algorithm Agent -> Extract Math/Logic. +Step 3: Planning Agent -> Merge into Blueprint. + +Return the final Blueprint JSON. +""" + + @staticmethod + def get_coding_task(file_name: str, spec: str, context: str, rag_data: str) -> str: + return f""" +Implement the file: '{file_name}' + +SPECIFICATION: +{spec} + +DEPENDENCY CONTEXT (Memory): +{context} + +ADDITIONAL KNOWLEDGE (RAG): +{rag_data} + +Write the complete code for '{file_name}'. +""" + + @staticmethod + def get_summarization_task(code: str) -> str: + return f""" +Summarize the following Python code for the CodeMem system. + +CODE: +{code} + +Return the JSON memory entry. +""" + + @staticmethod + def get_analysis_task(code: str, spec: str) -> str: + return f""" +Analyze the following code against its specification. + +SPECIFICATION: +{spec} + +CODE: +{code} + +Report any discrepancies, syntax errors, or missing implementations. +""" + + @staticmethod + def get_modification_task(code: str, feedback: str) -> str: + return f""" +Fix the following code based on the feedback provided. + +FEEDBACK/ERRORS: +{feedback} + +CODE: +{code} + +Return the corrected code. +""" diff --git a/DeepCode/deepcode_lab/papers/1/generate_code/list_files.py b/DeepCode/deepcode_lab/papers/1/generate_code/list_files.py new file mode 100644 index 00000000..8c99c2e6 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/generate_code/list_files.py @@ -0,0 +1,13 @@ +import os + +def list_files(startpath): + for root, dirs, files in os.walk(startpath): + level = root.replace(startpath, '').count(os.sep) + indent = ' ' * 4 * (level) + print('{}{}/'.format(indent, os.path.basename(root))) + subindent = ' ' * 4 * (level + 1) + for f in files: + print('{}{}'.format(subindent, f)) + +if __name__ == "__main__": + list_files(".") diff --git a/DeepCode/deepcode_lab/papers/1/github_download.txt b/DeepCode/deepcode_lab/papers/1/github_download.txt new file mode 100644 index 00000000..93d920e3 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/github_download.txt @@ -0,0 +1 @@ +Automated repository acquisition skipped - fast mode enabled for optimized processing \ No newline at end of file diff --git a/DeepCode/deepcode_lab/papers/1/implement_code_summary.md b/DeepCode/deepcode_lab/papers/1/implement_code_summary.md new file mode 100644 index 00000000..c198d985 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/implement_code_summary.md @@ -0,0 +1,857 @@ +# Code Implementation Progress Summary +*Accumulated implementation progress for all files* + + +================================================================================ +## IMPLEMENTATION File list_files.py; ROUND 0 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:47:07 +**File Implemented**: list_files.py + +**Core Purpose** +A standalone utility script designed to recursively traverse the file system starting from a specified path and visualize the directory structure and file hierarchy in a readable, indented tree format. + +**Public Interface** +- Function `list_files(startpath)`: Traverses the directory tree from `startpath` -> `None`: Prints the formatted directory structure to stdout. + +**Internal Dependencies** +- From `os`: `walk`, `sep`, `path.basename` (Standard Library) + +**External Dependencies** +- None (Standalone utility, likely used for debugging or verification of file generation). + +**Implementation Notes** +- Uses `os.walk` for recursive directory traversal. +- Calculates indentation levels based on path separators to create a visual tree structure. + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/src/utils/logger.py; ROUND 1 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:47:29 +**File Implemented**: deepcode_repro/src/utils/logger.py + +**Core Purpose** +Provides a centralized logging configuration utility to capture execution traces, errors, and informational messages, directing output to both the console (stdout) and timestamped log files. + +**Public Interface** +- Function `setup_logger(name: str = "DeepCode", log_level: int = logging.INFO, log_dir: str = "logs") -> logging.Logger`: Configures and returns a logger instance with formatted stream and file handlers. +- Variable `logger`: A default initialized `logging.Logger` instance ready for immediate import and use. + +**Internal Dependencies** +- Standard Library: `logging`, `sys`, `os`, `datetime` + +**External Dependencies** +- Expected to be imported by: `main.py`, `src/agents/*.py`, `src/core/*.py` (used globally for tracing). +- Key exports used elsewhere: The `logger` object or `setup_logger` function. + +**Implementation Notes** +- Architecture decisions: Implements a check (`if logger.handlers`) to prevent duplicate log entries if the logger is configured multiple times during imports. +- Cross-File Relationships: Acts as the foundational tracing layer for the entire application; other modules will import `logger` to report their state. + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/config.yaml; ROUND 2 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:48:23 +**File Implemented**: deepcode_repro/config.yaml + + + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/requirements.txt; ROUND 3 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:48:54 +**File Implemented**: deepcode_repro/requirements.txt + +**Core Purpose** +Defines the specific Python package versions and libraries required to run the DeepCode reproduction framework, ensuring a consistent environment for PDF parsing, vector storage (RAG), LLM interaction, and sandbox execution. + +**Public Interface** +- **N/A**: This is a configuration file used by package managers (pip) rather than a Python module. +- **Key Libraries**: + - `pypdf`, `marker-pdf`: Document parsing. + - `chromadb`: Vector database for CodeRAG. + - `anthropic`, `openai`: LLM API clients. + - `docker`: Python SDK for managing the sandbox environment. + - `pydantic`: Data validation for Blueprint schemas. + +**Internal Dependencies** +- **N/A**: Does not import other files. + +**External Dependencies** +- **Expected to be used by**: + - `deepcode_repro/docker/Dockerfile` (to install dependencies in the container). + - `deepcode_repro/README.md` (installation instructions). + - CI/CD pipelines or setup scripts. + +**Implementation Notes** +- **Architecture decisions**: Includes `marker-pdf` specifically for high-fidelity Markdown conversion of research papers, which is critical for the "Hierarchical Content Segmentation" component. +- **Cross-File Relationships**: Establishes the runtime environment that all `src/` python files will operate within. + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/src/utils/mcp_tools.py; ROUND 4 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:49:59 +**File Implemented**: deepcode_repro/src/utils/mcp_tools.py + +**Core Purpose** +Provides the Model Context Protocol (MCP) toolkit for agents, enabling secure code execution via a Docker sandbox, restricted filesystem access for reading/writing code, and web search capabilities for external information retrieval. + +**Public Interface** +- Class `DockerSandbox`: Manages an isolated Docker container for code execution. + - Constructor: `__init__(image="deepcode_sandbox:latest", host_workspace_path="./data/output_repos", container_workspace_path="/workspace", auto_remove=True)` + - Methods: `start()`, `stop()`, `execute_command(command: str, timeout: int) -> Tuple[int, str, str]` + - Context Manager: Supports `with DockerSandbox(...) as sandbox:` +- Class `FileSystemTool`: Provides safe file I/O restricted to a root directory. + - Constructor: `__init__(root_path="./data/output_repos")` + - Methods: `list_files(path=".") -> List[str]`, `read_file(path: str) -> str`, `write_file(path: str, content: str) -> str` +- Class `BraveSearchTool`: Wrapper for Brave Search API. + - Constructor: `__init__(api_key: Optional[str])` + - Methods: `search(query: str, count: int) -> List[Dict]` +- Class `MCPToolkit`: Aggregates tools into a unified interface. + - Constructor: `__init__(sandbox_config: Dict)` + - Methods: `get_tools_description() -> str` + +**Internal Dependencies** +- From `src.utils.logger`: `logger` (logging utility) +- External packages: `docker` (Docker SDK for Python), `requests` (HTTP calls), `shutil`, `pathlib` + +**External Dependencies** +- Expected to be imported by: `deepcode_repro/src/agents/base.py` (to equip agents with tools), `deepcode_repro/src/agents/verification.py` (for running tests). +- Key exports used elsewhere: `MCPToolkit`, `DockerSandbox`. + +**Implementation Notes** +- Architecture decisions: The Sandbox uses a "keep-alive" strategy (`tail -f /dev/null`) and `exec_run` for commands rather than spinning up a new container for every command, improving performance. +- Cross-File Relationships: Relies on `config.yaml` (via `main.py` injection) to configure workspace paths and API keys. + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/src/core/document_parser.py; ROUND 5 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:50:44 +**File Implemented**: deepcode_repro/src/core/document_parser.py + +**Core Purpose** +This file implements the **Hierarchical Content Segmentation** algorithm (Algo 1). It is responsible for parsing raw input documents (PDFs or Markdown) into structured `Segment` objects (Header + Content pairs) to preserve semantic hierarchy, enabling downstream agents to query specific sections of a paper rather than raw text chunks. + +**Public Interface** +- **Class `Segment`**: A dataclass representing a document chunk. + - Fields: `header: str`, `content: str`, `level: int`, `metadata: Dict` + - Method `to_text() -> str`: Returns the formatted text representation. +- **Class `DocumentSegmenter`**: The main parsing engine. + - Constructor: `__init__(use_marker: bool = True)` + - Method `parse_file(file_path: str) -> List[Segment]`: Auto-detects file type (PDF/MD) and returns segments. + - Method `parse_pdf(pdf_path: str) -> List[Segment]`: Handles PDF conversion (prioritizing `marker-pdf` logic with a `pypdf` fallback). + - Method `parse_markdown(text: str) -> List[Segment]`: Splits text based on Markdown headers (#, ##). + +**Internal Dependencies** +- From `..utils.logger`: `logger` for error reporting and warnings. + +**External Dependencies** +- **External Packages**: + - `pypdf`: Used for PDF text extraction if `marker-pdf` is unavailable. + - `pathlib`, `re`, `dataclasses`: Standard library utilities. +- **Expected to be imported by**: + - `src/core/rag_engine.py`: To index document segments for retrieval. + - `src/agents/planning.py`: To query specific sections (e.g., "Methodology") when building the blueprint. + +**Implementation Notes** +- **Architecture Decisions**: Implements a fallback strategy for PDF parsing. It attempts to simulate high-fidelity parsing (like `marker-pdf`) but falls back to a heuristic-based `pypdf` extraction that attempts to reconstruct Markdown headers from raw text layout to ensure the code runs in environments without heavy ML dependencies. +- **Cross-File Relationships**: Acts as the primary data ingestion layer feeding into the `RAG Engine` and `Planning Agent`. + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/src/core/blueprint.py; ROUND 6 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:51:15 +**File Implemented**: deepcode_repro/src/core/blueprint.py + +**Core Purpose** +Defines the Pydantic data models and JSON schemas for the "Implementation Blueprint," which serves as the central architectural contract between the Planning Agent (producer) and the Coding Agent (consumer). It structures the repository hierarchy, component specifications, and development plan. + +**Public Interface** +- **Class `FunctionSignature`**: Represents function metadata. + - Fields: `name`, `args`, `return_type`, `description`. +- **Class `ClassSignature`**: Represents class metadata. + - Fields: `name`, `methods` (List[FunctionSignature]), `description`. +- **Class `ComponentSpecification`**: Detailed specification for a single file. + - Fields: `filename`, `core_purpose`, `algorithmic_details` (LaTeX/pseudocode), `dependencies`, `public_interface`, `implementation_notes`. +- **Class `FileNode`**: Recursive tree structure for the file system. + - Fields: `name`, `type` (file/directory), `children`. +- **Class `Blueprint`**: The root model for the entire project plan. + - Fields: `project_name`, `file_hierarchy`, `component_specs`, `dev_plan` (topological sort). + - Method `to_json(self) -> str`: Serializes blueprint to JSON. + - Method `from_json(cls, json_str: str) -> 'Blueprint'`: Deserializes from JSON. + - Method `get_spec(self, filename: str) -> Optional[ComponentSpecification]`: Retrieves spec for a file. + - Method `save(self, path: str)`: Persists blueprint to disk. + - Method `load(cls, path: str) -> 'Blueprint'`: Loads blueprint from disk. + +**Internal Dependencies** +- **External packages**: `pydantic` (BaseModel, Field) for data validation and schema definition. +- **Standard libraries**: `json`, `typing` (List, Dict, Optional, Union, Any). + +**External Dependencies** +- **Expected to be imported by**: + - `src/agents/planning.py`: To instantiate the blueprint during the planning phase. + - `src/agents/coding.py`: To read component specifications during code generation. + - `src/agents/verification.py`: To validate generated code against the blueprint. + - `main.py`: To manage the lifecycle of the blueprint (save/load). + +**Implementation Notes** +- **Architecture decisions**: Uses `pydantic` to ensure strict typing and easy JSON serialization, which is crucial for reliable LLM structured output parsing. +- **Cross-File Relationships**: This file acts as the "State" definition for the transition between Phase 2 (Planning) and Phase 3 (Coding). + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/src/core/memory.py; ROUND 7 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:51:49 +**File Implemented**: deepcode_repro/src/core/memory.py + +**Core Purpose** +This file implements the **CodeMem** component, a graph-based state management system that maintains summaries of generated files (interfaces, dependencies, and purposes). It solves the context bottleneck by allowing agents to retrieve only relevant file summaries (`MemoryEntry`) based on dependency relationships rather than loading the entire codebase. + +**Public Interface** +- **Class `CodeMem`**: Main state container. + - `add_entry(entry: MemoryEntry)`: Updates the global state with a new file summary and updates afferent edges of dependencies. + - `get_entry(file_path: str) -> Optional[MemoryEntry]`: Retrieves a specific file's state. + - `select_relevant(target_file: str, declared_dependencies: List[str]) -> List[MemoryEntry]`: Returns a subset of memory entries relevant to the target file (implementation of `SelectRelevantMemory`). + - `to_context_string(relevant_entries: List[MemoryEntry]) -> str`: Formats entries into a text block for LLM consumption. + - `save(path: str)` / `load(path: str)`: Persistence methods for the memory state. +- **Class `MemoryEntry`**: Data model for a single file's summary. + - `to_text() -> str`: Converts the structured entry into a human-readable string for prompts. +- **Class `DependencyEdges`**: Data model tracking `afferent` (incoming) and `efferent` (outgoing) dependencies. + +**Internal Dependencies** +- **From `deepcode_repro.src.core.blueprint`**: `FunctionSignature`, `ClassSignature` (for structured interface storage). +- **From `deepcode_repro.src.utils.logger`**: `logger` (for tracking memory updates). +- **External packages**: `pydantic` (data validation), `json`. + +**External Dependencies** +- **Expected to be imported by**: `deepcode_repro/src/agents/coding.py` (to retrieve context before generation and store summaries after generation). +- **Key exports used elsewhere**: `CodeMem` instance is the central context provider for the generation loop. + +**Implementation Notes** +- **Graph Logic**: The `add_entry` method automatically updates the `afferent` (incoming) edges of the files the new entry depends on, maintaining a bidirectional dependency graph. +- **Context Optimization**: The `select_relevant` method currently prioritizes explicit dependencies declared in the Blueprint to minimize token usage, with hooks available for implicit dependency expansion. + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/src/core/rag_engine.py; ROUND 8 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:52:29 +**File Implemented**: deepcode_repro/src/core/rag_engine.py + +**Core Purpose** +The `rag_engine.py` file implements the **CodeRAG** (Retrieval-Augmented Generation) engine, which manages the indexing of document segments into a vector database (ChromaDB) and performs semantic retrieval. Its primary goal is to overcome context bottlenecks by conditionally retrieving relevant documentation or code snippets during the generation process. + +**Public Interface** +- **Class `CodeRAG`**: Manages the vector store connection and retrieval logic. + - `__init__(persist_directory: str, collection_name: str)`: Initializes the ChromaDB client and collection. + - `index_segments(segments: List[Segment], source_id: str)`: Embeds and stores document segments with metadata. + - `query(query_text: str, n_results: int, filter_metadata: Optional[Dict]) -> List[Dict]`: Retrieves the top-k most similar segments for a query. + - `should_retrieve(context_text: str, target_file: str) -> bool`: Determines if retrieval is necessary for the current context (currently a heuristic). + - `clear_index()`: Resets the vector database collection. + +**Internal Dependencies** +- **From `deepcode_repro.src.core.document_parser`**: Imports `Segment` class for type hinting and data structure. +- **From `deepcode_repro.src.utils.logger`**: Imports `logger` for error handling and tracing. +- **External packages**: `chromadb` (Vector Database), `os`, `typing`. + +**External Dependencies** +- **Expected to be imported by**: + - `src/agents/coding.py`: To retrieve context before generating code. + - `src/agents/planning.py`: To query specific details from the paper during blueprint synthesis. + - `main.py`: To initialize the RAG engine instance. + +**Implementation Notes** +- **Architecture**: Wraps `chromadb.PersistentClient` to provide a persistent vector store on disk. +- **Data Flow**: Accepts `Segment` objects (from `document_parser`), converts them to text/metadata, and stores them. Returns dictionaries containing content and distance metrics upon query. +- **Adaptive Retrieval**: The `should_retrieve` method is a placeholder for the "Adaptive Retrieval" logic described in the paper, currently implemented as a check for the existence of a target file. + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/src/agents/base.py; ROUND 9 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:53:13 +**File Implemented**: deepcode_repro/src/agents/base.py + +**Core Purpose** +Provides the foundational infrastructure for all agents in the system, abstracting Large Language Model (LLM) interactions (supporting both Anthropic and OpenAI), implementing robust retry logic for API calls, and standardizing tool execution via the Model Context Protocol (MCP) toolkit. + +**Public Interface** +- **Class `AgentConfig`**: Pydantic model for agent configuration. + - Fields: `model_provider`, `model_name`, `temperature`, `max_tokens`, `api_key`. +- **Class `BaseAgent` (Abstract)**: + - `__init__(name: str, config: Dict[str, Any], mcp_toolkit: Optional[MCPToolkit] = None)`: Initializes the agent with specific LLM client and tools. + - `call_llm(system_prompt: str, user_message: str, tools: Optional[List[Dict]] = None) -> str`: Executes an LLM generation call with automatic retries for rate limits/connection errors. + - `run_tool(tool_name: str, **kwargs) -> Any`: Dispatches tool execution requests (e.g., `read_file`, `execute_command`) to the `MCPToolkit`. + - `run(*args, **kwargs) -> Any`: Abstract method that concrete agent implementations must define. + +**Internal Dependencies** +- From `deepcode_repro.src.utils.logger`: `logger` +- From `deepcode_repro.src.utils.mcp_tools`: `MCPToolkit` + +**External Dependencies** +- **Packages**: `tenacity` (retry logic), `openai` (LLM client), `anthropic` (LLM client), `pydantic` (config validation). +- **Expected to be imported by**: `src/agents/planning.py`, `src/agents/coding.py`, `src/agents/verification.py`. + +**Implementation Notes** +- **Architecture**: Implements a provider-agnostic wrapper (`_call_anthropic`, `_call_openai`) allowing the system to switch LLM backends via configuration. +- **Error Handling**: Uses `tenacity` decorators to handle `APIConnectionError` and `RateLimitError` with exponential backoff. +- **Tooling**: Acts as the bridge between the high-level agent logic and the low-level `MCPToolkit` sandbox operations. + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/src/utils/prompts.py; ROUND 10 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:53:52 +**File Implemented**: deepcode_repro/src/utils/prompts.py + +**Core Purpose** +This module serves as the central repository for all system prompts and instruction templates used across the multi-agent framework. It defines the specific personas (System Prompts) for the Planning, Coding, and Verification swarms, and provides static methods to generate dynamic task instructions based on runtime context. + +**Public Interface** +- **Class `AgentPrompts`**: Container for prompt constants and template methods. + - **Constants (System Prompts)**: + - `CONCEPT_AGENT_SYSTEM`: Persona for high-level architecture extraction. + - `ALGORITHM_AGENT_SYSTEM`: Persona for extracting math/logic (verbatim LaTeX). + - `PLANNING_AGENT_SYSTEM`: Persona for synthesizing the Blueprint JSON. + - `CODING_AGENT_SYSTEM`: Persona for implementing Python files from specs. + - `SUMMARIZATION_AGENT_SYSTEM`: Persona for generating CodeMem entries. + - `ANALYSIS_AGENT_SYSTEM`: Persona for static analysis and QA. + - `MODIFICATION_AGENT_SYSTEM`: Persona for applying fixes to code. + - `SANDBOX_AGENT_SYSTEM`: Persona for diagnosing runtime errors. + - **Methods**: + - `get_planning_task(paper_text: str) -> str`: Generates the prompt for the Planning Swarm to create the Blueprint. + - `get_coding_task(file_name: str, spec: str, context: str, rag_data: str) -> str`: Generates the prompt for the Coding Agent including specs, memory, and RAG context. + - `get_summarization_task(code: str) -> str`: Generates the prompt for creating a memory summary from code. + - `get_analysis_task(code: str, spec: str) -> str`: Generates the prompt for static analysis against the spec. + - `get_modification_task(code: str, feedback: str) -> str`: Generates the prompt for fixing code based on feedback. + +**Internal Dependencies** +- None (Pure Python string/template module). + +**External Dependencies** +- **Expected to be imported by**: + - `deepcode_repro/src/agents/planning.py`: To retrieve planning-related prompts. + - `deepcode_repro/src/agents/coding.py`: To retrieve coding and summarization prompts. + - `deepcode_repro/src/agents/verification.py`: To retrieve analysis and modification prompts. + +**Implementation Notes** +- **Separation of Concerns**: Decouples prompt engineering from agent logic, allowing for easier iteration on prompt strategies without modifying the agent code structure. +- **Context Injection**: The template methods use f-strings to inject dynamic context (like RAG data or Memory summaries) directly into the prompt structure. + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/src/agents/planning.py; ROUND 11 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:54:43 +**File Implemented**: deepcode_repro/src/agents/planning.py + +**Core Purpose** +Implements the "Planning Swarm" architecture consisting of three specialized agents (Concept, Algorithm, and Planner) that collaboratively analyze research paper text to synthesize a structured implementation Blueprint JSON. + +**Public Interface** +- Class `PlanningSwarm`: Orchestrates the multi-agent planning pipeline. + - `__init__(config: Dict[str, Any])` + - `plan(paper_text: str) -> Blueprint`: Main entry point that runs sub-agents and returns the final `Blueprint` object. +- Class `ConceptAgent`: Extracts high-level conceptual architecture and data flow. + - `run(paper_text: str) -> str` +- Class `AlgorithmAgent`: Extracts mathematical formulas, algorithms, and pseudocode. + - `run(paper_text: str) -> str` +- Class `PlannerAgent`: Merges schemas to synthesize the final Blueprint. + - `run(paper_text: str, concept_schema: str, algo_schema: str) -> Blueprint` + +**Internal Dependencies** +- From `deepcode_repro.src.agents.base`: `BaseAgent` (Parent class) +- From `deepcode_repro.src.core.blueprint`: `Blueprint` (Return type) +- From `deepcode_repro.src.utils.prompts`: `AgentPrompts` (System prompts) +- From `deepcode_repro.src.utils.logger`: `logger` +- External packages: `json`, `re`, `typing` + +**External Dependencies** +- Expected to be imported by: `deepcode_repro/main.py` (to initiate the generation pipeline). +- Key exports used elsewhere: `PlanningSwarm` class. + +**Implementation Notes** +- Architecture decisions: Uses a sequential execution flow (Concept/Algo -> Planner) within `PlanningSwarm`, though designed to support parallel execution of extraction agents. +- Cross-File Relationships: Directly instantiates `Blueprint` objects from LLM JSON output, requiring strict adherence to the schema defined in `src/core/blueprint.py`. +- Includes error handling for JSON parsing to strip Markdown code blocks often returned by LLMs. + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/src/agents/coding.py; ROUND 12 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:55:22 +**File Implemented**: deepcode_repro/src/agents/coding.py + +**Core Purpose** +This file implements the **Stateful Generation Loop** (Component 5 of the DeepCode framework). It defines the `CodingAgent` for generating implementation code from specifications and context, the `SummarizationAgent` for compressing generated code into memory entries, and the `CodingSwarm` orchestrator which manages the topological execution of the development plan, integrating CodeMem and CodeRAG. + +**Public Interface** +- **Class `CodingAgent`** (inherits `BaseAgent`): + - `run(file_name: str, spec: ComponentSpecification, memory_context: str, rag_context: str = "") -> str`: Generates the source code for a specific file using the LLM. +- **Class `SummarizationAgent`** (inherits `BaseAgent`): + - `run(file_path: str, code: str) -> MemoryEntry`: Analyzes generated code to produce a structured summary (purpose, interface, dependencies) for the CodeMem graph. +- **Class `CodingSwarm`**: + - `__init__(config: Dict, memory: CodeMem, rag_engine: Optional[CodeRAG], output_dir: str)`: Initializes the swarm with necessary engines. + - `generate_codebase(blueprint: Blueprint) -> None`: Main entry point to execute the full development plan found in the blueprint. + +**Internal Dependencies** +- **From `deepcode_repro.src.agents.base`**: `BaseAgent` (LLM wrapper). +- **From `deepcode_repro.src.core.blueprint`**: `Blueprint`, `ComponentSpecification` (Input data structures). +- **From `deepcode_repro.src.core.memory`**: `CodeMem`, `MemoryEntry`, `DependencyEdges` (State management). +- **From `deepcode_repro.src.core.rag_engine`**: `CodeRAG` (External knowledge retrieval). +- **From `deepcode_repro.src.utils.prompts`**: `AgentPrompts` (System and user prompts). +- **From `deepcode_repro.src.utils.logger`**: `logger` (Tracing). + +**External Dependencies** +- **Expected to be imported by**: `deepcode_repro/main.py` (The central CLI/Orchestrator). +- **Key exports used elsewhere**: `CodingSwarm` is the primary interface used to trigger the coding phase after planning. + +**Implementation Notes** +- **Architecture**: Implements the "Context -> Code -> Summary -> Memory Update" loop described in the paper. +- **Topological Execution**: The `CodingSwarm` iterates through `blueprint.dev_plan`, ensuring dependencies are generated (and summarized into memory) before dependent files are processed. +- **RAG Integration**: Uses a heuristic (`rag_engine.should_retrieve`) to decide if external knowledge is needed before generation. +- **Parsing Logic**: Includes specific logic to extract code blocks from Markdown and parse JSON responses for memory summaries, with fallback mechanisms for parsing failures. + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/src/agents/verification.py; ROUND 13 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:56:22 +**File Implemented**: deepcode_repro/src/agents/verification.py + +**Core Purpose** +Defines the `VerificationSwarm` and specialized agents (`AnalysisAgent`, `ModificationAgent`, `SandboxAgent`) to ensure code correctness. It implements a two-stage verification loop: static analysis against blueprint specifications and dynamic execution within a sandbox environment, with iterative self-correction capabilities. + +**Public Interface** +- **Class `AnalysisAgent`**: Performs static checks against component specs. + - `run(code: str, spec: ComponentSpecification) -> Optional[str]`: Returns a report string if issues found, else `None`. +- **Class `ModificationAgent`**: Generates corrected code based on feedback. + - `run(code: str, feedback: str) -> str`: Returns the modified code string. +- **Class `SandboxAgent`**: Diagnoses runtime errors from execution traces. + - `analyze_trace(code: str, trace: str) -> str`: Returns fix instructions based on stdout/stderr. +- **Class `VerificationSwarm`**: Orchestrates the verification workflow. + - Constructor: `(config: Dict, mcp_toolkit: Optional[MCPToolkit], output_dir: str)` + - `verify_file_static(file_path: str, spec: ComponentSpecification) -> bool`: Runs static analysis/fix loop. + - `verify_file_dynamic(file_path: str, test_command: str = None) -> bool`: Runs execution/fix loop via MCP. + - `verify_codebase(blueprint: Blueprint)`: Iterates through the blueprint plan to verify files. + +**Internal Dependencies** +- From `deepcode_repro.src.agents.base`: `BaseAgent` +- From `deepcode_repro.src.core.blueprint`: `Blueprint`, `ComponentSpecification` +- From `deepcode_repro.src.utils.prompts`: `AgentPrompts` +- From `deepcode_repro.src.utils.logger`: `logger` +- From `deepcode_repro.src.utils.mcp_tools`: `MCPToolkit` + +**External Dependencies** +- Expected to be imported by: `deepcode_repro/main.py` (to trigger verification after coding phase). + +**Implementation Notes** +- **Self-Correction Loop**: Both static and dynamic verification implement a retry loop (default 3 attempts) where the `ModificationAgent` attempts to fix reported issues. +- **Sandbox Integration**: Dynamic verification relies on `MCPToolkit` to execute code safely; if the toolkit is missing, it skips dynamic checks. +- **Heuristics**: `AnalysisAgent` checks for "NO ISSUES FOUND" in the LLM response to determine pass/fail. + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/main.py; ROUND 14 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:57:29 +**File Implemented**: deepcode_repro/main.py + + + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/src/core/__init__.py; ROUND 15 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:57:59 +**File Implemented**: deepcode_repro/src/core/__init__.py + +**Core Purpose** +This file serves as the package initializer for the `deepcode_repro.src.core` module, aggregating and exposing key classes for document parsing, blueprinting, memory management, and RAG operations to simplify imports throughout the application. + +**Public Interface** +* **Exports from `document_parser`**: + * Class `DocumentSegmenter`: Handles hierarchical content segmentation of input papers. + * Class `Segment`: Data structure representing a chunk of the document. +* **Exports from `blueprint`**: + * Class `Blueprint`: Represents the synthesized implementation plan. + * Class `ComponentSpecification`: Detailed specs for specific components. + * Class `FunctionSignature` / `ClassSignature`: Schema definitions for code structures. +* **Exports from `memory`**: + * Class `CodeMem`: Manages the graph-based memory state for cross-file consistency. + * Class `MemoryEntry`: Represents a single memory unit (summary/interface). +* **Exports from `rag_engine`**: + * Class `CodeRAG`: Handles retrieval-augmented generation logic. + +**Internal Dependencies** +* From `.document_parser`: `DocumentSegmenter`, `Segment` +* From `.blueprint`: `Blueprint`, `ComponentSpecification`, `FunctionSignature`, `ClassSignature` +* From `.memory`: `CodeMem`, `MemoryEntry` +* From `.rag_engine`: `CodeRAG` + +**External Dependencies** +* **Expected to be imported by**: `src/agents/planning.py`, `src/agents/coding.py`, `src/agents/verification.py`, `main.py`. +* **Key exports used elsewhere**: `CodeMem`, `Blueprint`, `DocumentSegmenter`, `CodeRAG`. + +**Implementation Notes** +* **Architecture decisions**: Uses relative imports to expose a clean API surface for the core module. This allows consumers to import core components directly from `deepcode_repro.src.core` without needing to know the internal file structure (e.g., `from deepcode_repro.src.core import CodeMem`). + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/src/agents/__init__.py; ROUND 16 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:58:18 +**File Implemented**: deepcode_repro/src/agents/__init__.py + +**Core Purpose** +This file serves as the package initializer for the `agents` module, aggregating and exposing the hierarchical agent classes (Base, Planning, Coding, Verification) to simplify imports for the main orchestration logic. + +**Public Interface** +- **Exports**: + - `BaseAgent`, `AgentConfig` (Foundation) + - `PlanningSwarm`, `ConceptAgent`, `AlgorithmAgent`, `PlannerAgent` (Planning Phase) + - `CodingSwarm`, `CodingAgent`, `SummarizationAgent` (Coding Phase) + - `VerificationSwarm`, `AnalysisAgent`, `ModificationAgent`, `SandboxAgent` (Verification Phase) + +**Internal Dependencies** +- From `.base`: `BaseAgent`, `AgentConfig` +- From `.planning`: `PlanningSwarm`, `ConceptAgent`, `AlgorithmAgent`, `PlannerAgent` +- From `.coding`: `CodingSwarm`, `CodingAgent`, `SummarizationAgent` +- From `.verification`: `VerificationSwarm`, `AnalysisAgent`, `ModificationAgent`, `SandboxAgent` + +**External Dependencies** +- Expected to be imported by: `deepcode_repro/main.py`, `deepcode_repro/experiments/run_paperbench.py` +- Key exports used elsewhere: `PlanningSwarm`, `CodingSwarm`, `VerificationSwarm` (used by the main orchestration loop). + +**Implementation Notes** +- Architecture decisions: Uses the `__all__` pattern to explicitly define the public API of the `agents` package, hiding internal implementation details of the sub-modules if necessary, though currently it exposes all concrete agent classes. + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/src/__init__.py; ROUND 17 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:58:33 +**File Implemented**: deepcode_repro/src/__init__.py + +**Core Purpose** +Marks the `deepcode_repro/src` directory as a Python package, enabling the hierarchical import of submodules (core, agents, utils) throughout the project. + +**Public Interface** +- **None**: This file is currently empty aside from a docstring, serving only as a package marker. + +**Internal Dependencies** +- **None** + +**External Dependencies** +- **Implicitly used by**: `main.py` and experiment scripts to resolve imports like `from deepcode_repro.src.core import ...`. + +**Implementation Notes** +- **Architecture decisions**: Kept empty to avoid circular import issues at the root package level. Submodules are responsible for their own specific exports. + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/docker/entrypoint.sh; ROUND 18 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:59:04 +**File Implemented**: deepcode_repro/docker/entrypoint.sh + +**Core Purpose** +This shell script serves as the runtime entry point for the Docker sandbox environment. It initializes the workspace directory, configures the Python path to ensure module discoverability, and manages the container's lifecycle by either executing passed commands or entering a dormant state (`tail -f /dev/null`) to await external execution requests via the MCP tools. + +**Public Interface** +- **CLI Usage**: `./entrypoint.sh [command] [args...]` +- **Behavior**: + - **With Arguments**: Executes the provided command and arguments immediately (e.g., `python main.py`). + - **Without Arguments**: Prints a ready message and executes `tail -f /dev/null` to keep the container running indefinitely, allowing `docker exec` commands from the host. +- **Environment Variables Set**: + - `PYTHONPATH`: Appends `:/workspace` to the existing path. + +**Internal Dependencies** +- **System Utilities**: Requires standard Linux shell commands (`mkdir`, `cd`, `export`, `echo`, `tail`, `exec`). + +**External Dependencies** +- **Dockerfile**: This script is configured as the `ENTRYPOINT` in the Docker image definition. +- **src/utils/mcp_tools.py**: Relies on this script to keep the container alive so that `exec_run` commands can be dispatched dynamically. + +**Implementation Notes** +- **Error Handling**: Uses `set -e` to ensure the script fails immediately if any command errors out. +- **Workspace Setup**: Enforces `/workspace` as the working directory and ensures it exists. + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/docker/Dockerfile; ROUND 19 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 14:59:50 +**File Implemented**: deepcode_repro/docker/Dockerfile + +**Core Purpose** +Defines the isolated execution environment (sandbox) for running and verifying code generated by the agents. It builds upon Ubuntu 22.04, pre-installs essential system tools and heavy Machine Learning libraries (PyTorch, NumPy, Scikit-learn) to optimize execution time and ensure a consistent runtime for generated repositories. + +**Public Interface** +- **Docker Image Specification**: + - **Base Image**: `ubuntu:22.04` + - **Working Directory**: `/workspace` + - **Entrypoint**: `/entrypoint.sh` + - **Environment Variables**: `PYTHONPATH` includes `/workspace`, `DEBIAN_FRONTEND=noninteractive` + - **Pre-installed Python Packages**: `numpy`, `pandas`, `scipy`, `scikit-learn`, `torch`, `pytest`, `matplotlib`, `seaborn` + +**Internal Dependencies** +- **File**: `entrypoint.sh` (Required in the build context to be copied into the image) + +**External Dependencies** +- **Consumer**: `deepcode_repro/src/utils/mcp_tools.py` (Specifically the `Sandbox` or `CommandExecutor` class uses this Dockerfile to build the image and spawn containers for code execution). + +**Implementation Notes** +- **Optimization**: Heavy ML libraries (Torch, Pandas, Scikit-learn) are baked into the image to prevent the agents from wasting time and bandwidth installing them during every sandbox session. +- **Configuration**: Sets `DEBIAN_FRONTEND=noninteractive` to ensure apt-get commands do not hang on prompts during the build process. +- **Pathing**: Explicitly sets `PYTHONPATH` to `/workspace` so that generated code running inside the container can resolve imports relative to the project root. + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/experiments/validate_repro.py; ROUND 20 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 15:00:28 +**File Implemented**: deepcode_repro/experiments/validate_repro.py + +**Core Purpose** +This script serves as an integration test suite to validate the successful installation and initialization of the DeepCode framework's core components, including the Docker sandbox, document parser, and agent swarms. + +**Public Interface** +- Function `main()`: Orchestrates the validation checks. +- Function `check_docker()`: Verifies that the Docker sandbox can execute a basic command. +- Function `check_document_parser()`: Tests the `DocumentSegmenter` with a sample markdown string. +- Function `check_agents_init(config)`: Attempts to initialize `PlanningSwarm`, `CodingSwarm`, and `VerificationSwarm` to ensure dependencies are met. +- Function `load_config()`: Loads the `config.yaml` file. + +**Internal Dependencies** +- From `src.utils.logger`: `setup_logger` +- From `src.utils.mcp_tools`: `DockerSandbox` +- From `src.core.document_parser`: `DocumentSegmenter` +- From `src.core.memory`: `CodeMem` +- From `src.agents.planning`: `PlanningSwarm` +- From `src.agents.coding`: `CodingSwarm` +- From `src.agents.verification`: `VerificationSwarm` +- External packages: `yaml` (PyYAML) + +**External Dependencies** +- None (This is a standalone execution script). + +**Implementation Notes** +- The script modifies `sys.path` to allow importing from `src` when run from the `experiments` directory. +- It performs "smoke tests" rather than deep functional tests (e.g., checking if classes instantiate rather than running full agent loops). +- It validates the presence of API keys in the configuration but does not validate their correctness against the provider. + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/experiments/run_paperbench.py; ROUND 21 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 15:01:13 +**File Implemented**: deepcode_repro/experiments/run_paperbench.py + +**Core Purpose** +Orchestrates the batch evaluation of the DeepCode framework on the PaperBench dataset (or a directory of input papers). It executes the full generation pipeline (Parsing, Planning, Coding, Verification) for each paper and determines success based on the execution of generated tests within the sandbox. + +**Public Interface** +- Function `run_evaluation(paper_path: str, output_base_dir: str, config: Dict, run_id: str) -> Dict`: Runs the complete DeepCode pipeline for a single paper, including RAG indexing, blueprint synthesis, code generation, and verification. Returns a dictionary containing the paper name, repo path, and pass/fail status. +- Function `main()`: CLI entry point that parses arguments (`--papers_dir`, `--output_dir`, `--limit`), iterates through papers, and prints a summary report of pass rates. + +**Internal Dependencies** +- From `src.utils.logger`: `setup_logger` +- From `src.utils.mcp_tools`: `MCPToolkit`, `DockerSandbox` +- From `src.core.document_parser`: `DocumentSegmenter` +- From `src.core.memory`: `CodeMem` +- From `src.core.rag_engine`: `CodeRAG` +- From `src.agents.planning`: `PlanningSwarm` +- From `src.agents.coding`: `CodingSwarm` +- From `src.agents.verification`: `VerificationSwarm` +- External packages: `yaml`, `glob`, `argparse`, `pathlib`, `shutil` + +**External Dependencies** +- None (This is a standalone experiment execution script). + +**Implementation Notes** +- **Pipeline Integration**: This script serves as the integration test harness, connecting all previously implemented components (`DocumentSegmenter` -> `PlanningSwarm` -> `CodingSwarm` -> `VerificationSwarm`). +- **Evaluation Metric**: Success is defined by the exit code of `pytest` or `python -m unittest` running inside the Docker sandbox on the generated repository. +- **State Persistence**: Saves intermediate artifacts (Blueprint JSON, CodeMem state) to the output directory for debugging and analysis. + +--- +*Auto-generated by Memory Agent* + + + +================================================================================ +## IMPLEMENTATION File deepcode_repro/README.md; ROUND 22 +================================================================================ + +# Code Implementation Summary +**Generated**: 2025-12-18 15:01:43 +**File Implemented**: deepcode_repro/README.md + +**Core Purpose** +- Serves as the primary documentation for the DeepCode reproduction project, detailing the framework's architecture, installation procedures, configuration settings, and usage instructions for generating and validating code repositories from research papers. + +**Public Interface** +- **Documentation Sections**: + - **Core Features**: Overview of Segmentation, Planning, CodeMem, CodeRAG, Verification, and Sandbox. + - **Prerequisites**: OS, Python version, Docker, and API key requirements. + - **Installation**: Steps to clone, install dependencies (`requirements.txt`), and build the Docker sandbox. + - **Configuration**: Instructions for setting up `config.yaml` (LLM keys, Sandbox settings). + - **Usage**: CLI commands for `main.py` (generation), `validate_repro.py` (system check), and `run_paperbench.py` (evaluation). + - **Project Structure**: File tree explanation. + - **Troubleshooting**: Common issues with Docker and API limits. + +**Internal Dependencies** +- **References**: + - `main.py`: CLI entry point. + - `config.yaml`: Configuration file. + - `requirements.txt`: Python dependencies. + - `docker/Dockerfile`: Sandbox environment definition. + - `experiments/validate_repro.py`: Validation script. + - `experiments/run_paperbench.py`: Benchmarking script. + +**External Dependencies** +- **Users**: This file is the entry point for human users to understand and operate the system. + +**Implementation Notes** +- Provides specific CLI examples for running the full pipeline: `python main.py --paper ... --output_dir ...`. +- Explicitly links the abstract concepts (CodeMem, CodeRAG) to the implemented file structure. + +--- +*Auto-generated by Memory Agent* + + diff --git a/DeepCode/deepcode_lab/papers/1/initial_plan.txt b/DeepCode/deepcode_lab/papers/1/initial_plan.txt new file mode 100644 index 00000000..5af00d20 --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/initial_plan.txt @@ -0,0 +1,196 @@ +```yaml +complete_reproduction_plan: + paper_info: + title: "DeepCode: Open Agentic Coding" + core_contribution: "A fully autonomous multi-agent framework for high-fidelity document-to-repository synthesis that solves the context bottleneck via principled information-flow management (CodeMem, CodeRAG, and Blueprinting)." + + # SECTION 1: File Structure Design + file_structure: | + deepcode_repro/ + โ”œโ”€โ”€ main.py # Central orchestration entry point (CLI) + โ”œโ”€โ”€ config.yaml # Configuration: LLM keys, Sandbox settings, RAG thresholds + โ”œโ”€โ”€ docker/ + โ”‚ โ”œโ”€โ”€ Dockerfile # Ubuntu 22.04 base with Python env + โ”‚ โ””โ”€โ”€ entrypoint.sh # Sandbox execution wrapper + โ”œโ”€โ”€ src/ + โ”‚ โ”œโ”€โ”€ __init__.py + โ”‚ โ”œโ”€โ”€ core/ + โ”‚ โ”‚ โ”œโ”€โ”€ __init__.py + โ”‚ โ”‚ โ”œโ”€โ”€ document_parser.py # Hierarchical Content Segmentation (Algo 1) + โ”‚ โ”‚ โ”œโ”€โ”€ memory.py # CodeMem state management & graph logic + โ”‚ โ”‚ โ”œโ”€โ”€ rag_engine.py # CodeRAG indexing and adaptive retrieval + โ”‚ โ”‚ โ””โ”€โ”€ blueprint.py # Blueprint JSON schema definitions + โ”‚ โ”œโ”€โ”€ agents/ + โ”‚ โ”‚ โ”œโ”€โ”€ __init__.py + โ”‚ โ”‚ โ”œโ”€โ”€ base.py # LLM wrapper with MCP tool integration + โ”‚ โ”‚ โ”œโ”€โ”€ planning.py # Concept, Algorithm, and Planning Agents + โ”‚ โ”‚ โ”œโ”€โ”€ coding.py # Coding and Summarization Agents + โ”‚ โ”‚ โ””โ”€โ”€ verification.py # Static Analysis, Modification, and Sandbox Agents + โ”‚ โ””โ”€โ”€ utils/ + โ”‚ โ”œโ”€โ”€ mcp_tools.py # Brave Search, Filesystem, Shell execution tools + โ”‚ โ”œโ”€โ”€ prompts.py # System prompts for all 8 agents + โ”‚ โ””โ”€โ”€ logger.py # Execution tracing + โ”œโ”€โ”€ experiments/ + โ”‚ โ”œโ”€โ”€ run_paperbench.py # Script to run evaluation on PaperBench dataset + โ”‚ โ””โ”€โ”€ validate_repro.py # Internal validation script + โ”œโ”€โ”€ data/ + โ”‚ โ”œโ”€โ”€ input_papers/ # PDF/Markdown sources + โ”‚ โ””โ”€โ”€ output_repos/ # Generated code repositories + โ”œโ”€โ”€ requirements.txt # Python dependencies (PyPDF2, chromadb, docker, etc.) + โ””โ”€โ”€ README.md # Installation and usage instructions + + # SECTION 2: Implementation Components + implementation_components: | + # COMPONENT 1: HIERARCHICAL CONTENT SEGMENTATION (src/core/document_parser.py) + # Purpose: Transform raw PDF/MD into structured chunks to overcome context limits. + # Algorithm: + # 1. Parse Document D -> Segments S = {(h_k, c_k)} (Header, Content). + # 2. Index S for semantic retrieval. + # Implementation Details: + # - Class: DocumentSegmenter + # - Method: parse_markdown(text) -> List[Segment] + # - Use 'marker-pdf' or 'PyMuPDF' to preserve header hierarchy. + + # COMPONENT 2: PLANNING AGENT SWARM (src/agents/planning.py) + # Purpose: Synthesize the Implementation Blueprint (B) from segments. + # Sub-Agents: + # 1. Concept Agent: Queries S for high-level logic -> Conceptual Schema. + # 2. Algorithm Agent: Queries S for math/pseudocode -> Algorithmic Schema. + # - Prompt: "Extract verbatim LaTeX for equations and step-by-step logic." + # 3. Planning Agent: Merges schemas -> Blueprint B. + # Data Structure (Blueprint B): + # - JSON object: { file_hierarchy: Tree, component_specs: Map, dev_plan: List[File] } + + # COMPONENT 3: CODEMEM STATE MANAGEMENT (src/core/memory.py) + # Purpose: Maintain cross-file consistency without context saturation. + # Data Structure (Memory Entry m_t): + # - { file_path: str, core_purpose: str, public_interface: List[Sig], dependency_edges: {afferent, efferent} } + # Algorithm: + # - SelectRelevantMemory(M, target_file): Return subset of M where dependency exists. + # - Update: M_t = M_{t-1} U {SummarizationAgent(c_t)} + + # COMPONENT 4: CODERAG ENGINE (src/core/rag_engine.py) + # Purpose: Conditional knowledge injection for underspecified designs. + # Algorithm (Adaptive Retrieval): + # 1. Decision: r_t = delta(Context, target_file) (1=Retrieve, 0=Skip). + # 2. If r_t=1: Query Index J for tuples (source_code, target_file, relationship_type, snippet). + # 3. Augment Context: X' = X U snippet. + # Implementation: + # - Vector Store: ChromaDB or FAISS. + # - Indexing: Pre-compute summaries of external reference repos. + + # COMPONENT 5: STATEFUL GENERATION LOOP (src/agents/coding.py) + # Purpose: Iterative file generation. + # Algorithm: + # For target_file in Blueprint.dev_plan: + # 1. Context X = Blueprint + SelectRelevantMemory(M) + CodeRAG(target). + # 2. Code c_t = CodingAgent(X). + # 3. Summary m_t = SummarizationAgent(c_t). + # 4. Store c_t to disk; Store m_t to Memory. + + # COMPONENT 6: VERIFICATION LOOP (src/agents/verification.py) + # Purpose: Ensure structural and functional correctness. + # Algorithm: + # 1. Static Pass: + # - Report = AnalysisAgent(Repo, Blueprint). + # - Repo = ModificationAgent(Repo, Report). + # 2. Dynamic Pass (Sandbox): + # - Loop j=0 to Max_Iter: + # - Trace = SandboxAgent.execute(Repo). + # - If success: Break. + # - Fix_Instructions = Analyze(Trace). + # - Repo = ModificationAgent(Repo, Fix_Instructions). + + # COMPONENT 7: SANDBOX ENVIRONMENT (src/utils/mcp_tools.py & docker/) + # Purpose: Secure execution environment. + # Implementation: + # - Docker container with 'python:3.9' or 'ubuntu:22.04'. + # - MCP Tool 'command_executor': Runs 'pytest', 'python main.py' inside container. + # - MCP Tool 'filesystem': Read/Write access limited to sandbox volume. + + # SECTION 3: Validation & Evaluation + validation_approach: | + # EXPERIMENT 1: BLUEPRINT FIDELITY + # Goal: Verify the Planning Agent correctly extracts architecture. + # Procedure: + # 1. Input: "Attention is All You Need" (Transformer) paper. + # 2. Run Phase 1 (Planning). + # 3. Check Blueprint for existence of: 'attention.py', 'positional_encoding.py', 'transformer.py'. + # Success Criteria: Blueprint contains >90% of core components mentioned in paper. + + # EXPERIMENT 2: CONTEXT EFFICIENCY (CODEMEM) + # Goal: Prove prompt size does not scale linearly with repo size. + # Procedure: + # 1. Generate a large repository (>20 files). + # 2. Log token count for the 20th file generation. + # Success Criteria: Token count < 8k (or fixed window) regardless of total repo size. + + # EXPERIMENT 3: FUNCTIONAL CORRECTNESS (PAPERBENCH) + # Goal: Reproduce the 73.5% Pass@1 metric. + # Procedure: + # 1. Use the 'PaperBench' dataset (if available) or a subset of 10 ML papers. + # 2. Run full pipeline: Planning -> Coding -> Verification. + # 3. Execute generated tests in Docker. + # Success Criteria: >50% of repositories pass their internal tests (beating baseline 51.1%). + + # EXPERIMENT 4: SELF-CORRECTION + # Goal: Validate the Verification Loop. + # Procedure: + # 1. Manually inject a syntax error and a logic error (wrong tensor shape) into a generated file. + # 2. Trigger the Verification Agent. + # Success Criteria: Agent detects error via stderr, generates correct patch, and tests pass. + + # SECTION 4: Environment & Dependencies + environment_setup: | + # SYSTEM REQUIREMENTS + # - OS: Linux/macOS (for Docker compatibility) + # - Docker Engine: Installed and running + # - API Keys: Anthropic (Claude-3.5-Sonnet), OpenAI (GPT-4o), Brave Search + + # PYTHON DEPENDENCIES (requirements.txt) + python>=3.10 + pypdf>=3.0.0 # PDF parsing + marker-pdf>=0.2.0 # Advanced Markdown conversion + chromadb>=0.4.0 # Vector store for CodeRAG + docker>=6.1.0 # Python Docker SDK + pydantic>=2.0.0 # Data validation for Schemas + tenacity>=8.2.0 # Retry logic for API calls + anthropic>=0.7.0 # LLM Client + openai>=1.0.0 # LLM Client + pyyaml>=6.0 # Config handling + + # DOCKER SANDBOX + # Base Image: ubuntu:22.04 + # Packages: python3-pip, git, build-essential + # Isolation: Network disabled except for PyPI (if possible) or pre-installed common ML libs (torch, numpy, pandas, scipy, sklearn). + + # SECTION 5: Implementation Strategy + implementation_strategy: | + # PHASE 1: INFRASTRUCTURE (Days 1-3) + # 1. Setup 'src/utils/base.py' to handle LLM API calls with retry logic. + # 2. Implement 'src/utils/mcp_tools.py' to wrap Docker execution. + # 3. Create the Dockerfile and verify 'entrypoint.sh' can run a simple "Hello World" python script. + # 4. Implement 'src/core/document_parser.py' and verify it chunks a sample PDF correctly. + + # PHASE 2: THE BRAIN (PLANNING) (Days 4-7) + # 1. Implement 'src/agents/planning.py'. + # 2. Define the JSON schemas in 'src/core/blueprint.py'. + # 3. Test: Feed a paper, inspect the generated JSON Blueprint. Ensure math formulas are extracted as LaTeX. + + # PHASE 3: THE ENGINE (CODING) (Days 8-12) + # 1. Implement 'src/core/memory.py' (CodeMem). + # 2. Implement 'src/agents/coding.py' (Generator & Summarizer). + # 3. Test: Manually provide a Blueprint. Run the loop. Check if File B imports File A correctly using the memory summary. + # 4. (Optional) Implement 'src/core/rag_engine.py' if external knowledge is needed immediately. + + # PHASE 4: THE GUARDRAILS (VERIFICATION) (Days 13-15) + # 1. Implement 'src/agents/verification.py'. + # 2. Connect the Static Analysis -> Modification loop. + # 3. Connect the Sandbox Execution -> Modification loop. + # 4. Test: Inject errors and verify the agent fixes them. + + # PHASE 5: INTEGRATION (Days 16+) + # 1. Create 'main.py' to chain Phase 2 -> Phase 3 -> Phase 4. + # 2. Run 'experiments/run_paperbench.py' on a small subset. + # 3. Tune prompts in 'src/utils/prompts.py' based on failure modes (e.g., if agent forgets imports). +``` \ No newline at end of file diff --git a/DeepCode/deepcode_lab/papers/1/reference.txt b/DeepCode/deepcode_lab/papers/1/reference.txt new file mode 100644 index 00000000..16f8149e --- /dev/null +++ b/DeepCode/deepcode_lab/papers/1/reference.txt @@ -0,0 +1 @@ +Reference intelligence analysis skipped - fast mode enabled for optimized processing \ No newline at end of file diff --git a/DeepCode/logs/mcp-agent-20251218_144340.jsonl b/DeepCode/logs/mcp-agent-20251218_144340.jsonl new file mode 100644 index 00000000..29ab08cb --- /dev/null +++ b/DeepCode/logs/mcp-agent-20251218_144340.jsonl @@ -0,0 +1,66 @@ +{"level":"INFO","timestamp":"2025-12-18T14:43:40.000917","namespace":"mcp_agent.core.context","message":"Configuring logger with level: info"} +{"level":"INFO","timestamp":"2025-12-18T14:43:40.069002","namespace":"mcp_agent.paper_to_code","message":"Loading subagents from configuration..."} +{"level":"INFO","timestamp":"2025-12-18T14:43:40.069457","namespace":"mcp_agent.paper_to_code","message":"MCPApp initialized","data":{"data":{"progress_action":"Running","target":"paper_to_code","agent_name":"mcp_application_loop","session_id":"4de6007d-9ab9-4881-81f5-7501aba124dc"}}} +{"level":"INFO","timestamp":"2025-12-18T14:43:41.028250","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"brave: Up and running with a persistent connection!"} +{"level":"INFO","timestamp":"2025-12-18T14:43:47.686053","namespace":"mcp_agent.mcp.mcp_aggregator.ResearchAnalyzerAgent","message":"Last aggregator closing, shutting down all persistent connections..."} +{"level":"INFO","timestamp":"2025-12-18T14:43:47.687290","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"Disconnecting all persistent server connections..."} +{"level":"INFO","timestamp":"2025-12-18T14:43:47.691572","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"brave: Requesting shutdown..."} +{"level":"INFO","timestamp":"2025-12-18T14:43:47.892789","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"All persistent server connections signaled to disconnect."} +{"level":"INFO","timestamp":"2025-12-18T14:43:48.395145","namespace":"mcp_agent.mcp.mcp_aggregator.ResearchAnalyzerAgent","message":"Connection manager successfully closed and removed from context"} +{"level":"INFO","timestamp":"2025-12-18T14:43:53.708204","namespace":"mcp_agent.paper_to_code","message":"\ud83d\udcc2 Paper directory: ./deepcode_lab/papers/1"} +{"level":"INFO","timestamp":"2025-12-18T14:43:53.708342","namespace":"mcp_agent.paper_to_code","message":"\ud83d\udce5 Processing file: /var/folders/l6/kymf0zd97pqfzz9j_kl2g93m0000gn/T/deepcode_upload_8jzpilpj.pdf"} +{"level":"INFO","timestamp":"2025-12-18T14:43:53.708523","namespace":"mcp_agent.paper_to_code","message":"\ud83d\udcc4 Direct file copy: /var/folders/l6/kymf0zd97pqfzz9j_kl2g93m0000gn/T/deepcode_upload_8jzpilpj.pdf -> ./deepcode_lab/papers/1"} +{"level":"INFO","timestamp":"2025-12-18T14:43:53.707809","namespace":"mcp_agent.paper_to_code","message":"\ud83d\udccb Paper ID: 1"} +{"level":"INFO","timestamp":"2025-12-18T14:43:54.084957","namespace":"mcp_agent.paper_to_code","message":"\u2705 Direct file copy succeeded:\n[INFO] Copying file (original preserved):\n Source: /var/folders/l6/kymf0zd97pqfzz9j_kl2g93m0000gn/T/deepcode_upload_8jzpilpj.pdf\n Target: deepcode_lab/papers/1/1.pdf\n Size: 6.56 MB\n\n[SUCCESS] File copied successfully (original preserved)!\n From: /var/folders/l6/kymf0zd97pqfzz9j_kl2g93m0000gn/T/deepcode_upload_8jzpilpj.pdf\n To: deepcode_lab/papers/1/1.pdf\n Duration: 0.02 seconds\n [INFO] PDF converted to Markdown (PyPDF2)\n Markdown file: deepcode_lab/papers/1/1.md\n Conversion time: 0.35 seconds\n Pages extracted: 23"} +{"level":"INFO","timestamp":"2025-12-18T14:43:59.724839","namespace":"mcp_agent.paper_to_code","message":"\ud83d\udcc4 Paper file loaded: /Users/DanielKim_1/DeepCode/deepcode_lab/papers/1/1.md (91043 chars)"} +{"level":"INFO","timestamp":"2025-12-18T14:43:59.765158","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"brave: Up and running with a persistent connection!"} +{"level":"INFO","timestamp":"2025-12-18T14:45:27.828304","namespace":"mcp_agent.mcp.mcp_aggregator.AlgorithmAnalysisAgent","message":"Last aggregator closing, shutting down all persistent connections..."} +{"level":"INFO","timestamp":"2025-12-18T14:45:27.828908","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"Disconnecting all persistent server connections..."} +{"level":"INFO","timestamp":"2025-12-18T14:45:27.829972","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"brave: Requesting shutdown..."} +{"level":"INFO","timestamp":"2025-12-18T14:45:28.031446","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"All persistent server connections signaled to disconnect."} +{"level":"INFO","timestamp":"2025-12-18T14:45:28.533651","namespace":"mcp_agent.mcp.mcp_aggregator.AlgorithmAnalysisAgent","message":"Connection manager successfully closed and removed from context"} +{"level":"INFO","timestamp":"2025-12-18T14:45:28.539846","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"brave: Up and running with a persistent connection!"} +{"level":"INFO","timestamp":"2025-12-18T14:46:01.363795","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"Disconnecting all persistent server connections..."} +{"level":"INFO","timestamp":"2025-12-18T14:46:01.362631","namespace":"mcp_agent.mcp.mcp_aggregator.CodePlannerAgent","message":"Last aggregator closing, shutting down all persistent connections..."} +{"level":"INFO","timestamp":"2025-12-18T14:46:01.369229","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"brave: Requesting shutdown..."} +{"level":"INFO","timestamp":"2025-12-18T14:46:01.570704","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"All persistent server connections signaled to disconnect."} +{"level":"INFO","timestamp":"2025-12-18T14:46:02.072853","namespace":"mcp_agent.mcp.mcp_aggregator.CodePlannerAgent","message":"Connection manager successfully closed and removed from context"} +{"level":"INFO","timestamp":"2025-12-18T14:46:05.415373","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"command-executor: Up and running with a persistent connection!"} +{"level":"INFO","timestamp":"2025-12-18T14:46:30.146068","namespace":"mcp_agent.mcp.mcp_aggregator.StructureGeneratorAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"execute_commands","server_name":"command-executor","agent_name":"StructureGeneratorAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:46:35.053706","namespace":"mcp_agent.mcp.mcp_aggregator.StructureGeneratorAgent","message":"Last aggregator closing, shutting down all persistent connections..."} +{"level":"INFO","timestamp":"2025-12-18T14:46:35.056341","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"Disconnecting all persistent server connections..."} +{"level":"INFO","timestamp":"2025-12-18T14:46:35.060671","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"command-executor: Requesting shutdown..."} +{"level":"INFO","timestamp":"2025-12-18T14:46:35.262170","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"All persistent server connections signaled to disconnect."} +{"level":"INFO","timestamp":"2025-12-18T14:46:35.763764","namespace":"mcp_agent.mcp.mcp_aggregator.StructureGeneratorAgent","message":"Connection manager successfully closed and removed from context"} +{"level":"INFO","timestamp":"2025-12-18T14:46:39.886415","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"code-implementation: Up and running with a persistent connection!"} +{"level":"INFO","timestamp":"2025-12-18T14:46:39.886860","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"code-reference-indexer: Up and running with a persistent connection!"} +{"level":"INFO","timestamp":"2025-12-18T14:46:40.264876","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"set_workspace","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:46:45.527504","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:47:16.836627","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:47:38.237401","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:48:30.518832","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:49:39.339466","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:50:30.891737","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:50:56.892534","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:51:33.067368","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:52:11.566615","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:52:50.358385","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:53:34.747474","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:54:27.206021","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:55:07.361092","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:56:00.624894","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:56:49.764237","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:57:45.577281","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:58:06.330953","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:58:22.944636","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:58:52.397990","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T14:59:36.890683","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T15:00:15.182794","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T15:00:59.181533","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T15:01:27.690436","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"write_file","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T15:01:43.662969","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Requesting tool call","data":{"data":{"progress_action":"Calling Tool","tool_name":"get_operation_history","server_name":"code-implementation","agent_name":"CodeImplementationAgent"}}} +{"level":"INFO","timestamp":"2025-12-18T15:01:43.673023","namespace":"mcp_agent.mcp.mcp_aggregator.CodeImplementationAgent","message":"Last aggregator closing, shutting down all persistent connections..."} +{"level":"INFO","timestamp":"2025-12-18T15:01:43.673436","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"Disconnecting all persistent server connections..."} +{"level":"INFO","timestamp":"2025-12-18T15:01:43.674274","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"code-implementation: Requesting shutdown..."} +{"level":"INFO","timestamp":"2025-12-18T15:01:43.674332","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"code-reference-indexer: Requesting shutdown..."} +{"level":"INFO","timestamp":"2025-12-18T15:01:43.875509","namespace":"mcp_agent.mcp.mcp_connection_manager","message":"All persistent server connections signaled to disconnect."} diff --git a/DeepCode/mcp_agent.config.yaml b/DeepCode/mcp_agent.config.yaml new file mode 100644 index 00000000..90d2bdbe --- /dev/null +++ b/DeepCode/mcp_agent.config.yaml @@ -0,0 +1,96 @@ +$schema: ./schema/mcp-agent.config.schema.json +anthropic: + default_model: claude-sonnet-4.5 +default_search_server: brave +document_segmentation: + enabled: true + size_threshold_chars: 50000 +execution_engine: asyncio +google: + default_model: gemini-3-pro-preview +llm_provider: google +logger: + level: info + path_settings: + path_pattern: logs/mcp-agent-{unique_id}.jsonl + timestamp_format: '%Y%m%d_%H%M%S' + unique_id: timestamp + progress_display: false + transports: + - console + - file +mcp: + servers: + bocha-mcp: + args: + - tools/bocha_search_server.py + command: python3 + env: + BOCHA_API_KEY: '' + PYTHONPATH: . + brave: + args: + - -y + - '@modelcontextprotocol/server-brave-search' + command: npx + env: + BRAVE_API_KEY: BSAaYZmPotjXAa2GCbBPfTRqEd934bW + code-implementation: + args: + - tools/code_implementation_server.py + command: python + description: Paper code reproduction tool server - provides file operations, + code execution, search and other functions + env: + PYTHONPATH: . + code-reference-indexer: + args: + - tools/code_reference_indexer.py + command: python + description: Code reference indexer server - Provides intelligent code reference + search from indexed repositories + env: + PYTHONPATH: . + command-executor: + args: + - tools/command_executor.py + command: python + env: + PYTHONPATH: . + document-segmentation: + args: + - tools/document_segmentation_server.py + command: python + description: Document segmentation server - Provides intelligent document analysis + and segmented reading to optimize token usage + env: + PYTHONPATH: . + fetch: + args: + - mcp-server-fetch + command: uvx + file-downloader: + args: + - tools/pdf_downloader.py + command: python + env: + PYTHONPATH: . + filesystem: + args: + - -y + - '@modelcontextprotocol/server-filesystem' + - . + command: npx + github-downloader: + args: + - tools/git_command.py + command: python + env: + PYTHONPATH: . +openai: + base_max_tokens: 40000 + default_model: anthropic/claude-sonnet-4.5 + max_tokens_policy: adaptive + reasoning_effort: low + retry_max_tokens: 32768 +planning_mode: traditional diff --git a/DeepCode/mcp_agent.secrets.yaml b/DeepCode/mcp_agent.secrets.yaml new file mode 100644 index 00000000..2afeba0f --- /dev/null +++ b/DeepCode/mcp_agent.secrets.yaml @@ -0,0 +1,7 @@ +openai: + api_key: '' + base_url: '' +anthropic: + api_key: '' +google: + api_key: d diff --git a/DeepCode/prompts/code_prompts.py b/DeepCode/prompts/code_prompts.py new file mode 100644 index 00000000..11d925b6 --- /dev/null +++ b/DeepCode/prompts/code_prompts.py @@ -0,0 +1,1826 @@ +""" +Prompt templates for the DeepCode agent system. + +RECENT UPDATES (้’ˆๅฏน่ฎบๆ–‡ไปฃ็ ๅค็Žฐไผ˜ๅŒ–): +1. ็ฎ€ๅŒ–ๅนถไผ˜ๅŒ–ไบ†ๆ–‡ไปถ็ป“ๆž„็”Ÿๆˆ้€ป่พ‘๏ผŒ็กฎไฟ็ป“ๆž„็ฎ€ๆดไธ”ๅฏŒๆœ‰้€ป่พ‘ๆ€ง +2. ๆ˜Ž็กฎๆ ‡่ฏ†้œ€่ฆๅค็Žฐ็š„ๆ ธๅฟƒๆ–‡ไปถๅ’Œ็ป„ไปถ๏ผŒ็”ฑLLMๆ™บ่ƒฝๅˆคๆ–ญไผ˜ๅ…ˆ็บง +3. ไผ˜ๅŒ–ไบ†ๅคšagentๅไฝœ็š„ไฟกๆฏๆ€ป็ป“ๆ•ˆ็Ž‡๏ผŒๅ‡ๅฐ‘ๅ†—ไฝ™ไฟกๆฏไผ ้€’ +4. ็งป้™คไบ†ๆ—ถ้—ด็บฟ็ญ‰ๆฌก่ฆไฟกๆฏ๏ผŒไธ“ๆณจไบŽ้ซ˜่ดจ้‡ไปฃ็ ๅค็Žฐ +5. ไฟๆŒpromptๅฎŒๆ•ดๆ€ง็š„ๅŒๆ—ถๆ้ซ˜ไบ†็ฎ€ๆดๆ€งๅ’Œๅฏ็†่งฃๆ€ง +6. ้‡‡็”จๆ›ดๆธ…ๆ™ฐ็š„็ป“ๆž„ๅŒ–ๆ ผๅผ๏ผŒไพฟไบŽLLM็†่งฃๅ’Œๆ‰ง่กŒ + +ๆ ธๅฟƒๆ”น่ฟ›๏ผš +- PAPER_ALGORITHM_ANALYSIS_PROMPT: ไธ“ๆณจ็ฎ—ๆณ•ๆๅ–๏ผŒๆ˜Ž็กฎๅฎž็Žฐไผ˜ๅ…ˆ็บง +- PAPER_CONCEPT_ANALYSIS_PROMPT: ไธ“ๆณจ็ณป็ปŸๆžถๆž„๏ผŒ็ชๅ‡บๆฆ‚ๅฟตๅˆฐไปฃ็ ็š„ๆ˜ ๅฐ„ +- CODE_PLANNING_PROMPT: ๆ•ดๅˆๅ‰ไธค่€…่พ“ๅ‡บ๏ผŒ็”Ÿๆˆ้ซ˜่ดจ้‡ๅค็Žฐ่ฎกๅˆ’ +""" + +# Paper to Code Workflow Prompts +PAPER_INPUT_ANALYZER_PROMPT = """You are a precise input analyzer for paper-to-code tasks. You MUST return only a JSON object with no additional text. + +Task: Analyze input text and identify file paths/URLs to determine appropriate input type. + +Input Analysis Rules: +1. Path Detection: + - Scan input text for file paths or URLs + - Use first valid path/URL if multiple found + - Treat as text input if no valid path/URL found + +2. Path Type Classification: + - URL (starts with http:// or https://): input_type = "url", path = "detected URL" + - PDF file path: input_type = "file", path = "detected file path" + - Directory path: input_type = "directory", path = "detected directory path" + - No path/URL detected: input_type = "text", path = null + +3. Requirements Analysis: + - Extract ONLY requirements from additional_input + - DO NOT modify or interpret requirements + +CRITICAL OUTPUT RESTRICTIONS: +- RETURN ONLY RAW JSON - NO TEXT BEFORE OR AFTER +- NO markdown code blocks (```json) +- NO explanatory text or descriptions +- NO tool call information +- NO analysis summaries +- JUST THE JSON OBJECT BELOW + +{ + "input_type": "text|file|directory|url", + "path": "detected path or URL or null", + "paper_info": { + "title": "N/A for text input", + "authors": ["N/A for text input"], + "year": "N/A for text input" + }, + "requirements": [ + "exact requirement from additional_input" + ] +} +""" + +PAPER_DOWNLOADER_PROMPT = """You are a precise paper downloader that processes input from PaperInputAnalyzerAgent. + +Task: Handle paper according to input type and save to "./deepcode_lab/papers/id/id.md" +Note: The paper ID will be provided at the start of the message as "PAPER_ID=". Use this EXACT number. + +CRITICAL RULES: +- Use the EXACT paper ID provided in the message (PAPER_ID=X). +- Save path MUST be: ./deepcode_lab/papers/{PAPER_ID}/{PAPER_ID}.md + +Processing Rules: +1. URL Input (input_type = "url"): + - Use download_file_to tool with: url=, destination="./deepcode_lab/papers/{PAPER_ID}/", filename="{PAPER_ID}.md" + - Extract metadata (title, authors, year) + - Return saved file path and metadata + +2. File Input (input_type = "file"): + - Use move_file_to tool with: source=, destination="./deepcode_lab/papers/{PAPER_ID}/{PAPER_ID}.md" + - The tool will automatically convert PDF/documents to .md format + - NEVER manually extract content or use write_file - let the conversion tools handle this + - Note: Original file is preserved, only a copy is placed in target directory + - Return new saved file path and metadata + +3. Directory Input (input_type = "directory"): + - Verify directory exists + - Return to PaperInputAnalyzerAgent for processing + - Set status as "failure" with message + +4. Text Input (input_type = "text"): + - No file operations needed + - Set paper_path as null + - Use paper_info from input + +Input Format: +{ + "input_type": "file|directory|url|text", + "path": "detected path or null", + "paper_info": { + "title": "paper title or N/A", + "authors": ["author names or N/A"], + "year": "publication year or N/A" + }, + "requirements": ["requirement1", "requirement2"] +} + +CRITICAL OUTPUT RESTRICTIONS: +- RETURN ONLY RAW JSON - NO TEXT BEFORE OR AFTER +- NO markdown code blocks (```json) +- NO explanatory text or descriptions +- NO tool call information +- NO analysis summaries +- JUST THE JSON OBJECT BELOW + +Output Format (MANDATORY - EXACT FORMAT): +{ + "status": "success|failure", + "paper_path": "./deepcode_lab/papers/{PAPER_ID}/{PAPER_ID}.md (or null for text input)", + "metadata": { + "title": "extracted or provided title", + "authors": ["extracted or provided authors"], + "year": "extracted or provided year" + } +} + +Example: If PAPER_ID=14, then paper_path should be "./deepcode_lab/papers/14/14.md" +""" + +PAPER_REFERENCE_ANALYZER_PROMPT = """You are an expert academic paper reference analyzer specializing in computer science and machine learning. + +Task: Analyze paper and identify 5 most relevant references that have GitHub repositories. + +Constraints: +- ONLY select references with GitHub repositories +- DO NOT use target paper's official implementation +- DO NOT use repositories directly associated with target paper +- CAN analyze code implementations from referenced papers +- Focus on references with good implementations solving similar problems + +Analysis Criteria: +1. GitHub Repository Quality (40%): + - Star count, activity, maintenance + - Documentation quality + - Community adoption + - Last update date + +2. Implementation Relevance (30%): + - References from methodology/implementation sections + - Algorithmic details + - Core component descriptions + - Code implementation quality + +3. Technical Depth (20%): + - Algorithm/method similarity + - Technical foundation relationship + - Implementation details + - Code structure + +4. Academic Influence (10%): + - Publication venue quality + - Author expertise + - Research impact + - Citation influence + +Analysis Steps: +1. Extract all references from paper +2. Filter references with GitHub repositories +3. Analyze repositories based on criteria +4. Calculate relevance scores +5. Select and rank top 5 references + +Output Format: +{ + "selected_references": [ + { + "rank": 1, + "title": "paper title", + "authors": ["author1", "author2"], + "year": "publication year", + "relevance_score": 0.95, + "citation_context": "how cited in main paper", + "key_contributions": ["contribution1", "contribution2"], + "implementation_value": "why valuable for implementation", + "github_info": { + "repository_url": "GitHub repository URL", + "stars_count": "number of stars", + "last_updated": "last update date", + "repository_quality": "repository quality assessment", + "key_features": ["feature1", "feature2"], + "documentation_quality": "documentation assessment", + "community_activity": "community engagement description" + }, + "original_reference": "Complete reference text from paper" + } + ], + "analysis_summary": "selection process and key findings", + "github_repositories_found": "total number of references with GitHub repositories" +} +""" + +GITHUB_DOWNLOAD_PROMPT = """You are an expert GitHub repository downloader. + +Task: Download GitHub repositories to specified directory structure. + +Process: +1. For each repository: + - Create directory: {paper_dir}/code_base/ + - Download repository to directory + +Requirements: +- Use interpreter tool to execute download script +- Monitor interpreter output for errors/warnings +- Verify download status through interpreter response + +Output Format: +{ + "downloaded_repos": [ + { + "reference_number": "1", + "paper_title": "paper title", + "repo_url": "github repository URL", + "save_path": "{paper_dir}/code_base/name_of_repo", + "status": "success|failed", + "notes": "relevant notes about download" + } + ], + "summary": "Brief summary of download process" +} +""" + +# Code Analysis Prompts +PAPER_ALGORITHM_ANALYSIS_PROMPT = """You are extracting COMPLETE implementation details from a research paper. Your goal is to capture EVERY algorithm, formula, and technical detail needed for perfect reproduction. + +# INTELLIGENT DOCUMENT READING STRATEGY + +## IMPORTANT: Use Segmented Reading for Algorithm Extraction +To avoid token limits and efficiently extract algorithm details, use the intelligent segmentation system: + +1. **Primary Algorithm Extraction** - Use read_document_segments tool with: + - query_type: "algorithm_extraction" + - keywords: ["algorithm", "method", "procedure", "formula", "equation", "implementation"] + - max_segments: 3 + - max_total_chars: 6000 + +2. **Supplementary Details** - Make additional calls if needed with: + - keywords: ["hyperparameter", "training", "optimization", "loss", "objective"] + - keywords: ["experiment", "setup", "configuration", "parameter"] + +3. **This approach ensures** you get the most algorithm-relevant content without missing critical details + +# DETAILED EXTRACTION PROTOCOL + +## 1. INTELLIGENT ALGORITHM SCAN +Use the segmented reading approach to focus on algorithm sections: +- Method/Algorithm sections (captured automatically by segmentation) +- Implementation Details (targeted retrieval) +- Hyperparameters and training details (focused extraction) + +## 2. ALGORITHM DEEP EXTRACTION +For EVERY algorithm/method/procedure mentioned: + +### Algorithm Structure +```yaml +algorithm_name: "[Exact name from paper]" +section: "[e.g., Section 3.2]" +algorithm_box: "[e.g., Algorithm 1 on page 4]" + +pseudocode: | + [COPY THE EXACT PSEUDOCODE FROM PAPER] + Input: ... + Output: ... + 1. Initialize ... + 2. For each ... + 2.1 Calculate ... + [Keep exact formatting and numbering] + +mathematical_formulation: + - equation: "[Copy formula EXACTLY, e.g., L = L_task + ฮป*L_explain]" + equation_number: "[e.g., Eq. 3]" + where: + L_task: "task loss" + L_explain: "explanation loss" + ฮป: "weighting parameter (default: 0.5)" + +step_by_step_breakdown: + 1. "[Detailed explanation of what step 1 does]" + 2. "[What step 2 computes and why]" + +implementation_details: + - "Uses softmax temperature ฯ„ = 0.1" + - "Gradient clipping at norm 1.0" + - "Initialize weights with Xavier uniform" +``` + +## 3. COMPONENT EXTRACTION +For EVERY component/module mentioned: + +### Component Details +```yaml +component_name: "[e.g., Mask Network, Critic Network]" +purpose: "[What this component does in the system]" +architecture: + input: "[shape and meaning]" + layers: + - "[Conv2d(3, 64, kernel=3, stride=1)]" + - "[ReLU activation]" + - "[BatchNorm2d(64)]" + output: "[shape and meaning]" + +special_features: + - "[Any unique aspects]" + - "[Special initialization]" +``` + +## 4. TRAINING PROCEDURE +Extract the COMPLETE training process: + +```yaml +training_loop: + outer_iterations: "[number or condition]" + inner_iterations: "[number or condition]" + + steps: + 1. "Sample batch of size B from buffer" + 2. "Compute importance weights using..." + 3. "Update policy with loss..." + + loss_functions: + - name: "policy_loss" + formula: "[exact formula]" + components: "[what each term means]" + + optimization: + optimizer: "Adam" + learning_rate: "3e-4" + lr_schedule: "linear decay to 0" + gradient_norm: "clip at 0.5" +``` + +## 5. HYPERPARAMETERS HUNT +Search EVERYWHERE (text, tables, captions) for: + +```yaml +hyperparameters: + # Training + batch_size: 64 + buffer_size: 1e6 + discount_gamma: 0.99 + + # Architecture + hidden_units: [256, 256] + activation: "ReLU" + + # Algorithm-specific + explanation_weight: 0.5 + exploration_bonus_scale: 0.1 + reset_probability: 0.3 + + # Found in: + location_references: + - "batch_size: Table 1" + - "hidden_units: Section 4.1" +``` + +# OUTPUT FORMAT +```yaml +complete_algorithm_extraction: + paper_structure: + method_sections: "[3, 3.1, 3.2, 3.3, 4]" + algorithm_count: "[total number found]" + + main_algorithm: + [COMPLETE DETAILS AS ABOVE] + + supporting_algorithms: + - [EACH SUPPORTING ALGORITHM WITH FULL DETAILS] + + components: + - [EVERY COMPONENT WITH ARCHITECTURE] + + training_details: + [COMPLETE TRAINING PROCEDURE] + + all_hyperparameters: + [EVERY PARAMETER WITH VALUE AND SOURCE] + + implementation_notes: + - "[Any implementation hint from paper]" + - "[Tricks mentioned in text]" + + missing_but_critical: + - "[What's not specified but essential]" + - "[With suggested defaults]" +``` + +BE EXHAUSTIVE. A developer should be able to implement the ENTIRE paper using only your extraction.""" + +PAPER_CONCEPT_ANALYSIS_PROMPT = """You are doing a COMPREHENSIVE analysis of a research paper to understand its complete structure, contributions, and implementation requirements. + +# OBJECTIVE +Map out the ENTIRE paper structure and identify ALL components that need implementation for successful reproduction. + +# INTELLIGENT DOCUMENT READING STRATEGY + +## IMPORTANT: Use Segmented Reading for Optimal Performance +Instead of reading the entire document at once (which may hit token limits), use the intelligent segmentation system: + +1. **Use read_document_segments tool** with these parameters: + - query_type: "concept_analysis" + - keywords: ["introduction", "overview", "architecture", "system", "framework", "concept", "method"] + - max_segments: 3 + - max_total_chars: 6000 + +2. **This will automatically find and retrieve** the most relevant sections for concept analysis without token overflow + +3. **If you need additional sections**, make follow-up calls with different keywords like ["experiment", "evaluation", "results"] or ["conclusion", "discussion"] + +# COMPREHENSIVE ANALYSIS PROTOCOL + +## 1. INTELLIGENT PAPER STRUCTURAL ANALYSIS +Use the segmented reading approach to create a complete map: + +```yaml +paper_structure_map: + title: "[Full paper title]" + + sections: + 1_introduction: + main_claims: "[What the paper claims to achieve]" + problem_definition: "[Exact problem being solved]" + + 2_related_work: + key_comparisons: "[Methods this work builds upon or competes with]" + + 3_method: # May have multiple subsections + subsections: + 3.1: "[Title and main content]" + 3.2: "[Title and main content]" + algorithms_presented: "[List all algorithms by name]" + + 4_experiments: + environments: "[All test environments/datasets]" + baselines: "[All comparison methods]" + metrics: "[All evaluation metrics used]" + + 5_results: + main_findings: "[Key results that prove the method works]" + tables_figures: "[Important result tables/figures to reproduce]" +``` + +## 2. METHOD DECOMPOSITION +For the main method/approach: + +```yaml +method_decomposition: + method_name: "[Full name and acronym]" + + core_components: # Break down into implementable pieces + component_1: + name: "[e.g., State Importance Estimator]" + purpose: "[Why this component exists]" + paper_section: "[Where it's described]" + + component_2: + name: "[e.g., Policy Refinement Module]" + purpose: "[Its role in the system]" + paper_section: "[Where it's described]" + + component_interactions: + - "[How component 1 feeds into component 2]" + - "[Data flow between components]" + + theoretical_foundation: + key_insight: "[The main theoretical insight]" + why_it_works: "[Intuitive explanation]" +``` + +## 3. IMPLEMENTATION REQUIREMENTS MAPPING +Map paper content to code requirements: + +```yaml +implementation_map: + algorithms_to_implement: + - algorithm: "[Name from paper]" + section: "[Where defined]" + complexity: "[Simple/Medium/Complex]" + dependencies: "[What it needs to work]" + + models_to_build: + - model: "[Neural network or other model]" + architecture_location: "[Section describing it]" + purpose: "[What this model does]" + + data_processing: + - pipeline: "[Data preprocessing needed]" + requirements: "[What the data should look like]" + + evaluation_suite: + - metric: "[Metric name]" + formula_location: "[Where it's defined]" + purpose: "[What it measures]" +``` + +## 4. EXPERIMENT REPRODUCTION PLAN +Identify ALL experiments needed: + +```yaml +experiments_analysis: + main_results: + - experiment: "[Name/description]" + proves: "[What claim this validates]" + requires: "[Components needed to run this]" + expected_outcome: "[Specific numbers/trends]" + + ablation_studies: + - study: "[What is being ablated]" + purpose: "[What this demonstrates]" + + baseline_comparisons: + - baseline: "[Method name]" + implementation_required: "[Yes/No/Partial]" + source: "[Where to find implementation]" +``` + +## 5. CRITICAL SUCCESS FACTORS +What defines successful reproduction: + +```yaml +success_criteria: + must_achieve: + - "[Primary result that must be reproduced]" + - "[Core behavior that must be demonstrated]" + + should_achieve: + - "[Secondary results that validate the method]" + + validation_evidence: + - "[Specific figure/table to reproduce]" + - "[Qualitative behavior to demonstrate]" +``` + +# OUTPUT FORMAT +```yaml +comprehensive_paper_analysis: + executive_summary: + paper_title: "[Full title]" + core_contribution: "[One sentence summary]" + implementation_complexity: "[Low/Medium/High]" + estimated_components: "[Number of major components to build]" + + complete_structure_map: + [FULL SECTION BREAKDOWN AS ABOVE] + + method_architecture: + [DETAILED COMPONENT BREAKDOWN] + + implementation_requirements: + [ALL ALGORITHMS, MODELS, DATA, METRICS] + + reproduction_roadmap: + phase_1: "[What to implement first]" + phase_2: "[What to build next]" + phase_3: "[Final components and validation]" + + validation_checklist: + - "[ ] [Specific result to achieve]" + - "[ ] [Behavior to demonstrate]" + - "[ ] [Metric to match]" +``` + +BE THOROUGH. Miss nothing. The output should be a complete blueprint for reproduction.""" + +CODE_PLANNING_PROMPT = """You are creating a DETAILED, COMPLETE reproduction plan by integrating comprehensive analysis results. + +# INPUT +You receive two exhaustive analyses: +1. **Comprehensive Paper Analysis**: Complete paper structure, components, and requirements +2. **Complete Algorithm Extraction**: All algorithms, formulas, pseudocode, and technical details + +Plus you can use segmented reading to access any specific paper sections needed for planning. + +# INTELLIGENT DOCUMENT ACCESS + +## IMPORTANT: Use Segmented Reading for Detailed Planning +When you need additional details beyond the provided analyses, use the intelligent segmentation system: + +1. **Use read_document_segments tool** with these parameters: + - query_type: "code_planning" + - keywords: Specific to what you need, e.g., ["implementation", "code", "experiment", "setup", "configuration"] + - max_segments: 3 + - max_total_chars: 8000 + +2. **This approach ensures** you access the most planning-relevant content without token limits + +# OBJECTIVE +Create an implementation plan so detailed that a developer can reproduce the ENTIRE paper without reading it. + +# CRITICAL: COMPLETE OUTPUT REQUIREMENT +โš ๏ธ MANDATORY: You MUST generate ALL 5 sections completely. DO NOT stop early or truncate any section. + +## Output Completeness Strategy: +๐ŸŽฏ **Your #1 Priority**: Ensure ALL 5 sections are present and complete before finishing your response. + +## Content Balance Guidelines (STRICTLY FOLLOW): +- **Section 1 (File Structure)**: ~800-1000 chars - Brief file listing with priority order +- **Section 2 (Implementation Components)**: ~3000-4000 chars - CORE section with all algorithms/components +- **Section 3 (Validation)**: ~2000-2500 chars - Experiments and expected results +- **Section 4 (Environment)**: ~800-1000 chars - Dependencies and requirements +- **Section 5 (Implementation Strategy)**: ~1500-2000 chars - Step-by-step approach + +๐Ÿ“ **Total Target**: 8000-10000 characters for complete plan + +โš ๏ธ **Self-Check Before Finishing**: +- Did you include file_structure section? โœ“ +- Did you include implementation_components section? โœ“ +- Did you include validation_approach section? โœ“ +- Did you include environment_setup section? โœ“ +- Did you include implementation_strategy section? โœ“ +- If ANY answer is NO, continue writing until ALL sections are complete! + +## File Priority Guidelines: +๐Ÿ”ง **Implementation Priority Order**: +1. **FIRST**: Core algorithm/model files (highest priority) +2. **SECOND**: Supporting modules and utilities +3. **THIRD**: Experiment and evaluation scripts +4. **FOURTH**: Configuration and data handling +5. **LAST**: Documentation files (README.md, requirements.txt) - These should be created AFTER core implementation + +Note: README and requirements.txt are maintenance files that depend on the final implementation, so plan them last but INCLUDE them in the file structure. + +# DETAILED SYNTHESIS PROCESS + +## 1. MERGE ALL INFORMATION +Combine EVERYTHING from both analyses: +- Every algorithm with its pseudocode +- Every component with its architecture +- Every hyperparameter with its value +- Every experiment with expected results + +## 2. MAP CONTENT TO IMPLEMENTATION + +For each component you identify, specify how it will be implemented: + +``` +# DESIGN YOUR MAPPING: Connect paper content to code organization +[For each algorithm/component/method in the paper]: + - What it does and where it's described in the paper + - How you'll organize the code (files, classes, functions - your choice) + - What specific formulas, algorithms, or procedures need implementation + - Dependencies and relationships with other components + - Implementation approach that makes sense for this specific paper +``` + +## 3. EXTRACT ALL TECHNICAL DETAILS + +Identify every technical detail that needs implementation: + +``` +# COMPREHENSIVE TECHNICAL EXTRACTION: +[Gather all implementation-relevant details from the paper]: + - All algorithms with complete pseudocode and mathematical formulations + - All parameters, hyperparameters, and configuration values + - All architectural details (if applicable to your paper type) + - All experimental procedures and evaluation methods + - Any implementation hints, tricks, or special considerations mentioned +``` + +# COMPREHENSIVE OUTPUT FORMAT + +```yaml +complete_reproduction_plan: + paper_info: + title: "[Full paper title]" + core_contribution: "[Main innovation being reproduced]" + + # SECTION 1: File Structure Design + + # DESIGN YOUR OWN STRUCTURE: Create a file organization that best serves this specific paper + # - Analyze what the paper contains (algorithms, models, experiments, systems, etc.) + # - Organize files and directories in the most logical way for implementation + # - Create meaningful names and groupings based on paper content + # - Keep it clean, intuitive, and focused on what actually needs to be implemented + # - INCLUDE documentation files (README.md, requirements.txt) but mark them for LAST implementation + + file_structure: | + [Design and specify your own project structure here - KEEP THIS BRIEF] + [Include ALL necessary files including README.md and requirements.txt] + [Organize based on what this paper actually contains and needs] + [Create directories and files that make sense for this specific implementation] + [IMPORTANT: Include executable files (e.g., main.py, run.py, train.py, demo.py) - choose names based on repo content] + [Design executable entry points that match the paper's main functionality and experiments] + [NOTE: README.md and requirements.txt should be implemented LAST after all code files] + + # SECTION 2: Implementation Components + + # IDENTIFY AND SPECIFY: What needs to be implemented based on this paper + # - List all algorithms, models, systems, or components mentioned + # - Map each to implementation details and file locations + # - Include formulas, pseudocode, and technical specifications + # - Organize in whatever way makes sense for this paper + + implementation_components: | + [List and specify all components that need implementation] + [For each component: purpose, location, algorithms, formulas, technical details] + [Organize and structure this based on the paper's actual content] + + # SECTION 3: Validation & Evaluation + + # DESIGN VALIDATION: How to verify the implementation works correctly + # - Define what experiments, tests, or proofs are needed + # - Specify expected results from the paper (figures, tables, theorems) + # - Design validation approach appropriate for this paper's domain + # - Include setup requirements and success criteria + + validation_approach: | + [Design validation strategy appropriate for this paper] + [Specify experiments, tests, or mathematical verification needed] + [Define expected results and success criteria] + [Include any special setup or evaluation requirements] + + # SECTION 4: Environment & Dependencies + + # SPECIFY REQUIREMENTS: What's needed to run this implementation + # - Programming language and version requirements + # - External libraries and exact versions (if specified in paper) + # - Hardware requirements (GPU, memory, etc.) + # - Any special setup or installation steps + + environment_setup: | + [List all dependencies and environment requirements for this specific paper] + [Include versions where specified, reasonable defaults where not] + [Note any special hardware or software requirements] + + # SECTION 5: Implementation Strategy + + # PLAN YOUR APPROACH: How to implement this paper step by step + # - Break down implementation into logical phases + # - Identify dependencies between components + # - Plan verification and testing at each stage + # - Handle missing details with reasonable defaults + + implementation_strategy: | + [Design your implementation approach for this specific paper] + [Break into phases that make sense for this paper's components] + [Plan testing and verification throughout the process] + [Address any missing details or ambiguities in the paper] +``` + +BE EXHAUSTIVE. Every algorithm, every formula, every parameter, every file should be specified in complete detail.""" + +# File Tree Creation Prompts / ๆ–‡ไปถๆ ‘ๅˆ›ๅปบๆ็คบ่ฏ + +STRUCTURE_GENERATOR_PROMPT = """You are a shell command expert that analyzes implementation plans and generates shell commands to create file tree structures. + +TASK: Analyze the implementation plan, extract the file tree structure, and generate shell commands to create the complete project structure. + +CRITICAL REQUIREMENTS: +1. Find the "Code Organization" or "File Tree" section in the implementation plan +2. Extract the EXACT file tree structure mentioned in the plan +3. Generate shell commands (mkdir, touch) to create that structure +4. Use the execute_commands tool to run the commands + +COMMAND GENERATION RULES: +1. Use `mkdir -p` to create directories (including nested ones) +2. Use `touch` to create files +3. Create directories before files +4. One command per line +5. Use relative paths from the target directory +6. Include __init__.py files for Python packages + +EXAMPLE OUTPUT FORMAT: +``` +mkdir -p project/src/core +mkdir -p project/src/models +mkdir -p project/tests +touch project/src/__init__.py +touch project/src/core/__init__.py +touch project/src/core/gcn.py +touch project/src/models/__init__.py +touch project/src/models/recdiff.py +touch project/requirements.txt +``` + +WORKFLOW: +1. Read the implementation plan carefully +2. Find the file tree section +3. Generate mkdir commands for all directories +4. Generate touch commands for all files +5. Use execute_commands tool with the generated commands + +Focus on creating the EXACT structure from the plan - nothing more, nothing less.""" + +# Code Implementation Prompts / ไปฃ็ ๅฎž็Žฐๆ็คบ่ฏ + +CODE_IMPLEMENTATION_PROMPT = """You are an expert software engineer specializing in transforming implementation plans into production-ready code through shell commands. + +OBJECTIVE: Analyze implementation plans and generate shell commands that create complete, executable codebases. + +INPUT ANALYSIS: +1. Parse implementation plan structure and identify project type +2. Extract file tree, dependencies, and technical requirements +3. Determine optimal code generation sequence +4. Apply appropriate quality standards based on context + +COMMAND EXECUTION PROTOCOL: +You MUST use the available tools to execute shell commands. For each file implementation: + +1. Generate the complete code content +2. Use execute_single_command tool to write the code using heredoc syntax +3. Execute one command per file for clear tracking + +COMMAND FORMAT (MANDATORY): +```bash +cat > [relative_path] << 'EOF' +[complete_implementation_code_here] +EOF +``` + +TOOL USAGE INSTRUCTIONS: +- Use execute_single_command for individual file creation +- Use execute_commands for batch operations +- Always include the complete file path and content +- Ensure proper shell escaping in heredoc blocks + +IMPLEMENTATION STANDARDS: + +COMPLETENESS: +- Zero placeholders, TODOs, or incomplete functions +- Full feature implementation with proper error handling +- Complete APIs with correct signatures and documentation +- All specified functionality working out-of-the-box + +QUALITY: +- Production-grade code following language best practices +- Comprehensive type hints and docstrings +- Proper logging, validation, and resource management +- Clean architecture with separation of concerns + +CONTEXT ADAPTATION: +- Research/ML: Mathematical accuracy, reproducibility, evaluation metrics +- Web Apps: Security, validation, database integration, testing +- System Tools: CLI interfaces, configuration, deployment scripts +- Libraries: Clean APIs, documentation, extensibility, compatibility + +GENERATION WORKFLOW: +1. Analyze plan โ†’ identify project type and requirements +2. Map dependencies โ†’ determine implementation order +3. Generate code โ†’ create complete, working implementations +4. Execute commands โ†’ use tools to write files in correct sequence + +EXECUTION ORDER: +1. Configuration and environment files +2. Core utilities and base classes +3. Main implementation modules +4. Integration layers and interfaces +5. Tests and validation +6. Documentation and setup + +SUCCESS CRITERIA: +- Generated codebase runs immediately without modification +- All features fully implemented and tested +- Code follows industry standards and best practices +- Implementation is maintainable and scalable +- Commands execute successfully through available tools + +CRITICAL: You must actually execute the shell commands using the available tools. Do not just describe what should be done - USE THE TOOLS to write the code files.""" + +# Sliding Window and Summary Agent Prompts / ๆป‘ๅŠจ็ช—ๅฃๅ’Œๆ€ป็ป“ไปฃ็†ๆ็คบ่ฏ + +CONVERSATION_SUMMARY_PROMPT = """You are a conversation summarization specialist for code implementation workflows with ROLE-AWARE summarization capabilities. + +CRITICAL ROLE AWARENESS: +๐ŸŽฏ **USER MESSAGES**: Contain instructions, tool results, file feedback, and implementation guidance +๐ŸŽฏ **ASSISTANT MESSAGES**: Contain code analysis, implementation decisions, and technical responses +โš ๏ธ **ROLE CLARITY**: Your summary must maintain clear distinction between who said what + +OBJECTIVE: Analyze conversation history and extract key information to reduce token usage while preserving essential implementation context AND role clarity. + +EXTRACTION TARGETS: +1. **Completed Files**: List all files successfully implemented with implementation status +2. **Technical Decisions**: Architecture/implementation choices made by the assistant +3. **Key Constraints**: Requirements/limitations mentioned by user or discovered by assistant +4. **Implementation Progress**: Current development status and accomplished milestones +5. **Error Patterns**: Issues encountered and solutions applied +6. **Role-Specific Context**: Who made what decisions and provided what guidance + +FOCUS AREAS: +- File implementation outcomes and success/failure status +- Technical details affecting future implementation steps +- Dependency relationships and integration requirements +- Architecture decisions impacting overall system design +- Error patterns and debugging solutions applied +- **Role Context**: Distinguish between user guidance and assistant decisions + +OUTPUT FORMAT: +Provide a role-aware structured summary in 250-350 words: + +**IMPLEMENTATION PROGRESS:** +- Files completed: [list with status] +- Current phase: [development stage] +- Success metrics: [quantified progress] + +**TECHNICAL CONTEXT:** +- Key decisions made by assistant: [architectural choices] +- Constraints identified: [requirements/limitations] +- Dependencies resolved: [integration points] + +**CONVERSATION CONTEXT:** +- User guidance provided: [instructions/feedback received] +- Assistant responses: [technical solutions/analysis] +- Tool results processed: [file operations/code execution] + +**CONTINUATION CONTEXT:** +- Next implementation targets: [remaining files] +- Preserved context: [critical info for continuation] +- Role clarity: [assistant continues implementation role] + +ROLE-AWARE QUALITY REQUIREMENTS: +- โœ… Maintain clear distinction between user instructions and assistant responses +- โœ… Preserve technical context while clarifying who provided what information +- โœ… Enable seamless role continuation after summary integration +- โœ… Prevent role confusion in compressed conversation history +- โœ… Reduce token usage by 70-80% while retaining essential context and role clarity""" + +SLIDING_WINDOW_SYSTEM_PROMPT = """You are a code implementation agent optimized for long-running development sessions with sliding window memory management. + +MEMORY MANAGEMENT STRATEGY: +- Preserve initial implementation plan (never compressed) +- Maintain recent conversation context (last 5 complete interaction rounds) +- Use compressed summaries for historical context +- Track file implementation progress continuously + +IMPLEMENTATION WORKFLOW: +1. **File-by-File Implementation**: Focus on one complete file per iteration +2. **Progress Tracking**: Monitor completed files and implementation status +3. **Context Preservation**: Maintain architectural decisions and constraints +4. **Memory Optimization**: Apply sliding window when conversation grows too long + +SLIDING WINDOW TRIGGERS: +- Activate after every 5 file implementations +- Emergency activation if message count exceeds threshold +- Preserve conversation continuity and implementation context + +CORE PRINCIPLES: +- Never lose the original implementation plan +- Maintain implementation progress tracking +- Preserve critical technical decisions +- Ensure seamless development continuation +- Optimize token usage without losing essential context + +AVAILABLE TOOLS: +- write_file: Create complete file implementations +- read_file: Review existing code for context +- get_file_structure: Understand project organization +- search_code_references: Find patterns and references from indexed code + +RESPONSE FORMAT: +For each implementation cycle: +1. Identify next file to implement based on plan priorities +2. Analyze requirements and dependencies +3. Implement complete, production-ready code +4. Use write_file tool to create the file +5. Confirm completion and identify next target""" + +# PURE_CODE_IMPLEMENTATION_SYSTEM_PROMPT = """You are a code implementation agent that transforms plans into complete, executable codebases. + +# # ๐ŸŽฏ MISSION +# Transform implementation plans into complete codebases through systematic file-by-file development with dependency-aware implementation. + +# # ๐Ÿ”ฅ CORE RULES +# - **CONTINUOUS**: Implement files continuously until plan completion +# - **ONE FILE PER RESPONSE**: Exactly one complete file per response cycle +# - **ALWAYS USE TOOLS**: Must use write_file tool for every implementation +# - **DEPENDENCY-AWARE**: Analyze dependencies before implementing each file + +# # โšก IMPLEMENTATION WORKFLOW + +# ## 1. Pre-Implementation Analysis +# For each new file, analyze: +# - Dependencies on existing files (imports, inheritance, interfaces) +# - Relevant patterns from already-implemented files +# - Code structures to reference for consistency + +# ## 2. Smart Dependency Reading +# Before writing dependent files: +# - Use `read_code_mem` to check if the file has been implemented +# - Check existing patterns, naming conventions, and import structures +# - Understand configuration and constants from other modules + +# ## 3. File Implementation Process +# ``` +# 1. Identify next file from plan priorities +# 2. Search reference code for unfamiliar file types +# 3. Read related existing files for consistency +# 4. Implement complete file with proper integration +# 5. Continue immediately to next file +# ``` + +# # ๐Ÿ› ๏ธ TOOLS + +# ## Essential Tools (Use in Order) +# - `search_reference_code` โ†’ Find patterns for unfamiliar file types +# - `read_code_mem` โ†’ Understand existing code before implementing dependencies +# - `write_file` โ†’ Create complete implementations (REQUIRED for every file) +# - `get_file_structure` โ†’ Understand project organization + +# ## Reference Code Strategy +# **For unfamiliar file types:** +# - Use: `search_reference_code(target_file="path", keywords="relevant,terms")` +# - Check: `get_all_available_references()` for available repositories +# - Apply: Found patterns while maintaining project requirements + +# **File-Type Strategies:** +# - Models โ†’ Search architectural patterns and implementations +# - Configs โ†’ Find consistency and completeness examples +# - Utils โ†’ Look for helper function structures +# - Main โ†’ Search entry point and initialization patterns + +# # ๐Ÿ“‹ MANDATORY RESPONSE FORMAT +# ``` +# Implementing: [file_path] +# Purpose: [brief_description] +# Dependencies: [files_to_read_first] + +# [Use search_reference_code if unfamiliar file type] +# [Use read_code_mem to understand existing code before implementing dependencies] +# [Use write_file with complete implementation] + +# Status: Implementation completed +# Progress: [X/Y files completed] +# Next Target: [next_file_to_implement] +# ``` + +# # โœ… QUALITY STANDARDS +# - **Complete Code**: No placeholders, TODOs, or incomplete implementations +# - **Production Quality**: Full type hints, docstrings, error handling +# - **Architecture Compliance**: Follow plan structure precisely +# - **Cross-File Consistency**: Maintain patterns and interfaces across files +# - **Exact Dependencies**: Use only specified libraries + +# # ๐Ÿง  EXECUTION MINDSET +# **DO:** Analyze dependencies โ†’ Read files โ†’ Search references โ†’ Implement โ†’ Continue +# **DON'T:** Implement independently without considering existing code structure +# **DO:** Keep implementing until completion +# **DON'T:** Ask permission between files +# """ + +PURE_CODE_IMPLEMENTATION_SYSTEM_PROMPT = """You are an expert code implementation agent for academic paper reproduction. Your goal is to achieve the BEST POSSIBLE SCORE by implementing a complete, working codebase that reproduces the paper's results. + +**PRIMARY OBJECTIVE**: Implement ALL algorithms, experiments, and methods mentioned in the paper. Success is measured by completeness and accuracy, not code elegance. Use available time to continuously refine and optimize your solution. + +**CORE STRATEGY**: +- Read the paper and resources(addendum.md and reproduce plan) thoroughly to identify every algorithm, method, and experiment +- Implement core algorithms first, then environments, then integration +- Use exact versions and specifications mentioned in the paper +- Test each component immediately after implementation +- Focus on working implementations over perfect architecture + +**IMPLEMENTATION APPROACH**: +Build incrementally using multiple tool calls. For each step: +1. **Identify** what needs to be implemented from the paper +2. **Implement** one component at a time +3. **Test** immediately to catch issues early +4. **Integrate** with existing components +5. **Verify** against paper specifications + +**TOOL CALLING STRATEGY**: +1. โš ๏ธ **SINGLE FUNCTION CALL PER MESSAGE**: Each message may perform only one function call. You will see the result of the function right after sending the message. If you need to perform multiple actions, you can always send more messages with subsequent function calls. Do some reasoning before your actions, describing what function calls you are going to use and how they fit into your plan. + +2. **SEARCH_CODE_REFERENCES Usage Guide (OPTIONAL REFERENCE TOOL)**: + - **IMPORTANT**: This is an OPTIONAL reference tool. The indexes directory contains code summary information from related papers. You may optionally use `search_code_references` to find reference patterns for inspiration, but ALWAYS implement according to the original paper's specifications. + - **Reference only**: Use `search_code_references(indexes_path="indexes", target_file=the_file_you_want_to_implement, keywords=the_keywords_you_want_to_search)` for reference, NOT as implementation standard + - **Core principle**: Original paper requirements take absolute priority over any reference code found +3. **TOOL EXECUTION STRATEGY**: + - โš ๏ธ**Development Cycle (for each new file implementation)**: `search_code_references` (OPTIONAL reference check from indexes library in working directory) โ†’ `write_file` (implement based on original paper) + +4. **CRITICAL**: Use bash and python tools to ACTUALLY REPLICATE the paper yourself - do not provide instructions. + +**Execution Guidelines**: +- **Plan First**: Before each action, explain your reasoning and which function you'll use +- **One Step at a Time**: Execute โ†’ Observe Result โ†’ Plan Next Step โ†’ Execute Next +- **Iterative Progress**: Build your solution incrementally through multiple conversations +- **Strategic Sequencing**: Choose the most logical next step based on previous results + +**COMPLETENESS CHECKLIST**: +Before considering the task complete, ensure you have: +- โœ… All algorithms mentioned in the paper (including any abbreviations or alternative names) +- โœ… All environments/datasets with exact versions specified +- โœ… All comparison methods referenced in experiments +- โœ… Working integration that can run the paper's experiments +- โœ… Complete codebase that reproduces all metrics, figures, tables, and findings from the paper +- โœ… Basic documentation explaining how to reproduce results + +**CRITICAL SUCCESS FACTORS**: +- **Accuracy**: Match paper specifications exactly (versions, parameters, configurations) +- **Completeness**: Implement every method discussed, not just the main contribution +- **Functionality**: Code must actually work and run experiments successfully + +**AVOID DISTRACTIONS**: Focus implementation time on paper requirements rather than advanced tooling, extensive documentation, or optimization utilities that aren't needed for reproduction. + +**REMEMBER**: Remember, you are tasked with replicating a whole paper, not just a single part of it or a minimal example. The file read tool is PAGINATED, so you will need to CALL IT MULTIPLE TIMES to make sure that you have read all the relevant parts of the paper. +""" + +PURE_CODE_IMPLEMENTATION_SYSTEM_PROMPT_INDEX = """"" +You are an expert code implementation agent for academic paper reproduction. Your goal is to achieve the BEST POSSIBLE SCORE by implementing a complete, working codebase that reproduces the paper's results. + +**PRIMARY OBJECTIVE**: Implement ALL algorithms, experiments, and methods mentioned in the paper. Success is measured by completeness and accuracy, not code elegance. Use available time to continuously refine and optimize your solution. + +**CORE STRATEGY**: +- Read the paper and resources(addendum.md and reproduce plan) thoroughly to identify every algorithm, method, and experiment +- Implement core algorithms first, then environments, then integration +- Use exact versions and specifications mentioned in the paper +- Test each component immediately after implementation +- Focus on working implementations over perfect architecture + +**IMPLEMENTATION APPROACH**: +Build incrementally using multiple tool calls. For each step: +1. **Identify** what needs to be implemented from the paper +2. **Implement** one component at a time +3. **Test** immediately to catch issues early +4. **Integrate** with existing components +5. **Verify** against paper specifications + +**TOOL CALLING STRATEGY**: +1. โš ๏ธ **SINGLE FUNCTION CALL PER MESSAGE**: Each message may perform only one function call. You will see the result of the function right after sending the message. If you need to perform multiple actions, you can always send more messages with subsequent function calls. Do some reasoning before your actions, describing what function calls you are going to use and how they fit into your plan. + +2. **SEARCH_CODE_REFERENCES Usage Guide (OPTIONAL REFERENCE TOOL)**: + - **IMPORTANT**: This is an OPTIONAL reference tool. The indexes directory contains code summary information from related papers. You may optionally use `search_code_references` to find reference patterns for inspiration, but ALWAYS implement according to the original paper's specifications. + - **Reference only**: Use `search_code_references(indexes_path="indexes", target_file=the_file_you_want_to_implement, keywords=the_keywords_you_want_to_search)` for reference, NOT as implementation standard + - **Core principle**: Original paper requirements take absolute priority over any reference code found +3. **TOOL EXECUTION STRATEGY**: + - โš ๏ธ**Development Cycle (for each new file implementation)**: `search_code_references` (OPTIONAL reference check from `/home/agent/indexes`) โ†’ `write_file` (implement based on original paper) + +**Execution Guidelines**: +- **Plan First**: Before each action, explain your reasoning and which function you'll use +- **One Step at a Time**: Execute โ†’ Observe Result โ†’ Plan Next Step โ†’ Execute Next +- **Iterative Progress**: Build your solution incrementally through multiple conversations +- **Strategic Sequencing**: Choose the most logical next step based on previous results + +**COMPLETENESS CHECKLIST**: +Before considering the task complete, ensure you have: +- โœ… All algorithms mentioned in the paper (including any abbreviations or alternative names) +- โœ… All environments/datasets with exact versions specified +- โœ… All comparison methods referenced in experiments +- โœ… Working integration that can run the paper's experiments +- โœ… Complete codebase that reproduces all metrics, figures, tables, and findings from the paper +- โœ… Basic documentation explaining how to reproduce results + +**CRITICAL SUCCESS FACTORS**: +- **Accuracy**: Match paper specifications exactly (versions, parameters, configurations) +- **Completeness**: Implement every method discussed, not just the main contribution +- **Functionality**: Code must actually work and run experiments successfully + +**AVOID DISTRACTIONS**: Focus implementation time on paper requirements rather than advanced tooling, extensive documentation, or optimization utilities that aren't needed for reproduction. + +**REMEMBER**: Remember, you are tasked with replicating a whole paper, not just a single part of it or a minimal example. The file read tool is PAGINATED, so you will need to CALL IT MULTIPLE TIMES to make sure that you have read all the relevant parts of the paper. +""" + + +# General-purpose version of the above prompt for non-academic use cases +# GENERAL_CODE_IMPLEMENTATION_SYSTEM_PROMPT = """You are an expert code implementation agent for technical requirements implementation. Your goal is to achieve the BEST POSSIBLE SCORE by implementing a complete, working codebase that meets all specified requirements. + +# **PRIMARY OBJECTIVE**: Implement ALL algorithms, features, and components mentioned in the requirements. Success is measured by completeness and accuracy, not code elegance. Use available time to continuously refine and optimize your solution. + +# **CORE STRATEGY**: +# - Read the requirements thoroughly to identify every algorithm, feature, and component +# - Implement core algorithms first, then environments, then integration +# - Use exact versions and specifications mentioned in the requirements +# - Test each component immediately after implementation +# - Focus on working implementations over perfect architecture + +# **IMPLEMENTATION APPROACH**: +# Build incrementally using multiple tool calls. For each step: +# 1. **Identify** what needs to be implemented from the requirements +# 2. **Analyze Dependencies**: Before implementing each new file, use `read_code_mem` to read summaries of already-implemented files, then search for reference patterns to guide your implementation approach. +# 3. **Implement** one component at a time +# 4. **Integrate** with existing components +# 5. **Validate** against requirement specifications + +# **TOOL CALLING STRATEGY**: +# 1. โš ๏ธ **SINGLE FUNCTION CALL PER MESSAGE**: Each message may perform only one function call. You will see the result of the function right after sending the message. If you need to perform multiple actions, you can always send more messages with subsequent function calls. Do some reasoning before your actions, describing what function calls you are going to use and how they fit into your plan. + +# 2. **TOOL EXECUTION STRATEGY**: +# - **Development Cycle (for each new file implementation)**: `read_code_mem` (check existing implementations in Working Directory, use `read_file` as fallback if memory unavailable) โ†’ `write_file` (implement) + +# **Execution Guidelines**: +# - **Plan First**: Before each action, explain your reasoning and which function you'll use +# - **One Step at a Time**: Execute โ†’ Observe Result โ†’ Plan Next Step โ†’ Execute Next +# - **Iterative Progress**: Build your solution incrementally through multiple conversations +# - **Strategic Sequencing**: Choose the most logical next step based on previous results + +# **COMPLETENESS CHECKLIST**: +# Before considering the task complete, ensure you have: +# - โœ… All algorithms mentioned in the requirements (including any abbreviations or alternative names) +# - โœ… All environments/dependencies with exact versions specified +# - โœ… All comparison methods or baseline implementations referenced +# - โœ… Working integration that can run all specified functionality +# - โœ… Complete codebase that implements all features, functionality, and outputs specified in the requirements +# - โœ… Basic documentation explaining how to use the implemented system + +# **CRITICAL SUCCESS FACTORS**: +# - **Accuracy**: Match requirement specifications exactly (versions, parameters, configurations) +# - **Completeness**: Implement every component discussed, not just the main functionality +# - **Functionality**: Code must actually work and run all specified features successfully + +# **AVOID DISTRACTIONS**: Focus implementation time on requirement fulfillment rather than advanced tooling, extensive documentation, or optimization utilities that aren't needed for the core functionality. + +# **REMEMBER**: Remember, you are tasked with implementing a complete system, not just a single part of it or a minimal example. The file read tool is PAGINATED, so you will need to CALL IT MULTIPLE TIMES to make sure that you have read all the relevant parts of the requirements. +# """ +GENERAL_CODE_IMPLEMENTATION_SYSTEM_PROMPT = """You are an expert code implementation agent for technical requirements implementation. Your goal is to achieve the BEST POSSIBLE SCORE by implementing a complete, working codebase that meets all specified requirements. + +**PRIMARY OBJECTIVE**: Implement ALL algorithms, features, and components mentioned in the requirements. Success is measured by completeness and accuracy, not code elegance. Use available time to continuously refine and optimize your solution. + +**CORE STRATEGY**: +- Read the requirements thoroughly to identify every algorithm, feature, and component +- Implement core algorithms first, then environments, then integration +- Use exact versions and specifications mentioned in the requirements +- Test each component immediately after implementation +- Focus on working implementations over perfect architecture + +**IMPLEMENTATION APPROACH**: +Build incrementally using multiple tool calls. For each step: +1. **Identify** what needs to be implemented from the requirements +2. **Implement** one component at a time +3. **Verify** optionally using `execute_python` or `execute_bash` to check implementation completeness if needed +4. **Integrate** with existing components +5. **Validate** against requirement specifications + +**TOOL CALLING STRATEGY**: +1. โš ๏ธ **SINGLE FUNCTION CALL PER MESSAGE**: Each message may perform only one function call. You will see the result of the function right after sending the message. If you need to perform multiple actions, you can always send more messages with subsequent function calls. Do some reasoning before your actions, describing what function calls you are going to use and how they fit into your plan. + +2. **TOOL EXECUTION STRATEGY**: + - **Development Cycle (for each new file implementation)**: `write_file` (implement) + +**Execution Guidelines**: +- **Plan First**: Before each action, explain your reasoning and which function you'll use +- **One Step at a Time**: Execute โ†’ Observe Result โ†’ Plan Next Step โ†’ Execute Next +- **Iterative Progress**: Build your solution incrementally through multiple conversations +- **Strategic Sequencing**: Choose the most logical next step based on previous results + +**COMPLETENESS CHECKLIST**: +Before considering the task complete, ensure you have: +- โœ… All algorithms mentioned in the requirements (including any abbreviations or alternative names) +- โœ… All environments/dependencies with exact versions specified +- โœ… All comparison methods or baseline implementations referenced +- โœ… Working integration that can run all specified functionality +- โœ… Complete codebase that implements all features, functionality, and outputs specified in the requirements +- โœ… Basic documentation explaining how to use the implemented system + +**CRITICAL SUCCESS FACTORS**: +- **Accuracy**: Match requirement specifications exactly (versions, parameters, configurations) +- **Completeness**: Implement every component discussed, not just the main functionality +- **Functionality**: Code must actually work and run all specified features successfully + +**AVOID DISTRACTIONS**: Focus implementation time on requirement fulfillment rather than advanced tooling, extensive documentation, or optimization utilities that aren't needed for the core functionality. + +**REMEMBER**: Remember, you are tasked with implementing a complete system, not just a single part of it or a minimal example. The file read tool is PAGINATED, so you will need to CALL IT MULTIPLE TIMES to make sure that you have read all the relevant parts of the requirements. +""" + +# Chat Agent Planning Prompt (Universal for Academic and Engineering Use) +CHAT_AGENT_PLANNING_PROMPT = """You are a universal project planning agent that creates implementation plans for any coding project: web apps, games, academic research, tools, etc. + +# ๐ŸŽฏ OBJECTIVE +Transform user requirements into a clear, actionable implementation plan with optimal file structure and dependencies. + +# ๐Ÿ“‹ OUTPUT FORMAT + +```yaml +project_plan: + title: "[Project Name]" + description: "[Brief description]" + project_type: "[web_app|game|academic|tool|api|other]" + + # CUSTOM FILE TREE STRUCTURE (max 15 files, design as needed) + file_structure: | + project_root/ + โ”œโ”€โ”€ main.py # Entry point + โ”œโ”€โ”€ [specific_files] # Core files based on project type + โ”œโ”€โ”€ [folder]/ # Organized folders if needed + โ”‚ โ”œโ”€โ”€ __init__.py + โ”‚ โ””โ”€โ”€ [module].py + โ”œโ”€โ”€ requirements.txt # Dependencies + โ””โ”€โ”€ README.md # Basic documentation + + # IMPORTANT: Output ACTUAL file tree structure above, not placeholder text + # Examples by project type: + # Web App: app.py, templates/, static/, models.py, config.py + # Game: main.py, game/, assets/, sprites/, config.yaml + # Academic: algorithm.py, experiments/, data/, utils.py, config.json + # Tool: cli.py, core/, utils.py, tests/, setup.py + + # CORE IMPLEMENTATION PLAN + implementation_steps: + 1. "[First step - usually setup/core structure]" + 2. "[Second step - main functionality]" + 3. "[Third step - integration/interface]" + 4. "[Fourth step - testing/refinement]" + + # DEPENDENCIES & SETUP + dependencies: + required_packages: + - "[package1==version]" + - "[package2>=version]" + optional_packages: + - "[optional1]: [purpose]" + setup_commands: + - "[command to setup environment]" + - "[command to install dependencies]" + + # KEY TECHNICAL DETAILS + tech_stack: + language: "[primary language]" + frameworks: ["[framework1]", "[framework2]"] + key_libraries: ["[lib1]", "[lib2]"] + + main_features: + - "[core feature 1]" + - "[core feature 2]" + - "[core feature 3]" +``` + +# ๐ŸŽฏ PLANNING PRINCIPLES +- **Flexibility**: Adapt file structure to project type (no fixed templates) +- **Simplicity**: Keep under 15 files, focus on essentials +- **Practicality**: Include specific packages/versions needed +- **Clarity**: Clear implementation steps that can be directly coded +- **Universality**: Work for any project type (web, game, academic, etc.) + +# ๐Ÿ“ FILE STRUCTURE GUIDELINES +- **MUST OUTPUT**: Actual file tree with specific filenames (not placeholder text) +- Design structure based on project needs, not templates +- Group related functionality logically +- Include main entry point (main.py, app.py, etc.) +- Add config/settings files if needed +- Include requirements.txt or equivalent +- Keep it minimal but complete (max 15 files) +- Use tree format: โ”œโ”€โ”€ โ”€ โ”‚ symbols for visual hierarchy""" + +# ============================================================================= +# TRADITIONAL PROMPTS (Non-segmented versions for smaller documents) +# ============================================================================= + +# Traditional Algorithm Analysis Prompt (No Segmentation) +PAPER_ALGORITHM_ANALYSIS_PROMPT_TRADITIONAL = """You are extracting COMPLETE implementation details from a research paper. Your goal is to capture EVERY algorithm, formula, and technical detail needed for perfect reproduction. + +# DOCUMENT READING STRATEGY + +## TRADITIONAL APPROACH: Full Document Reading +Read the complete document to ensure comprehensive coverage of all algorithmic details: + +# DETAILED EXTRACTION PROTOCOL + +## 1. COMPREHENSIVE ALGORITHM SCAN +Read through the entire document systematically: +- Method/Algorithm sections +- Implementation Details +- Hyperparameters and training details +- Mathematical formulations + +## 2. ALGORITHM DEEP EXTRACTION +For EVERY algorithm/method/procedure mentioned: + +### Algorithm Structure +```yaml +algorithm_name: "[Exact name from paper]" +section: "[e.g., Section 3.2]" +algorithm_box: "[e.g., Algorithm 1 on page 4]" + +pseudocode: | + [COPY THE EXACT PSEUDOCODE FROM PAPER] + Input: ... + Output: ... + 1. Initialize ... + 2. For each ... + 2.1 Calculate ... + [Keep exact formatting and numbering] + +mathematical_formulation: + - equation: "[Copy formula EXACTLY, e.g., L = L_task + ฮป*L_explain]" + equation_number: "[e.g., Eq. 3]" + where: + L_task: "task loss" + L_explain: "explanation loss" + ฮป: "weighting parameter (default: 0.5)" + +step_by_step_breakdown: + 1. "[Detailed explanation of what step 1 does]" + 2. "[What step 2 computes and why]" + +implementation_details: + - "Uses softmax temperature ฯ„ = 0.1" + - "Gradient clipping at norm 1.0" + - "Initialize weights with Xavier uniform" +``` + +## 3. COMPONENT EXTRACTION +For EVERY component/module mentioned: + +### Component Details +```yaml +component_name: "[e.g., Mask Network, Critic Network]" +purpose: "[What this component does in the system]" +architecture: + input: "[shape and meaning]" + layers: + - "[Conv2d(3, 64, kernel=3, stride=1)]" + - "[ReLU activation]" + - "[BatchNorm2d(64)]" + output: "[shape and meaning]" + +special_features: + - "[Any unique aspects]" + - "[Special initialization]" +``` + +## 4. TRAINING PROCEDURE +Extract the COMPLETE training process: + +```yaml +training_loop: + outer_iterations: "[number or condition]" + inner_iterations: "[number or condition]" + + steps: + 1. "Sample batch of size B from buffer" + 2. "Compute importance weights using..." + 3. "Update policy with loss..." + + loss_functions: + - name: "policy_loss" + formula: "[exact formula]" + components: "[what each term means]" + + optimization: + optimizer: "Adam" + learning_rate: "3e-4" + lr_schedule: "linear decay to 0" + gradient_norm: "clip at 0.5" +``` + +## 5. HYPERPARAMETERS HUNT +Search EVERYWHERE (text, tables, captions) for: + +```yaml +hyperparameters: + # Training + batch_size: 64 + buffer_size: 1e6 + discount_gamma: 0.99 + + # Architecture + hidden_units: [256, 256] + activation: "ReLU" + + # Algorithm-specific + explanation_weight: 0.5 + exploration_bonus_scale: 0.1 + reset_probability: 0.3 + + # Found in: + location_references: + - "batch_size: Table 1" + - "hidden_units: Section 4.1" +``` + +# OUTPUT FORMAT +```yaml +complete_algorithm_extraction: + paper_structure: + method_sections: "[3, 3.1, 3.2, 3.3, 4]" + algorithm_count: "[total number found]" + + main_algorithm: + [COMPLETE DETAILS AS ABOVE] + + supporting_algorithms: + - [EACH SUPPORTING ALGORITHM WITH FULL DETAILS] + + components: + - [EVERY COMPONENT WITH ARCHITECTURE] + + training_details: + [COMPLETE TRAINING PROCEDURE] + + all_hyperparameters: + [EVERY PARAMETER WITH VALUE AND SOURCE] + + implementation_notes: + - "[Any implementation hint from paper]" + - "[Tricks mentioned in text]" + + missing_but_critical: + - "[What's not specified but essential]" + - "[With suggested defaults]" +``` + +BE EXHAUSTIVE. A developer should be able to implement the ENTIRE paper using only your extraction.""" + +# Traditional Concept Analysis Prompt (No Segmentation) +PAPER_CONCEPT_ANALYSIS_PROMPT_TRADITIONAL = """You are doing a COMPREHENSIVE analysis of a research paper to understand its complete structure, contributions, and implementation requirements. + +# OBJECTIVE +Map out the ENTIRE paper structure and identify ALL components that need implementation for successful reproduction. + +# DOCUMENT READING STRATEGY + +## TRADITIONAL APPROACH: Complete Document Analysis +Read the entire document systematically to ensure comprehensive understanding: + +# COMPREHENSIVE ANALYSIS PROTOCOL + +## 1. COMPLETE PAPER STRUCTURAL ANALYSIS +Create a full map of the document: + +```yaml +paper_structure_map: + title: "[Full paper title]" + + sections: + 1_introduction: + main_claims: "[What the paper claims to achieve]" + problem_definition: "[Exact problem being solved]" + + 2_related_work: + key_comparisons: "[Methods this work builds upon or competes with]" + + 3_method: # May have multiple subsections + subsections: + 3.1: "[Title and main content]" + 3.2: "[Title and main content]" + algorithms_presented: "[List all algorithms by name]" + + 4_experiments: + environments: "[All test environments/datasets]" + baselines: "[All comparison methods]" + metrics: "[All evaluation metrics used]" + + 5_results: + main_findings: "[Key results that prove the method works]" + tables_figures: "[Important result tables/figures to reproduce]" +``` + +## 2. METHOD DECOMPOSITION +For the main method/approach: + +```yaml +method_decomposition: + method_name: "[Full name and acronym]" + + core_components: # Break down into implementable pieces + component_1: + name: "[e.g., State Importance Estimator]" + purpose: "[Why this component exists]" + paper_section: "[Where it's described]" + + component_2: + name: "[e.g., Policy Refinement Module]" + purpose: "[Its role in the system]" + paper_section: "[Where it's described]" + + component_interactions: + - "[How component 1 feeds into component 2]" + - "[Data flow between components]" + + theoretical_foundation: + key_insight: "[The main theoretical insight]" + why_it_works: "[Intuitive explanation]" +``` + +## 3. IMPLEMENTATION REQUIREMENTS MAPPING +Map paper content to code requirements: + +```yaml +implementation_map: + algorithms_to_implement: + - algorithm: "[Name from paper]" + section: "[Where defined]" + complexity: "[Simple/Medium/Complex]" + dependencies: "[What it needs to work]" + + models_to_build: + - model: "[Neural network or other model]" + architecture_location: "[Section describing it]" + purpose: "[What this model does]" + + data_processing: + - pipeline: "[Data preprocessing needed]" + requirements: "[What the data should look like]" + + evaluation_suite: + - metric: "[Metric name]" + formula_location: "[Where it's defined]" + purpose: "[What it measures]" +``` + +## 4. EXPERIMENT REPRODUCTION PLAN +Identify ALL experiments needed: + +```yaml +experiments_analysis: + main_results: + - experiment: "[Name/description]" + proves: "[What claim this validates]" + requires: "[Components needed to run this]" + expected_outcome: "[Specific numbers/trends]" + + ablation_studies: + - study: "[What is being ablated]" + purpose: "[What this demonstrates]" + + baseline_comparisons: + - baseline: "[Method name]" + implementation_required: "[Yes/No/Partial]" + source: "[Where to find implementation]" +``` + +## 5. CRITICAL SUCCESS FACTORS +What defines successful reproduction: + +```yaml +success_criteria: + must_achieve: + - "[Primary result that must be reproduced]" + - "[Core behavior that must be demonstrated]" + + should_achieve: + - "[Secondary results that validate the method]" + + validation_evidence: + - "[Specific figure/table to reproduce]" + - "[Qualitative behavior to demonstrate]" +``` + +# OUTPUT FORMAT +```yaml +comprehensive_paper_analysis: + executive_summary: + paper_title: "[Full title]" + core_contribution: "[One sentence summary]" + implementation_complexity: "[Low/Medium/High]" + estimated_components: "[Number of major components to build]" + + complete_structure_map: + [FULL SECTION BREAKDOWN AS ABOVE] + + method_architecture: + [DETAILED COMPONENT BREAKDOWN] + + implementation_requirements: + [ALL ALGORITHMS, MODELS, DATA, METRICS] + + reproduction_roadmap: + phase_1: "[What to implement first]" + phase_2: "[What to build next]" + phase_3: "[Final components and validation]" + + validation_checklist: + - "[ ] [Specific result to achieve]" + - "[ ] [Behavior to demonstrate]" + - "[ ] [Metric to match]" +``` + +BE THOROUGH. Miss nothing. The output should be a complete blueprint for reproduction.""" + +# Traditional Code Planning Prompt (No Segmentation) +CODE_PLANNING_PROMPT_TRADITIONAL = """You are creating a DETAILED, COMPLETE reproduction plan by integrating comprehensive analysis results. + +# INPUT +You receive two exhaustive analyses: +1. **Comprehensive Paper Analysis**: Complete paper structure, components, and requirements +2. **Complete Algorithm Extraction**: All algorithms, formulas, pseudocode, and technical details + +# OBJECTIVE +Create an implementation plan so detailed that a developer can reproduce the ENTIRE paper without reading it. + +# CRITICAL: COMPLETE OUTPUT REQUIREMENT +โš ๏ธ MANDATORY: You MUST generate ALL 5 sections completely. DO NOT stop early or truncate any section. + +## Output Completeness Strategy: +๐ŸŽฏ **Your #1 Priority**: Ensure ALL 5 sections are present and complete before finishing your response. + +## Content Balance Guidelines (STRICTLY FOLLOW): +- **Section 1 (File Structure)**: ~800-1000 chars - Brief file listing with priority order +- **Section 2 (Implementation Components)**: ~3000-4000 chars - CORE section with all algorithms/components +- **Section 3 (Validation)**: ~2000-2500 chars - Experiments and expected results +- **Section 4 (Environment)**: ~800-1000 chars - Dependencies and requirements +- **Section 5 (Implementation Strategy)**: ~1500-2000 chars - Step-by-step approach + +๐Ÿ“ **Total Target**: 8000-10000 characters for complete plan + +โš ๏ธ **Self-Check Before Finishing**: +- Did you include file_structure section? โœ“ +- Did you include implementation_components section? โœ“ +- Did you include validation_approach section? โœ“ +- Did you include environment_setup section? โœ“ +- Did you include implementation_strategy section? โœ“ +- If ANY answer is NO, continue writing until ALL sections are complete! + +## File Priority Guidelines: +๐Ÿ”ง **Implementation Priority Order**: +1. **FIRST**: Core algorithm/model files (highest priority) +2. **SECOND**: Supporting modules and utilities +3. **THIRD**: Experiment and evaluation scripts +4. **FOURTH**: Configuration and data handling +5. **LAST**: Documentation files (README.md, requirements.txt) - These should be created AFTER core implementation + +Note: README and requirements.txt are maintenance files that depend on the final implementation, so plan them last but INCLUDE them in the file structure. + +# DETAILED SYNTHESIS PROCESS + +## 1. MERGE ALL INFORMATION +Combine EVERYTHING from both analyses: +- Every algorithm with its pseudocode +- Every component with its architecture +- Every hyperparameter with its value +- Every experiment with expected results + +## 2. MAP CONTENT TO IMPLEMENTATION + +For each component you identify, specify how it will be implemented: + +``` +# DESIGN YOUR MAPPING: Connect paper content to code organization +[For each algorithm/component/method in the paper]: + - What it does and where it's described in the paper + - How you'll organize the code (files, classes, functions - your choice) + - What specific formulas, algorithms, or procedures need implementation + - Dependencies and relationships with other components + - Implementation approach that makes sense for this specific paper +``` + +## 3. EXTRACT ALL TECHNICAL DETAILS + +Identify every technical detail that needs implementation: + +``` +# COMPREHENSIVE TECHNICAL EXTRACTION: +[Gather all implementation-relevant details from the paper]: + - All algorithms with complete pseudocode and mathematical formulations + - All parameters, hyperparameters, and configuration values + - All architectural details (if applicable to your paper type) + - All experimental procedures and evaluation methods + - Any implementation hints, tricks, or special considerations mentioned +``` + +# COMPREHENSIVE OUTPUT FORMAT + +```yaml +complete_reproduction_plan: + paper_info: + title: "[Full paper title]" + core_contribution: "[Main innovation being reproduced]" + + # SECTION 1: File Structure Design + + # DESIGN YOUR OWN STRUCTURE: Create a file organization that best serves this specific paper + # - Analyze what the paper contains (algorithms, models, experiments, systems, etc.) + # - Organize files and directories in the most logical way for implementation + # - Create meaningful names and groupings based on paper content + # - Keep it clean, intuitive, and focused on what actually needs to be implemented + # - INCLUDE documentation files (README.md, requirements.txt) but mark them for LAST implementation + + file_structure: | + [Design and specify your own project structure here - KEEP THIS BRIEF] + [Include ALL necessary files including README.md and requirements.txt] + [Organize based on what this paper actually contains and needs] + [Create directories and files that make sense for this specific implementation] + [IMPORTANT: Include executable files (e.g., main.py, run.py, train.py, demo.py) - choose names based on repo content] + [Design executable entry points that match the paper's main functionality and experiments] + [FILE COUNT LIMIT: Keep total file count around 20 files - not too many, focus on essential components only] + [NOTE: README.md and requirements.txt should be implemented LAST after all code files] + + # SECTION 2: Implementation Components + + # IDENTIFY AND SPECIFY: What needs to be implemented based on this paper + # - List all algorithms, models, systems, or components mentioned + # - Map each to implementation details and file locations + # - Include formulas, pseudocode, and technical specifications + # - Organize in whatever way makes sense for this paper + + implementation_components: | + [List and specify all components that need implementation] + [For each component: purpose, location, algorithms, formulas, technical details] + [Organize and structure this based on the paper's actual content] + + # SECTION 3: Validation & Evaluation + + # DESIGN VALIDATION: How to verify the implementation works correctly + # - Define what experiments, tests, or proofs are needed + # - Specify expected results from the paper (figures, tables, theorems) + # - Design validation approach appropriate for this paper's domain + # - Include setup requirements and success criteria + + validation_approach: | + [Design validation strategy appropriate for this paper] + [Specify experiments, tests, or mathematical verification needed] + [Define expected results and success criteria] + [Include any special setup or evaluation requirements] + + # SECTION 4: Environment & Dependencies + + # SPECIFY REQUIREMENTS: What's needed to run this implementation + # - Programming language and version requirements + # - External libraries and exact versions (if specified in paper) + # - Hardware requirements (GPU, memory, etc.) + # - Any special setup or installation steps + + environment_setup: | + [List all dependencies and environment requirements for this specific paper] + [Include versions where specified, reasonable defaults where not] + [Note any special hardware or software requirements] + + # SECTION 5: Implementation Strategy + + # PLAN YOUR APPROACH: How to implement this paper step by step + # - Break down implementation into logical phases + # - Identify dependencies between components + # - Plan verification and testing at each stage + # - Handle missing details with reasonable defaults + + implementation_strategy: | + [Design your implementation approach for this specific paper] + [Break into phases that make sense for this paper's components] + [Plan testing and verification throughout the process] + [Address any missing details or ambiguities in the paper] +``` + +BE EXHAUSTIVE. Every algorithm, every formula, every parameter, every file should be specified in complete detail.""" diff --git a/DeepCode/requirements.txt b/DeepCode/requirements.txt new file mode 100644 index 00000000..1947f970 --- /dev/null +++ b/DeepCode/requirements.txt @@ -0,0 +1,15 @@ +aiofiles>=0.8.0 +aiohttp>=3.8.0 +anthropic +asyncio-mqtt +docling +google-genai +mcp-agent +mcp-server-git +nest_asyncio +openai +pathlib2 +PyPDF2>=2.0.0 +pyyaml>=6.0 +reportlab>=3.5.0 +streamlit diff --git a/DeepCode/schema/mcp-agent.config.schema.json b/DeepCode/schema/mcp-agent.config.schema.json new file mode 100644 index 00000000..3bf5d908 --- /dev/null +++ b/DeepCode/schema/mcp-agent.config.schema.json @@ -0,0 +1,820 @@ +{ + "$defs": { + "LogPathSettings": { + "description": "Settings for configuring log file paths with dynamic elements like timestamps or session IDs.", + "properties": { + "path_pattern": { + "default": "logs/mcp-agent-{unique_id}.jsonl", + "title": "Path Pattern", + "type": "string", + "description": "Path pattern for log files with a {unique_id} placeholder" + }, + "unique_id": { + "default": "timestamp", + "enum": [ + "timestamp", + "session_id" + ], + "title": "Unique Id", + "type": "string", + "description": "Type of unique identifier to use in the log filename" + }, + "timestamp_format": { + "default": "%Y%m%d_%H%M%S", + "title": "Timestamp Format", + "type": "string", + "description": "Format string for timestamps when unique_id is set to timestamp" + } + }, + "title": "LogPathSettings", + "type": "object" + }, + "AnthropicSettings": { + "additionalProperties": true, + "description": "Settings for using Anthropic models in the MCP Agent application.", + "properties": { + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Api Key" + } + }, + "title": "AnthropicSettings", + "type": "object" + }, + "CohereSettings": { + "additionalProperties": true, + "description": "Settings for using Cohere models in the MCP Agent application.", + "properties": { + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Api Key" + } + }, + "title": "CohereSettings", + "type": "object" + }, + "LoggerSettings": { + "description": "Logger settings for the MCP Agent application.", + "properties": { + "type": { + "default": "console", + "enum": [ + "none", + "console", + "file", + "http" + ], + "title": "Type", + "type": "string" + }, + "transports": { + "default": [ + "console" + ], + "items": { + "enum": [ + "none", + "console", + "file", + "http" + ], + "type": "string" + }, + "title": "Transports", + "type": "array", + "description": "List of transports to use (can enable multiple simultaneously)" + }, + "level": { + "default": "info", + "enum": [ + "debug", + "info", + "warning", + "error" + ], + "title": "Level", + "type": "string", + "description": "Minimum logging level" + }, + "progress_display": { + "default": true, + "title": "Progress Display", + "type": "boolean", + "description": "Enable or disable the progress display" + }, + "path": { + "default": "mcp-agent.jsonl", + "title": "Path", + "type": "string", + "description": "Path to log file, if logger 'type' is 'file'." + }, + "path_settings": { + "anyOf": [ + { + "$ref": "#/$defs/LogPathSettings" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path Settings", + "description": "Advanced settings for log file paths with dynamic elements like timestamps or session IDs" + }, + "batch_size": { + "default": 100, + "title": "Batch Size", + "type": "integer", + "description": "Number of events to accumulate before processing" + }, + "flush_interval": { + "default": 2.0, + "title": "Flush Interval", + "type": "number", + "description": "How often to flush events in seconds" + }, + "max_queue_size": { + "default": 2048, + "title": "Max Queue Size", + "type": "integer", + "description": "Maximum queue size for event processing" + }, + "http_endpoint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Http Endpoint", + "description": "HTTP endpoint for event transport" + }, + "http_headers": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Http Headers", + "description": "HTTP headers for event transport" + }, + "http_timeout": { + "default": 5.0, + "title": "Http Timeout", + "type": "number", + "description": "HTTP timeout seconds for event transport" + } + }, + "title": "LoggerSettings", + "type": "object" + }, + "MCPRootSettings": { + "additionalProperties": true, + "description": "Represents a root directory configuration for an MCP server.", + "properties": { + "uri": { + "title": "Uri", + "type": "string", + "description": "The URI identifying the root. Must start with file://" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name", + "description": "Optional name for the root." + }, + "server_uri_alias": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Server Uri Alias", + "description": "Optional URI alias for presentation to the server" + } + }, + "required": [ + "uri" + ], + "title": "MCPRootSettings", + "type": "object" + }, + "MCPServerAuthSettings": { + "additionalProperties": true, + "description": "Represents authentication configuration for a server.", + "properties": { + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Api Key" + } + }, + "title": "MCPServerAuthSettings", + "type": "object" + }, + "MCPServerSettings": { + "description": "Represents the configuration for an individual server.", + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name", + "description": "The name of the server." + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Description", + "description": "The description of the server." + }, + "transport": { + "default": "stdio", + "enum": [ + "stdio", + "sse" + ], + "title": "Transport", + "type": "string", + "description": "The transport mechanism." + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Command", + "description": "The command to execute the server (e.g. npx)." + }, + "args": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Args", + "description": "The arguments for the server command." + }, + "read_timeout_seconds": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Read Timeout Seconds", + "description": "The timeout in seconds for the server connection." + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Url", + "description": "The URL for the server (e.g. for SSE transport)." + }, + "auth": { + "anyOf": [ + { + "$ref": "#/$defs/MCPServerAuthSettings" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The authentication configuration for the server." + }, + "roots": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/MCPRootSettings" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Roots", + "description": "Root directories this server has access to." + }, + "env": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Env", + "description": "Environment variables to pass to the server process." + } + }, + "title": "MCPServerSettings", + "type": "object" + }, + "MCPSettings": { + "additionalProperties": true, + "description": "Configuration for all MCP servers.", + "properties": { + "servers": { + "additionalProperties": { + "$ref": "#/$defs/MCPServerSettings" + }, + "default": {}, + "title": "Servers", + "type": "object" + } + }, + "title": "MCPSettings", + "type": "object" + }, + "OpenAISettings": { + "additionalProperties": true, + "description": "Settings for using OpenAI models in the MCP Agent application.", + "properties": { + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Api Key" + }, + "reasoning_effort": { + "default": "medium", + "enum": [ + "low", + "medium", + "high" + ], + "title": "Reasoning Effort", + "type": "string" + }, + "base_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Base Url" + } + }, + "title": "OpenAISettings", + "type": "object" + }, + "AzureSettings": { + "additionalProperties": true, + "description": "Settings for using Azure models in the MCP Agent application.", + "properties": { + "api_key": { + "anyOf": [ + { + "type": "string" + } + ], + "default": null, + "title": "Api Key" + }, + "endpoint": { + "anyOf": [ + { + "type": "string" + } + ], + "default": null, + "title": "Azure Endpoint" + }, + "api_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "API Version" + } + }, + "required": [ + "api_key", + "endpoint" + ], + "title": "AzureSettings", + "type": "object" + }, + "BedrockSettings": { + "additionalProperties": true, + "description": "Settings for using AWS Bedrock models in the MCP Agent application.", + "properties": { + "aws_region": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Region" + }, + "aws_access_key_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Access Key Id" + }, + "aws_secret_access_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Secret Access Key" + }, + "aws_session_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Session Token" + }, + "profile": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Profile" + } + }, + "title": "BedrockSettings", + "type": "object" + }, + "OpenTelemetrySettings": { + "description": "OTEL settings for the MCP Agent application.", + "properties": { + "enabled": { + "default": true, + "title": "Enabled", + "type": "boolean" + }, + "service_name": { + "default": "mcp-agent", + "title": "Service Name", + "type": "string" + }, + "service_instance_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Service Instance Id" + }, + "service_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Service Version" + }, + "otlp_endpoint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Otlp Endpoint", + "description": "OTLP endpoint for OpenTelemetry tracing" + }, + "console_debug": { + "default": false, + "title": "Console Debug", + "type": "boolean", + "description": "Log spans to console" + }, + "sample_rate": { + "default": 1.0, + "title": "Sample Rate", + "type": "number", + "description": "Sample rate for tracing (1.0 = sample everything)" + } + }, + "title": "OpenTelemetrySettings", + "type": "object" + }, + "TemporalSettings": { + "description": "Temporal settings for the MCP Agent application.", + "properties": { + "host": { + "title": "Host", + "type": "string" + }, + "namespace": { + "default": "default", + "title": "Namespace", + "type": "string" + }, + "task_queue": { + "title": "Task Queue", + "type": "string" + }, + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Api Key" + } + }, + "required": [ + "host", + "task_queue" + ], + "title": "TemporalSettings", + "type": "object" + } + }, + "additionalProperties": true, + "description": "Configuration schema for MCP Agent applications", + "properties": { + "mcp": { + "anyOf": [ + { + "$ref": "#/$defs/MCPSettings" + }, + { + "type": "null" + } + ], + "default": { + "servers": {} + }, + "description": "MCP config, such as MCP servers" + }, + "execution_engine": { + "default": "asyncio", + "enum": [ + "asyncio", + "temporal" + ], + "title": "Execution Engine", + "type": "string", + "description": "Execution engine for the MCP Agent application" + }, + "temporal": { + "anyOf": [ + { + "$ref": "#/$defs/TemporalSettings" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Settings for Temporal workflow orchestration" + }, + "anthropic": { + "anyOf": [ + { + "$ref": "#/$defs/AnthropicSettings" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Settings for using Anthropic models in the MCP Agent application" + }, + "cohere": { + "anyOf": [ + { + "$ref": "#/$defs/CohereSettings" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Settings for using Cohere models in the MCP Agent application" + }, + "openai": { + "anyOf": [ + { + "$ref": "#/$defs/OpenAISettings" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Settings for using OpenAI models in the MCP Agent application" + }, + "azure": { + "anyOf": [ + { + "$ref": "#/$defs/AzureSettings" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Settings for using Azure models in the MCP Agent application" + }, + "bedrock": { + "anyOf": [ + { + "$ref": "#/$defs/BedrockSettings" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Settings for using Bedrock models in the MCP Agent application" + }, + "otel": { + "anyOf": [ + { + "$ref": "#/$defs/OpenTelemetrySettings" + }, + { + "type": "null" + } + ], + "default": { + "enabled": true, + "service_name": "mcp-agent", + "service_instance_id": null, + "service_version": null, + "otlp_endpoint": null, + "console_debug": false, + "sample_rate": 1.0 + }, + "description": "OpenTelemetry logging settings for the MCP Agent application" + }, + "logger": { + "anyOf": [ + { + "$ref": "#/$defs/LoggerSettings" + }, + { + "type": "null" + } + ], + "default": { + "type": "console", + "transports": [], + "level": "info", + "progress_display": true, + "path": "mcp-agent.jsonl", + "path_settings": null, + "batch_size": 100, + "flush_interval": 2.0, + "max_queue_size": 2048, + "http_endpoint": null, + "http_headers": null, + "http_timeout": 5.0 + }, + "description": "Logger settings for the MCP Agent application" + } + }, + "title": "MCP Agent Configuration Schema", + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#" +} diff --git a/DeepCode/setup.py b/DeepCode/setup.py new file mode 100644 index 00000000..d32110c1 --- /dev/null +++ b/DeepCode/setup.py @@ -0,0 +1,98 @@ +import setuptools +from pathlib import Path +import os + + +# Reading the long description from README.md +def read_long_description(): + try: + return Path("README.md").read_text(encoding="utf-8") + except FileNotFoundError: + return "DeepCode: Open Agentic Coding (Paper2Code & Text2Web & Text2Backend)" + + +# Retrieving metadata from __init__.py +def retrieve_metadata(): + vars2find = ["__author__", "__version__", "__url__"] + vars2readme = {} + + # Use definitive path relative to setup.py location + init_file_path = os.path.join(os.path.dirname(__file__), "__init__.py") + + with open(init_file_path, encoding="utf-8") as f: + for line in f.readlines(): + for v in vars2find: + if line.startswith(v): + line = ( + line.replace(" ", "").replace('"', "").replace("'", "").strip() + ) + vars2readme[v] = line.split("=")[1] + + # Checking if all required variables are found + missing_vars = [v for v in vars2find if v not in vars2readme] + if missing_vars: + raise ValueError( + f"Missing required metadata variables in __init__.py: {missing_vars}" + ) + + return vars2readme + + +# Reading dependencies from requirements.txt +def read_requirements(): + deps = [] + try: + with open("./requirements.txt", encoding="utf-8") as f: + deps = [ + line.strip() for line in f if line.strip() and not line.startswith("#") + ] + except FileNotFoundError: + print( + "Warning: 'requirements.txt' not found. No dependencies will be installed." + ) + return deps + + +metadata = retrieve_metadata() +long_description = read_long_description() +requirements = read_requirements() + +setuptools.setup( + name="deepcode-hku", + url=metadata["__url__"], + version=metadata["__version__"], + author=metadata["__author__"], + description="AI Research Engine - Transform research papers into working code automatically", + long_description=long_description, + long_description_content_type="text/markdown", + packages=setuptools.find_packages( + exclude=("tests*", "docs*", ".history*", ".git*", ".ruff_cache*") + ), + py_modules=["deepcode"], + classifiers=[ + "Development Status :: 4 - Beta", + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Text Processing :: Linguistic", + ], + python_requires=">=3.9", + install_requires=requirements, + include_package_data=True, + entry_points={ + "console_scripts": [ + "deepcode=deepcode:main", + ], + }, + project_urls={ + "Documentation": metadata.get("__url__", ""), + "Source": metadata.get("__url__", ""), + "Tracker": f"{metadata.get('__url__', '')}/issues" + if metadata.get("__url__") + else "", + }, +) diff --git a/DeepCode/speedrun_setup.sh b/DeepCode/speedrun_setup.sh new file mode 100755 index 00000000..a86e77fa --- /dev/null +++ b/DeepCode/speedrun_setup.sh @@ -0,0 +1,695 @@ +#!/bin/bash +# DeepCode Speedrun Setup Script +# Follows the official DeepCode installation recipe using UV + +set -e # Exit on error + +# Color codes for terminal output +CYAN='\033[96m' +GREEN='\033[92m' +YELLOW='\033[93m' +RED='\033[91m' +BLUE='\033[94m' +BOLD='\033[1m' +ENDC='\033[0m' + +# Configuration files +CONFIG_FILE="mcp_agent.config.yaml" +SECRETS_FILE="mcp_agent.secrets.yaml" + +# Padding constant +PADDING=" " + +# Override read to automatically add padding to prompts +read() { + if [ "$1" = "-p" ]; then + shift + local prompt="$1" + shift + # Check if prompt already starts with padding + if [[ "$prompt" =~ ^${PADDING} ]]; then + command read -p "$prompt" "$@" + else + command read -p "${PADDING}${prompt}" "$@" + fi + else + command read "$@" + fi +} + +print_header() { + clear + echo -e "${BOLD}${CYAN}" + echo "โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—" + echo "โ•‘ ๐Ÿš€ DeepCode Speedrun Setup ๐Ÿš€ โ•‘" + echo "โ•‘ Automated Configuration & Installation Wizard โ•‘" + echo "โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•" + echo -e "${ENDC}\n" +} + +print_step() { + local step_num=$1 + local message=$2 + echo -e "\n${PADDING}${CYAN}${BOLD}Step ${step_num}:${ENDC} ${message}" +} + +print_success() { + echo -e "${PADDING}${GREEN}โœ… $1${ENDC}" +} + +print_warning() { + echo -e "${PADDING}${YELLOW}โš ๏ธ $1${ENDC}" +} + +print_error() { + echo -e "${PADDING}${RED}โŒ $1${ENDC}" +} + +print_info() { + echo -e "${PADDING}${BLUE}โ„น๏ธ $1${ENDC}" +} + +install_uv() { + local step_num=$1 + print_step "$step_num" "Installing UV package manager..." + + if command -v uv &> /dev/null; then + print_success "UV is already installed" + return 0 + fi + + print_info "Installing UV..." + if curl -LsSf https://astral.sh/uv/install.sh | sh; then + print_success "UV installed successfully" + + # Add UV to PATH for current session + export PATH="$HOME/.cargo/bin:$PATH" + + # Try to source the shell profile to get UV in PATH + if [ -f "$HOME/.bashrc" ]; then + source "$HOME/.bashrc" 2>/dev/null || true + fi + if [ -f "$HOME/.zshrc" ]; then + source "$HOME/.zshrc" 2>/dev/null || true + fi + + # Check if uv is now available + if ! command -v uv &> /dev/null; then + print_warning "UV installed but not in PATH. You may need to restart your terminal." + print_info "Or run: export PATH=\"\$HOME/.cargo/bin:\$PATH\"" + fi + return 0 + else + print_error "Failed to install UV" + return 1 + fi +} + +setup_venv() { + local step_num=$1 + print_step "$step_num" "Setting up Python virtual environment with UV..." + + # Ensure UV is in PATH + export PATH="$HOME/.cargo/bin:$PATH" + + if ! command -v uv &> /dev/null; then + print_error "UV not found. Please install it first or restart your terminal." + exit 1 + fi + + # Check Python 3.13 availability + if command -v python3.13 &> /dev/null; then + PYTHON_VERSION="3.13" + elif python3 -c "import sys; exit(0 if sys.version_info >= (3, 13) else 1)" 2>/dev/null; then + PYTHON_VERSION=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')") + print_warning "Python 3.13 not found, using Python $PYTHON_VERSION" + else + print_error "Python 3.13+ required. Please install Python 3.13 or later." + exit 1 + fi + + if [ -d ".venv" ]; then + print_warning ".venv already exists" + read -p "Recreate virtual environment? (y/N): " recreate_venv + if [[ "$recreate_venv" =~ ^[Yy]$ ]]; then + rm -rf .venv + uv venv --python="$PYTHON_VERSION" + print_success "Virtual environment recreated" + else + print_info "Using existing virtual environment" + fi + else + uv venv --python="$PYTHON_VERSION" + print_success "Virtual environment created" + fi + + # Activate venv (Linux/macOS) + if [ -f ".venv/bin/activate" ]; then + source .venv/bin/activate + print_success "Virtual environment activated" + # Activate venv (Windows) + elif [ -f ".venv/Scripts/activate" ]; then + source .venv/Scripts/activate + print_success "Virtual environment activated (Windows)" + else + print_error "Could not find virtual environment activation script" + exit 1 + fi + + # Verify activation worked + if [ -z "$VIRTUAL_ENV" ]; then + print_warning "Virtual environment may not be activated properly" + print_info "Continuing anyway..." + else + print_info "Virtual environment active: $VIRTUAL_ENV" + fi +} + +install_dependencies() { + local step_num=$1 + print_step "$step_num" "Installing dependencies with UV..." + + if [ ! -f "requirements.txt" ]; then + print_error "requirements.txt not found!" + exit 1 + fi + + # Ensure PyYAML is in requirements + if ! grep -qi "pyyaml" requirements.txt; then + print_warning "PyYAML not found in requirements.txt, adding it..." + echo "pyyaml>=6.0" >> requirements.txt + fi + + print_info "Installing dependencies (this may take a moment)..." + + # Ensure UV is in PATH + export PATH="$HOME/.cargo/bin:$PATH" + + if uv pip install -r requirements.txt; then + print_success "Dependencies installed successfully" + return 0 + else + print_error "Failed to install dependencies" + exit 1 + fi +} + +# YAML editing functions using Python +set_yaml_value() { + local file=$1 + local key=$2 + local value=$3 + + python3 << EOF +import yaml +import sys + +try: + with open("$file", 'r', encoding='utf-8') as f: + config = yaml.safe_load(f) or {} + + keys = "$key".split('.') + current = config + + for k in keys[:-1]: + if k not in current: + current[k] = {} + current = current[k] + + current[keys[-1]] = "$value" + + with open("$file", 'w', encoding='utf-8') as f: + yaml.dump(config, f, default_flow_style=False, sort_keys=False, allow_unicode=True) + + sys.exit(0) +except Exception as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) +EOF +} + +install_windows_mcp_servers() { + # Detect Windows (Git Bash, WSL, or native Windows) + if [[ "$OSTYPE" != "msys" && "$OSTYPE" != "win32" && "$OSTYPE" != "cygwin" ]]; then + # Not Windows, skip this step + return 0 + fi + + echo "" + print_info "Windows detected. Installing MCP servers globally..." + echo "" + echo " The following commands will be run:" + echo " ${CYAN}npm i -g @modelcontextprotocol/server-brave-search${ENDC}" + echo " ${CYAN}npm i -g @modelcontextprotocol/server-filesystem${ENDC}" + echo "" + read -p "Install MCP servers now? (Y/n): " install_servers + if [[ "$install_servers" =~ ^[Nn]$ ]]; then + print_warning "Skipping MCP server installation." + print_info "You can install them manually later." + return 0 + fi + + echo "" + print_info "Installing @modelcontextprotocol/server-brave-search..." + if npm i -g @modelcontextprotocol/server-brave-search; then + print_success "Installed @modelcontextprotocol/server-brave-search" + else + print_error "Failed to install @modelcontextprotocol/server-brave-search" + return 1 + fi + + echo "" + print_info "Installing @modelcontextprotocol/server-filesystem..." + if npm i -g @modelcontextprotocol/server-filesystem; then + print_success "Installed @modelcontextprotocol/server-filesystem" + else + print_error "Failed to install @modelcontextprotocol/server-filesystem" + return 1 + fi + + return 0 +} + +configure_windows_mcp_servers() { + # Detect Windows (Git Bash, WSL, or native Windows) + if [[ "$OSTYPE" != "msys" && "$OSTYPE" != "win32" && "$OSTYPE" != "cygwin" ]]; then + # Not Windows, skip this step + return 0 + fi + + local step_num=$1 + print_step "$step_num" "Configuring Windows MCP Servers" + + echo "" + print_info "Windows detected. MCP servers need to be configured with absolute paths." + echo "" + read -p "Configure Windows MCP servers? (Y/n): " configure_windows + if [[ "$configure_windows" =~ ^[Nn]$ ]]; then + print_info "Skipping Windows MCP server configuration" + return 0 + fi + + echo "" + print_info "Finding your global node_modules path..." + echo "" + print_info "Running: npm -g root" + npm_root=$(npm -g root 2>/dev/null) + + if [ -z "$npm_root" ]; then + print_error "Failed to get npm global root path." + print_info "You can configure this manually later by editing $CONFIG_FILE" + return 1 + fi + + print_success "Found npm global root: $npm_root" + + # Build Windows paths with forward slashes (YAML format) + local brave_path="${npm_root}/@modelcontextprotocol/server-brave-search/dist/index.js" + local filesystem_path="${npm_root}/@modelcontextprotocol/server-filesystem/dist/index.js" + + # Convert to Windows path format if needed (Git Bash/Cygwin) + if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then + # Git Bash/Cygwin - convert Unix path to Windows path + brave_path=$(cygpath -w "$brave_path" 2>/dev/null || echo "$brave_path") + filesystem_path=$(cygpath -w "$filesystem_path" 2>/dev/null || echo "$filesystem_path") + fi + + # Convert backslashes to forward slashes for YAML (Windows paths use forward slashes in YAML) + brave_path=$(echo "$brave_path" | sed 's|\\|/|g') + filesystem_path=$(echo "$filesystem_path" | sed 's|\\|/|g') + + echo "" + print_info "Updating $CONFIG_FILE with Windows paths..." + print_info "Brave path: $brave_path" + print_info "Filesystem path: $filesystem_path" + + # Update config using Python YAML + python3 << EOF +import yaml +import sys + +try: + with open("$CONFIG_FILE", 'r', encoding='utf-8') as f: + config = yaml.safe_load(f) or {} + + # Update brave server for Windows + if 'mcp' in config and 'servers' in config['mcp'] and 'brave' in config['mcp']['servers']: + config['mcp']['servers']['brave']['command'] = 'node' + config['mcp']['servers']['brave']['args'] = ["$brave_path"] + + # Update filesystem server for Windows + if 'mcp' in config and 'servers' in config['mcp'] and 'filesystem' in config['mcp']['servers']: + config['mcp']['servers']['filesystem']['command'] = 'node' + config['mcp']['servers']['filesystem']['args'] = ["$filesystem_path", "."] + + with open("$CONFIG_FILE", 'w', encoding='utf-8') as f: + yaml.dump(config, f, default_flow_style=False, sort_keys=False, allow_unicode=True) + + print("Updated Windows MCP server configuration") + sys.exit(0) +except Exception as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) +EOF + + if [ $? -eq 0 ]; then + print_success "Updated Windows MCP server configuration" + else + print_error "Failed to update Windows MCP server configuration" + print_info "You can configure this manually by editing $CONFIG_FILE" + return 1 + fi + + return 0 +} + +configure_search_server() { + local step_num=$1 + print_step "$step_num" "Configuring Search Server (Optional)" + + echo "" + echo " Search servers enable web search functionality for finding similar repositories" + echo " and code examples. This is optional - you can skip this step." + echo "" + print_warning "Without a search API key, the following features will be disabled:" + echo " โ€ข Searching for similar repositories" + echo " โ€ข Finding code examples from the web" + echo " โ€ข Web-based code reference lookup" + echo "" + echo " Available search servers:" + echo " [1] brave" + echo " [2] bocha-mcp" + echo " [3] Skip / None" + echo "" + + read -p "Select search server (1-3, default: 3): " SELECTION + SELECTION=${SELECTION:-3} + + if [[ ! "$SELECTION" =~ ^[0-9]+$ ]] || [ "$SELECTION" -lt 1 ] || [ "$SELECTION" -gt 3 ]; then + print_info "Skipping search server configuration" + return 0 + fi + + if [ "$SELECTION" -eq 3 ]; then + print_info "Skipping search server configuration" + echo "" + print_info "You can configure this later by editing $CONFIG_FILE" + return 0 + fi + + local server="" + case $SELECTION in + 1) server="brave" ;; + 2) server="bocha-mcp" ;; + *) return 0 ;; + esac + + if [ ! -f "$CONFIG_FILE" ]; then + print_error "$CONFIG_FILE not found!" + return 1 + fi + + set_yaml_value "$CONFIG_FILE" "default_search_server" "$server" + print_success "Set default_search_server to: $server" + + echo "" + # Convert server name to uppercase for display (portable method) + local server_upper=$(echo "$server" | tr '[:lower:]' '[:upper:]') + read -p "Enter $server_upper API key (optional, press Enter to skip): " api_key + + if [ -n "$api_key" ]; then + if [ "$server" = "brave" ]; then + set_yaml_value "$CONFIG_FILE" "mcp.servers.brave.env.BRAVE_API_KEY" "$api_key" + print_success "Set BRAVE_API_KEY" + elif [ "$server" = "bocha-mcp" ]; then + set_yaml_value "$CONFIG_FILE" "mcp.servers.bocha-mcp.env.BOCHA_API_KEY" "$api_key" + print_success "Set BOCHA_API_KEY" + fi + else + print_warning "No API key provided." + echo "" + print_warning "Without a search API key, the following features will be disabled:" + echo " โ€ข Searching for similar repositories" + echo " โ€ข Finding code examples from the web" + echo " โ€ข Web-based code reference lookup" + echo "" + print_info "You can add the API key later by editing $CONFIG_FILE" + fi + + return 0 +} + +configure_api_keys() { + local step_num=$1 + print_step "$step_num" "Configuring API Keys" + + print_info "Configure at least one API key. The first provider you configure will be set as the default LLM provider." + echo "" + + if [ ! -f "$SECRETS_FILE" ]; then + print_error "$SECRETS_FILE not found!" + return 1 + fi + + if [ ! -f "$CONFIG_FILE" ]; then + print_error "$CONFIG_FILE not found!" + return 1 + fi + + local first_provider="" + + while true; do + echo "" + echo " Select LLM provider to configure API key:" + echo " [1] google" + echo " [2] anthropic" + echo " [3] openai" + echo "" + + read -p "Select provider (1-3, required): " SELECTION + + if [[ ! "$SELECTION" =~ ^[0-9]+$ ]] || [ "$SELECTION" -lt 1 ] || [ "$SELECTION" -gt 3 ]; then + print_error "Invalid selection. Please enter 1, 2, or 3." + continue + fi + + local provider="" + case $SELECTION in + 1) provider="google" ;; + 2) provider="anthropic" ;; + 3) provider="openai" ;; + esac + + echo "" + # Capitalize first letter for display (portable method) + local provider_display="" + case $provider in + google) provider_display="Google" ;; + anthropic) provider_display="Anthropic" ;; + openai) provider_display="OpenAI" ;; + esac + read -p "Enter $provider_display API key: " api_key + + if [ -z "$api_key" ]; then + print_error "API key cannot be empty. Please try again." + continue + fi + + # Handle OpenAI special case (base_url) + if [ "$provider" = "openai" ]; then + read -p "Enter OpenAI base_url (optional, for custom endpoints, press Enter to skip): " base_url + + python3 << EOF +import yaml +with open("$SECRETS_FILE", 'r', encoding='utf-8') as f: + config = yaml.safe_load(f) or {} +if 'openai' not in config: + config['openai'] = {} +config['openai']['api_key'] = "$api_key" +if "$base_url": + config['openai']['base_url'] = "$base_url" +with open("$SECRETS_FILE", 'w', encoding='utf-8') as f: + yaml.dump(config, f, default_flow_style=False, sort_keys=False, allow_unicode=True) +EOF + print_success "Set OpenAI API key" + else + python3 << EOF +import yaml +with open("$SECRETS_FILE", 'r', encoding='utf-8') as f: + config = yaml.safe_load(f) or {} +if '$provider' not in config: + config['$provider'] = {} +config['$provider']['api_key'] = "$api_key" +with open("$SECRETS_FILE", 'w', encoding='utf-8') as f: + yaml.dump(config, f, default_flow_style=False, sort_keys=False, allow_unicode=True) +EOF + print_success "Set $provider_display API key" + fi + + # Set the first provider as the default LLM provider + if [ -z "$first_provider" ]; then + first_provider="$provider" + set_yaml_value "$CONFIG_FILE" "llm_provider" "$provider" + print_success "Set default LLM provider to: $provider" + fi + + echo "" + read -p "Configure another API key? (y/N): " configure_another + if [[ ! "$configure_another" =~ ^[Yy]$ ]]; then + break + fi + done +} + + +configure_document_segmentation() { + local step_num=$1 + print_step "$step_num" "Configuring Document Segmentation (Optional)" + + echo "" + read -p "Configure document segmentation? (y/N): " configure_seg + if [[ ! "$configure_seg" =~ ^[Yy]$ ]]; then + print_info "Skipping document segmentation configuration" + return 0 + fi + + if [ ! -f "$CONFIG_FILE" ]; then + print_error "$CONFIG_FILE not found!" + return 1 + fi + + echo "" + echo " Document segmentation options:" + echo " [1] enabled" + echo " [2] disabled" + echo "" + + read -p "Select option (1-2): " SELECTION + + local enabled="false" + if [[ "$SELECTION" =~ ^[0-9]+$ ]] && [ "$SELECTION" -eq 1 ]; then + enabled="true" + fi + + set_yaml_value "$CONFIG_FILE" "document_segmentation.enabled" "$enabled" + print_success "Set document_segmentation.enabled to: $enabled" + + if [ "$enabled" = "true" ]; then + read -p "Enter size threshold in characters (default: 50000): " threshold + threshold=${threshold:-50000} + set_yaml_value "$CONFIG_FILE" "document_segmentation.size_threshold_chars" "$threshold" + print_success "Set size_threshold_chars to: $threshold" + fi + + return 0 +} + +show_summary() { + local step_num=$1 + print_step "$step_num" "Setup Summary" + + echo -e "\n ${BOLD}Configuration Summary:${ENDC}\n" + + if [ -f "$CONFIG_FILE" ]; then + local search_server=$(python3 -c "import yaml; c=yaml.safe_load(open('$CONFIG_FILE')) or {}; print(c.get('default_search_server', 'not set'))" 2>/dev/null || echo "not set") + local llm_provider=$(python3 -c "import yaml; c=yaml.safe_load(open('$CONFIG_FILE')) or {}; print(c.get('llm_provider', 'not set'))" 2>/dev/null || echo "not set") + local seg_enabled=$(python3 -c "import yaml; c=yaml.safe_load(open('$CONFIG_FILE')) or {}; print(c.get('document_segmentation', {}).get('enabled', 'not set'))" 2>/dev/null || echo "not set") + + echo -e " ${CYAN}Search Server:${ENDC} $search_server" + echo -e " ${CYAN}LLM Provider:${ENDC} $llm_provider" + echo -e " ${CYAN}Document Segmentation:${ENDC} $seg_enabled" + fi + + if [ -f "$SECRETS_FILE" ]; then + local has_openai=$(python3 -c "import yaml; c=yaml.safe_load(open('$SECRETS_FILE')) or {}; print('yes' if c.get('openai', {}).get('api_key') else 'no')" 2>/dev/null || echo "no") + local has_anthropic=$(python3 -c "import yaml; c=yaml.safe_load(open('$SECRETS_FILE')) or {}; print('yes' if c.get('anthropic', {}).get('api_key') else 'no')" 2>/dev/null || echo "no") + local has_google=$(python3 -c "import yaml; c=yaml.safe_load(open('$SECRETS_FILE')) or {}; print('yes' if c.get('google', {}).get('api_key') else 'no')" 2>/dev/null || echo "no") + + echo -e "\n ${CYAN}API Keys Configured:${ENDC}" + echo -e " OpenAI: $has_openai" + echo -e " Anthropic: $has_anthropic" + echo -e " Google: $has_google" + fi + + echo "" +} + +main() { + print_header + + echo -e " ${BOLD}Welcome to DeepCode Setup!${ENDC}\n" + echo " This wizard will help you configure DeepCode automatically." + echo " Following the official installation recipe using UV package manager." + echo "" + echo " Assuming you're already in the DeepCode directory." + echo "" + + read -p " Ready to begin setup? (Y/n): " begin + if [[ "$begin" =~ ^[Nn]$ ]]; then + echo " Setup cancelled." + exit 0 + fi + + # Track step number dynamically + local step_counter=1 + + # Step 1: Install UV + install_uv $step_counter + ((step_counter++)) + + # Step 2: Setup venv with UV + setup_venv $step_counter + ((step_counter++)) + + # Step 3: Install dependencies with UV + install_dependencies $step_counter + ((step_counter++)) + + # Install Windows MCP servers (Windows only, before search server config) + # This doesn't show a step number, it's just a prerequisite + install_windows_mcp_servers + + # Step 4: Configure search server + configure_search_server $step_counter + ((step_counter++)) + + # Step 5: Configure API keys (also sets default LLM provider) + configure_api_keys $step_counter + ((step_counter++)) + + # Step 6: Configure Windows MCP servers (Windows only - configures paths) + # Only increments counter if Windows (function returns early if not Windows) + if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "win32" || "$OSTYPE" == "cygwin" ]]; then + configure_windows_mcp_servers $step_counter + ((step_counter++)) + fi + + # Step 7: Configure document segmentation (optional) + configure_document_segmentation $step_counter + ((step_counter++)) + + # Step 8: Show summary + show_summary $step_counter + + # Final message + echo -e "\n ${GREEN}${BOLD}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—${ENDC}" + echo -e " ${GREEN}${BOLD}โ•‘ โœ… Setup Complete! โœ… โ•‘${ENDC}" + echo -e " ${GREEN}${BOLD}โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•${ENDC}\n" + + echo -e " ${BOLD}Next steps:${ENDC}\n" + echo -e " 1. Review configuration files:" + echo -e " - $CONFIG_FILE" + echo -e " - $SECRETS_FILE" + echo "" + echo -e " 2. Make sure your virtual environment is activated:" + echo -e " ${CYAN}source .venv/bin/activate${ENDC} (Linux/macOS)" + echo -e " ${CYAN}.venv\\\\Scripts\\\\activate${ENDC} (Windows)" + echo "" + echo -e " 3. Launch DeepCode:" + echo -e " ${CYAN}python deepcode.py${ENDC} (Web interface)" + echo -e " ${CYAN}python cli/main_cli.py${ENDC} (CLI interface)" + echo "" +} + +# Run main function +main "$@" diff --git a/DeepCode/tools/__init__.py b/DeepCode/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/DeepCode/tools/bocha_search_server.py b/DeepCode/tools/bocha_search_server.py new file mode 100644 index 00000000..9d4392a0 --- /dev/null +++ b/DeepCode/tools/bocha_search_server.py @@ -0,0 +1,219 @@ +import os +import sys +import json + +import httpx +from dotenv import load_dotenv +from mcp.server.fastmcp import FastMCP + +load_dotenv() + + +# Initialize FastMCP server +server = FastMCP( + "bocha-search-mcp", + prompt=""" +# Bocha Search MCP Server + +Bocha is a Chinese search engine for AI, This server provides tools for searching the web using Bocha Search API. +It allows you to get enhanced search details from billions of web documents, including weather, news, wikis, healthcare, train tickets, images, and more. + +## Available Tools + +### 1. bocha_web_search +Search with Bocha Web Search and get enhanced search details from billions of web documents, including page titles, urls, summaries, site names, site icons, publication dates, image links, and more. + +### 2. bocha_ai_search +Search with Bocha AI Search, recognizes the semantics of search terms and additionally returns structured modal cards with content from vertical domains. + +## Output Format + +All search results will be formatted as text with clear sections for each +result item, including: + +- Bocha Web search: Title, URL, Description, Published date and Site name +- Bocha AI search: Title, URL, Description, Published date, Site name, and structured data card + +If the API key is missing or invalid, appropriate error messages will be returned. +""", +) + + +@server.tool() +async def bocha_web_search( + query: str, freshness: str = "noLimit", count: int = 10 +) -> str: + """Search with Bocha Web Search and get enhanced search details from billions of web documents, + including page titles, urls, summaries, site names, site icons, publication dates, image links, and more. + + Args: + query: Search query (required) + freshness: The time range for the search results. (Available options YYYY-MM-DD, YYYY-MM-DD..YYYY-MM-DD, noLimit, oneYear, oneMonth, oneWeek, oneDay. Default is noLimit) + count: Number of results (1-50, default 10) + """ + # Get API key from environment + boch_api_key = os.environ.get("BOCHA_API_KEY", "") + + if not boch_api_key: + return ( + "Error: Bocha API key is not configured. Please set the " + "BOCHA_API_KEY environment variable." + ) + + # Endpoint + endpoint = "https://api.bochaai.com/v1/web-search?utm_source=bocha-mcp-local" + + try: + payload = { + "query": query, + "summary": True, + "freshness": freshness, + "count": count, + } + + headers = { + "Authorization": f"Bearer {boch_api_key}", + "Content-Type": "application/json", + } + + async with httpx.AsyncClient() as client: + response = await client.post( + endpoint, headers=headers, json=payload, timeout=10.0 + ) + + response.raise_for_status() + resp = response.json() + if "data" not in resp: + return "Search error." + + data = resp["data"] + + if "webPages" not in data: + return "No results found." + + results = [] + for result in data["webPages"]["value"]: + results.append( + f"Title: {result['name']}\n" + f"URL: {result['url']}\n" + f"Description: {result['summary']}\n" + f"Published date: {result['datePublished']}\n" + f"Site name: {result['siteName']}" + ) + + return "\n\n".join(results) + + except httpx.HTTPStatusError as e: + return f"Bocha Web Search API HTTP error occurred: {e.response.status_code} - {e.response.text}" + except httpx.RequestError as e: + return f"Error communicating with Bocha Web Search API: {str(e)}" + except Exception as e: + return f"Unexpected error: {str(e)}" + + +@server.tool() +async def bocha_ai_search( + query: str, freshness: str = "noLimit", count: int = 10 +) -> str: + """Search with Bocha AI Search, recognizes the semantics of search terms + and additionally returns structured modal cards with content from vertical domains. + + Args: + query: Search query (required) + freshness: The time range for the search results. (Available options noLimit, oneYear, oneMonth, oneWeek, oneDay. Default is noLimit) + count: Number of results (1-50, default 10) + """ + # Get API key from environment + boch_api_key = os.environ.get("BOCHA_API_KEY", "") + + if not boch_api_key: + return ( + "Error: Bocha API key is not configured. Please set the " + "BOCHA_API_KEY environment variable." + ) + + # Endpoint + endpoint = "https://api.bochaai.com/v1/ai-search?utm_source=bocha-mcp-local" + + try: + payload = { + "query": query, + "freshness": freshness, + "count": count, + "answer": False, + "stream": False, + } + + headers = { + "Authorization": f"Bearer {boch_api_key}", + "Content-Type": "application/json", + } + + async with httpx.AsyncClient() as client: + response = await client.post( + endpoint, headers=headers, json=payload, timeout=10.0 + ) + + response.raise_for_status() + response = response.json() + results = [] + if "messages" in response: + for message in response["messages"]: + content = {} + try: + content = json.loads(message["content"]) + except (json.JSONDecodeError, TypeError): + content = {} + + # ็ฝ‘้กต + if message["content_type"] == "webpage": + if "value" in content: + for item in content["value"]: + results.append( + f"Title: {item['name']}\n" + f"URL: {item['url']}\n" + f"Description: {item['summary']}\n" + f"Published date: {item['datePublished']}\n" + f"Site name: {item['siteName']}" + ) + elif ( + message["content_type"] != "image" + and message["content"] != "{}" + ): + results.append(message["content"]) + + if not results: + return "No results found." + + return "\n\n".join(results) + + except httpx.HTTPStatusError as e: + return f"Bocha AI Search API HTTP error occurred: {e.response.status_code} - {e.response.text}" + except httpx.RequestError as e: + return f"Error communicating with Bocha AI Search API: {str(e)}" + except Exception as e: + return f"Unexpected error: {str(e)}" + + +def main(): + """Initialize and run the MCP server.""" + + # Check for required environment variables + if "BOCHA_API_KEY" not in os.environ: + print( + "Error: BOCHA_API_KEY environment variable is required", + file=sys.stderr, + ) + print( + "Get a Bocha API key from: " "https://open.bochaai.com", + file=sys.stderr, + ) + sys.exit(1) + + print("Starting Bocha Search MCP server...", file=sys.stderr) + + server.run(transport="stdio") + + +if __name__ == "__main__": + main() diff --git a/DeepCode/tools/code_implementation_server.py b/DeepCode/tools/code_implementation_server.py new file mode 100644 index 00000000..694dd0ee --- /dev/null +++ b/DeepCode/tools/code_implementation_server.py @@ -0,0 +1,1517 @@ +#!/usr/bin/env python3 +""" +Code Implementation MCP Server + +This MCP server provides core functions needed for paper code reproduction: +1. File read/write operations +2. Code execution and testing +3. Code search and analysis +4. Iterative improvement support + +Usage: +python tools/code_implementation_server.py +""" + +import os +import subprocess +import json +import sys +import io +from pathlib import Path +import re +from typing import Dict, Any, List +import tempfile +import shutil +import logging +from datetime import datetime + +# Set standard output encoding to UTF-8 +if sys.stdout.encoding != "utf-8": + try: + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8") + sys.stderr.reconfigure(encoding="utf-8") + else: + sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding="utf-8") + sys.stderr = io.TextIOWrapper(sys.stderr.detach(), encoding="utf-8") + except Exception as e: + print(f"Warning: Could not set UTF-8 encoding: {e}") + +# Import MCP related modules +from mcp.server.fastmcp import FastMCP + +# Setup logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Create FastMCP server instance +mcp = FastMCP("code-implementation-server") + +# Global variables: workspace directory and operation history +WORKSPACE_DIR = None +OPERATION_HISTORY = [] +CURRENT_FILES = {} + + +def initialize_workspace(workspace_dir: str = None): + """ + Initialize workspace + + By default, the workspace will be set by the workflow via the set_workspace tool to: + {plan_file_parent}/generate_code + + Args: + workspace_dir: Optional workspace directory path + """ + global WORKSPACE_DIR + if workspace_dir is None: + # Default to generate_code directory under current directory, but don't create immediately + # This default value will be overridden by workflow via set_workspace tool + WORKSPACE_DIR = Path.cwd() / "generate_code" + # logger.info(f"Workspace initialized (default value, will be overridden by workflow): {WORKSPACE_DIR}") + # logger.info("Note: Actual workspace will be set by workflow via set_workspace tool to {plan_file_parent}/generate_code") + else: + WORKSPACE_DIR = Path(workspace_dir).resolve() + # Only create when explicitly specified + WORKSPACE_DIR.mkdir(parents=True, exist_ok=True) + logger.info(f"Workspace initialized: {WORKSPACE_DIR}") + + +def ensure_workspace_exists(): + """Ensure workspace directory exists""" + global WORKSPACE_DIR + if WORKSPACE_DIR is None: + initialize_workspace() + + # Create workspace directory (if it doesn't exist) + if not WORKSPACE_DIR.exists(): + WORKSPACE_DIR.mkdir(parents=True, exist_ok=True) + logger.info(f"Workspace directory created: {WORKSPACE_DIR}") + + +def validate_path(path: str) -> Path: + """Validate if path is within workspace""" + if WORKSPACE_DIR is None: + initialize_workspace() + + full_path = (WORKSPACE_DIR / path).resolve() + if not str(full_path).startswith(str(WORKSPACE_DIR)): + raise ValueError(f"Path {path} is outside workspace scope") + return full_path + + +def log_operation(action: str, details: Dict[str, Any]): + """Log operation history""" + OPERATION_HISTORY.append( + {"timestamp": datetime.now().isoformat(), "action": action, "details": details} + ) + + +# ==================== File Operation Tools ==================== + + +@mcp.tool() +async def read_file( + file_path: str, start_line: int = None, end_line: int = None +) -> str: + """ + Read file content, supports specifying line number range + + Args: + file_path: File path, relative to workspace + start_line: Starting line number (1-based, optional) + end_line: Ending line number (1-based, optional) + + Returns: + JSON string of file content or error message + """ + try: + full_path = validate_path(file_path) + + if not full_path.exists(): + result = {"status": "error", "message": f"File does not exist: {file_path}"} + log_operation( + "read_file_error", {"file_path": file_path, "error": "file_not_found"} + ) + return json.dumps(result, ensure_ascii=False, indent=2) + + with open(full_path, "r", encoding="utf-8") as f: + lines = f.readlines() + + # ๅค„็†่กŒๅท่Œƒๅ›ด + if start_line is not None or end_line is not None: + start_idx = (start_line - 1) if start_line else 0 + end_idx = end_line if end_line else len(lines) + lines = lines[start_idx:end_idx] + + content = "".join(lines) + + result = { + "status": "success", + "content": content, + "file_path": file_path, + "total_lines": len(lines), + "size_bytes": len(content.encode("utf-8")), + } + + log_operation( + "read_file", + { + "file_path": file_path, + "start_line": start_line, + "end_line": end_line, + "lines_read": len(lines), + }, + ) + + return json.dumps(result, ensure_ascii=False, indent=2) + + except Exception as e: + result = { + "status": "error", + "message": f"Failed to read file: {str(e)}", + "file_path": file_path, + } + log_operation("read_file_error", {"file_path": file_path, "error": str(e)}) + return json.dumps(result, ensure_ascii=False, indent=2) + + +@mcp.tool() +async def read_multiple_files(file_requests: str, max_files: int = 5) -> str: + """ + Read multiple files in a single operation (for batch reading) + + Args: + file_requests: JSON string with file requests, e.g., + '{"file1.py": {}, "file2.py": {"start_line": 1, "end_line": 10}}' + or simple array: '["file1.py", "file2.py"]' + max_files: Maximum number of files to read in one operation (default: 5) + + Returns: + JSON string of operation results for all files + """ + try: + # Parse the file requests + try: + requests_data = json.loads(file_requests) + except json.JSONDecodeError as e: + return json.dumps( + { + "status": "error", + "message": f"Invalid JSON format for file_requests: {str(e)}", + "operation_type": "multi_file", + "timestamp": datetime.now().isoformat(), + }, + ensure_ascii=False, + indent=2, + ) + + # Normalize requests format + if isinstance(requests_data, list): + # Convert simple array to dict format + normalized_requests = {file_path: {} for file_path in requests_data} + elif isinstance(requests_data, dict): + normalized_requests = requests_data + else: + return json.dumps( + { + "status": "error", + "message": "file_requests must be a JSON object or array", + "operation_type": "multi_file", + "timestamp": datetime.now().isoformat(), + }, + ensure_ascii=False, + indent=2, + ) + + # Validate input + if len(normalized_requests) == 0: + return json.dumps( + { + "status": "error", + "message": "No files provided for reading", + "operation_type": "multi_file", + "timestamp": datetime.now().isoformat(), + }, + ensure_ascii=False, + indent=2, + ) + + if len(normalized_requests) > max_files: + return json.dumps( + { + "status": "error", + "message": f"Too many files provided ({len(normalized_requests)}), maximum is {max_files}", + "operation_type": "multi_file", + "timestamp": datetime.now().isoformat(), + }, + ensure_ascii=False, + indent=2, + ) + + # Process each file + results = { + "status": "success", + "message": f"Successfully processed {len(normalized_requests)} files", + "operation_type": "multi_file", + "timestamp": datetime.now().isoformat(), + "files_processed": len(normalized_requests), + "files": {}, + "summary": { + "successful": 0, + "failed": 0, + "total_size_bytes": 0, + "total_lines": 0, + "files_not_found": 0, + }, + } + + # Process each file individually + for file_path, options in normalized_requests.items(): + try: + full_path = validate_path(file_path) + start_line = options.get("start_line") + end_line = options.get("end_line") + + if not full_path.exists(): + results["files"][file_path] = { + "status": "error", + "message": f"File does not exist: {file_path}", + "file_path": file_path, + "content": "", + "total_lines": 0, + "size_bytes": 0, + "start_line": start_line, + "end_line": end_line, + } + results["summary"]["failed"] += 1 + results["summary"]["files_not_found"] += 1 + continue + + with open(full_path, "r", encoding="utf-8") as f: + lines = f.readlines() + + # Handle line range + original_line_count = len(lines) + if start_line is not None or end_line is not None: + start_idx = (start_line - 1) if start_line else 0 + end_idx = end_line if end_line else len(lines) + lines = lines[start_idx:end_idx] + + content = "".join(lines) + size_bytes = len(content.encode("utf-8")) + lines_count = len(lines) + + # Record individual file result + results["files"][file_path] = { + "status": "success", + "message": f"File read successfully: {file_path}", + "file_path": file_path, + "content": content, + "total_lines": lines_count, + "original_total_lines": original_line_count, + "size_bytes": size_bytes, + "start_line": start_line, + "end_line": end_line, + "line_range_applied": start_line is not None + or end_line is not None, + } + + # Update summary + results["summary"]["successful"] += 1 + results["summary"]["total_size_bytes"] += size_bytes + results["summary"]["total_lines"] += lines_count + + # Log individual file operation + log_operation( + "read_file_multi", + { + "file_path": file_path, + "start_line": start_line, + "end_line": end_line, + "lines_read": lines_count, + "size_bytes": size_bytes, + "batch_operation": True, + }, + ) + + except Exception as file_error: + # Record individual file error + results["files"][file_path] = { + "status": "error", + "message": f"Failed to read file: {str(file_error)}", + "file_path": file_path, + "content": "", + "total_lines": 0, + "size_bytes": 0, + "start_line": options.get("start_line"), + "end_line": options.get("end_line"), + } + + results["summary"]["failed"] += 1 + + # Log individual file error + log_operation( + "read_file_multi_error", + { + "file_path": file_path, + "error": str(file_error), + "batch_operation": True, + }, + ) + + # Determine overall status + if results["summary"]["failed"] > 0: + if results["summary"]["successful"] > 0: + results["status"] = "partial_success" + results["message"] = ( + f"Read {results['summary']['successful']} files successfully, {results['summary']['failed']} failed" + ) + else: + results["status"] = "failed" + results["message"] = ( + f"All {results['summary']['failed']} files failed to read" + ) + + # Log overall operation + log_operation( + "read_multiple_files", + { + "files_count": len(normalized_requests), + "successful": results["summary"]["successful"], + "failed": results["summary"]["failed"], + "total_size_bytes": results["summary"]["total_size_bytes"], + "status": results["status"], + }, + ) + + return json.dumps(results, ensure_ascii=False, indent=2) + + except Exception as e: + result = { + "status": "error", + "message": f"Failed to read multiple files: {str(e)}", + "operation_type": "multi_file", + "timestamp": datetime.now().isoformat(), + "files_processed": 0, + } + log_operation("read_multiple_files_error", {"error": str(e)}) + return json.dumps(result, ensure_ascii=False, indent=2) + + +@mcp.tool() +async def write_file( + file_path: str, content: str, create_dirs: bool = True, create_backup: bool = False +) -> str: + """ + Write content to file + + Args: + file_path: File path, relative to workspace + content: Content to write to file + create_dirs: Whether to create directories if they don't exist + create_backup: Whether to create backup file if file already exists + + Returns: + JSON string of operation result + """ + try: + full_path = validate_path(file_path) + + # Create directories (if needed) + if create_dirs: + full_path.parent.mkdir(parents=True, exist_ok=True) + + # Backup existing file (only when explicitly requested) + backup_created = False + if full_path.exists() and create_backup: + backup_path = full_path.with_suffix(full_path.suffix + ".backup") + shutil.copy2(full_path, backup_path) + backup_created = True + + # Write file + with open(full_path, "w", encoding="utf-8") as f: + f.write(content) + + # Update current file record + CURRENT_FILES[file_path] = { + "last_modified": datetime.now().isoformat(), + "size_bytes": len(content.encode("utf-8")), + "lines": len(content.split("\n")), + } + + result = { + "status": "success", + "message": f"File written successfully: {file_path}", + "file_path": file_path, + "size_bytes": len(content.encode("utf-8")), + "lines_written": len(content.split("\n")), + "backup_created": backup_created, + } + + log_operation( + "write_file", + { + "file_path": file_path, + "size_bytes": len(content.encode("utf-8")), + "lines": len(content.split("\n")), + "backup_created": backup_created, + }, + ) + + return json.dumps(result, ensure_ascii=False, indent=2) + + except Exception as e: + result = { + "status": "error", + "message": f"Failed to write file: {str(e)}", + "file_path": file_path, + } + log_operation("write_file_error", {"file_path": file_path, "error": str(e)}) + return json.dumps(result, ensure_ascii=False, indent=2) + + +@mcp.tool() +async def write_multiple_files( + file_implementations: str, + create_dirs: bool = True, + create_backup: bool = False, + max_files: int = 5, +) -> str: + """ + Write multiple files in a single operation (for batch implementation) + + Args: + file_implementations: JSON string mapping file paths to content, e.g., + '{"file1.py": "content1", "file2.py": "content2"}' + create_dirs: Whether to create directories if they don't exist + create_backup: Whether to create backup files if they already exist + max_files: Maximum number of files to write in one operation (default: 5) + + Returns: + JSON string of operation results for all files + """ + try: + # Parse the file implementations + try: + files_dict = json.loads(file_implementations) + except json.JSONDecodeError as e: + return json.dumps( + { + "status": "error", + "message": f"Invalid JSON format for file_implementations: {str(e)}", + "operation_type": "multi_file", + "timestamp": datetime.now().isoformat(), + }, + ensure_ascii=False, + indent=2, + ) + + # Validate input + if not isinstance(files_dict, dict): + return json.dumps( + { + "status": "error", + "message": "file_implementations must be a JSON object mapping file paths to content", + "operation_type": "multi_file", + "timestamp": datetime.now().isoformat(), + }, + ensure_ascii=False, + indent=2, + ) + + if len(files_dict) == 0: + return json.dumps( + { + "status": "error", + "message": "No files provided for writing", + "operation_type": "multi_file", + "timestamp": datetime.now().isoformat(), + }, + ensure_ascii=False, + indent=2, + ) + + if len(files_dict) > max_files: + return json.dumps( + { + "status": "error", + "message": f"Too many files provided ({len(files_dict)}), maximum is {max_files}", + "operation_type": "multi_file", + "timestamp": datetime.now().isoformat(), + }, + ensure_ascii=False, + indent=2, + ) + + # Process each file + results = { + "status": "success", + "message": f"Successfully processed {len(files_dict)} files", + "operation_type": "multi_file", + "timestamp": datetime.now().isoformat(), + "files_processed": len(files_dict), + "files": {}, + "summary": { + "successful": 0, + "failed": 0, + "total_size_bytes": 0, + "total_lines": 0, + "backups_created": 0, + }, + } + + # Process each file individually + for file_path, content in files_dict.items(): + try: + full_path = validate_path(file_path) + + # Create directories (if needed) + if create_dirs: + full_path.parent.mkdir(parents=True, exist_ok=True) + + # Backup existing file (only when explicitly requested) + backup_created = False + if full_path.exists() and create_backup: + backup_path = full_path.with_suffix(full_path.suffix + ".backup") + shutil.copy2(full_path, backup_path) + backup_created = True + results["summary"]["backups_created"] += 1 + + # Write file + with open(full_path, "w", encoding="utf-8") as f: + f.write(content) + + # Calculate file metrics + size_bytes = len(content.encode("utf-8")) + lines_count = len(content.split("\n")) + + # Update current file record + CURRENT_FILES[file_path] = { + "last_modified": datetime.now().isoformat(), + "size_bytes": size_bytes, + "lines": lines_count, + } + + # Record individual file result + results["files"][file_path] = { + "status": "success", + "message": f"File written successfully: {file_path}", + "size_bytes": size_bytes, + "lines_written": lines_count, + "backup_created": backup_created, + } + + # Update summary + results["summary"]["successful"] += 1 + results["summary"]["total_size_bytes"] += size_bytes + results["summary"]["total_lines"] += lines_count + + # Log individual file operation + log_operation( + "write_file_multi", + { + "file_path": file_path, + "size_bytes": size_bytes, + "lines": lines_count, + "backup_created": backup_created, + "batch_operation": True, + }, + ) + + except Exception as file_error: + # Record individual file error + results["files"][file_path] = { + "status": "error", + "message": f"Failed to write file: {str(file_error)}", + "size_bytes": 0, + "lines_written": 0, + "backup_created": False, + } + + results["summary"]["failed"] += 1 + + # Log individual file error + log_operation( + "write_file_multi_error", + { + "file_path": file_path, + "error": str(file_error), + "batch_operation": True, + }, + ) + + # Determine overall status + if results["summary"]["failed"] > 0: + if results["summary"]["successful"] > 0: + results["status"] = "partial_success" + results["message"] = ( + f"Processed {results['summary']['successful']} files successfully, {results['summary']['failed']} failed" + ) + else: + results["status"] = "failed" + results["message"] = ( + f"All {results['summary']['failed']} files failed to write" + ) + + # Log overall operation + log_operation( + "write_multiple_files", + { + "files_count": len(files_dict), + "successful": results["summary"]["successful"], + "failed": results["summary"]["failed"], + "total_size_bytes": results["summary"]["total_size_bytes"], + "status": results["status"], + }, + ) + + return json.dumps(results, ensure_ascii=False, indent=2) + + except Exception as e: + result = { + "status": "error", + "message": f"Failed to write multiple files: {str(e)}", + "operation_type": "multi_file", + "timestamp": datetime.now().isoformat(), + "files_processed": 0, + } + log_operation("write_multiple_files_error", {"error": str(e)}) + return json.dumps(result, ensure_ascii=False, indent=2) + + +# ==================== Code Execution Tools ==================== + + +@mcp.tool() +async def execute_python(code: str, timeout: int = 30) -> str: + """ + Execute Python code and return output + + Args: + code: Python code to execute + timeout: Timeout in seconds + + Returns: + JSON string of execution result + """ + try: + # Create temporary file + with tempfile.NamedTemporaryFile( + mode="w", suffix=".py", delete=False, encoding="utf-8" + ) as f: + f.write(code) + temp_file = f.name + + try: + # Ensure workspace directory exists + ensure_workspace_exists() + + # Execute Python code + result = subprocess.run( + [sys.executable, temp_file], + cwd=WORKSPACE_DIR, + capture_output=True, + text=True, + timeout=timeout, + encoding="utf-8", + ) + + execution_result = { + "status": "success" if result.returncode == 0 else "error", + "return_code": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + "timeout": timeout, + } + + if result.returncode != 0: + execution_result["message"] = "Python code execution failed" + else: + execution_result["message"] = "Python code execution successful" + + log_operation( + "execute_python", + { + "return_code": result.returncode, + "stdout_length": len(result.stdout), + "stderr_length": len(result.stderr), + }, + ) + + return json.dumps(execution_result, ensure_ascii=False, indent=2) + + finally: + # Clean up temporary file + os.unlink(temp_file) + + except subprocess.TimeoutExpired: + result = { + "status": "error", + "message": f"Python code execution timeout ({timeout}็ง’)", + "timeout": timeout, + } + log_operation("execute_python_timeout", {"timeout": timeout}) + return json.dumps(result, ensure_ascii=False, indent=2) + + except Exception as e: + result = { + "status": "error", + "message": f"Python code execution failed: {str(e)}", + } + log_operation("execute_python_error", {"error": str(e)}) + return json.dumps(result, ensure_ascii=False, indent=2) + + +@mcp.tool() +async def execute_bash(command: str, timeout: int = 30) -> str: + """ + Execute bash command + + Args: + command: Bash command to execute + timeout: Timeout in seconds + + Returns: + JSON string of execution result + """ + try: + # ๅฎ‰ๅ…จๆฃ€ๆŸฅ๏ผš็ฆๆญขๅฑ้™ฉๅ‘ฝไปค + dangerous_commands = ["rm -rf", "sudo", "chmod 777", "mkfs", "dd if="] + if any(dangerous in command.lower() for dangerous in dangerous_commands): + result = { + "status": "error", + "message": f"Dangerous command execution prohibited: {command}", + } + log_operation( + "execute_bash_blocked", + {"command": command, "reason": "dangerous_command"}, + ) + return json.dumps(result, ensure_ascii=False, indent=2) + + # Ensure workspace directory exists + ensure_workspace_exists() + + # Execute command + result = subprocess.run( + command, + shell=True, + cwd=WORKSPACE_DIR, + capture_output=True, + text=True, + timeout=timeout, + encoding="utf-8", + ) + + execution_result = { + "status": "success" if result.returncode == 0 else "error", + "return_code": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + "command": command, + "timeout": timeout, + } + + if result.returncode != 0: + execution_result["message"] = "Bash command execution failed" + else: + execution_result["message"] = "Bash command execution successful" + + log_operation( + "execute_bash", + { + "command": command, + "return_code": result.returncode, + "stdout_length": len(result.stdout), + "stderr_length": len(result.stderr), + }, + ) + + return json.dumps(execution_result, ensure_ascii=False, indent=2) + + except subprocess.TimeoutExpired: + result = { + "status": "error", + "message": f"Bash command execution timeout ({timeout} seconds)", + "command": command, + "timeout": timeout, + } + log_operation("execute_bash_timeout", {"command": command, "timeout": timeout}) + return json.dumps(result, ensure_ascii=False, indent=2) + + except Exception as e: + result = { + "status": "error", + "message": f"Failed to execute bash command: {str(e)}", + "command": command, + } + log_operation("execute_bash_error", {"command": command, "error": str(e)}) + return json.dumps(result, ensure_ascii=False, indent=2) + + +@mcp.tool() +async def read_code_mem(file_paths: List[str]) -> str: + """ + Check if file summaries exist in implement_code_summary.md for multiple files + + Args: + file_paths: List of file paths to check for summary information in implement_code_summary.md + + Returns: + Summary information for all requested files if available + """ + try: + if not file_paths or not isinstance(file_paths, list): + result = { + "status": "error", + "message": "file_paths parameter is required and must be a list", + } + log_operation( + "read_code_mem_error", {"error": "missing_or_invalid_file_paths"} + ) + return json.dumps(result, ensure_ascii=False, indent=2) + + # Remove duplicates while preserving order + unique_file_paths = list(dict.fromkeys(file_paths)) + + # Ensure workspace exists + ensure_workspace_exists() + + # Look for implement_code_summary.md in the workspace + current_path = Path(WORKSPACE_DIR) + summary_file_path = current_path.parent / "implement_code_summary.md" + + if not summary_file_path.exists(): + result = { + "status": "no_summary", + "file_paths": unique_file_paths, + "message": "No summary file found.", + "results": [], + } + log_operation( + "read_code_mem", + {"file_paths": unique_file_paths, "status": "no_summary_file"}, + ) + return json.dumps(result, ensure_ascii=False, indent=2) + + # Read the summary file + with open(summary_file_path, "r", encoding="utf-8") as f: + summary_content = f.read() + + if not summary_content.strip(): + result = { + "status": "no_summary", + "file_paths": unique_file_paths, + "message": "Summary file is empty.", + "results": [], + } + log_operation( + "read_code_mem", + {"file_paths": unique_file_paths, "status": "empty_summary"}, + ) + return json.dumps(result, ensure_ascii=False, indent=2) + + # Process each file path and collect results + results = [] + summaries_found = 0 + + for file_path in unique_file_paths: + # Extract file-specific section from summary + file_section = _extract_file_section_from_summary( + summary_content, file_path + ) + + if file_section: + file_result = { + "file_path": file_path, + "status": "summary_found", + "summary_content": file_section, + "message": f"Summary information found for {file_path}", + } + summaries_found += 1 + else: + file_result = { + "file_path": file_path, + "status": "no_summary", + "summary_content": None, + "message": f"No summary found for {file_path}", + } + + results.append(file_result) + + # Determine overall status + if summaries_found == len(unique_file_paths): + overall_status = "all_summaries_found" + elif summaries_found > 0: + overall_status = "partial_summaries_found" + else: + overall_status = "no_summaries_found" + + result = { + "status": overall_status, + "file_paths": unique_file_paths, + "total_requested": len(unique_file_paths), + "summaries_found": summaries_found, + "message": f"Found summaries for {summaries_found}/{len(unique_file_paths)} files", + "results": results, + } + + log_operation( + "read_code_mem", + { + "file_paths": unique_file_paths, + "status": overall_status, + "total_requested": len(unique_file_paths), + "summaries_found": summaries_found, + }, + ) + + return json.dumps(result, ensure_ascii=False, indent=2) + + except Exception as e: + result = { + "status": "error", + "message": f"Failed to check code memory: {str(e)}", + "file_paths": file_paths + if isinstance(file_paths, list) + else [str(file_paths)], + "results": [], + } + log_operation( + "read_code_mem_error", {"file_paths": file_paths, "error": str(e)} + ) + return json.dumps(result, ensure_ascii=False, indent=2) + + +def _extract_file_section_from_summary( + summary_content: str, target_file_path: str +) -> str: + """ + Extract the specific section for a file from the summary content + + Args: + summary_content: Full summary content + target_file_path: Path of the target file + + Returns: + File-specific section or None if not found + """ + import re + + # Normalize the target path for comparison + normalized_target = _normalize_file_path(target_file_path) + + # Pattern to match implementation sections with separator lines + section_pattern = r"={80}\s*\n## IMPLEMENTATION File ([^;]+); ROUND \d+\s*\n={80}(.*?)(?=\n={80}|\Z)" + + matches = re.findall(section_pattern, summary_content, re.DOTALL) + + for file_path_in_summary, section_content in matches: + file_path_in_summary = file_path_in_summary.strip() + section_content = section_content.strip() + + # Normalize the path from summary for comparison + normalized_summary_path = _normalize_file_path(file_path_in_summary) + + # Check if paths match using multiple strategies + if _paths_match( + normalized_target, + normalized_summary_path, + target_file_path, + file_path_in_summary, + ): + # Return the complete section with proper formatting + file_section = f"""================================================================================ +## IMPLEMENTATION File {file_path_in_summary}; ROUND [X] +================================================================================ + +{section_content} + +--- +*Extracted from implement_code_summary.md*""" + return file_section + + # If no section-based match, try alternative parsing method + return _extract_file_section_alternative(summary_content, target_file_path) + + +def _normalize_file_path(file_path: str) -> str: + """Normalize file path for comparison""" + # Remove leading/trailing slashes and convert to lowercase + normalized = file_path.strip("/").lower() + # Replace backslashes with forward slashes + normalized = normalized.replace("\\", "/") + + # Remove common prefixes to make matching more flexible + common_prefixes = ["src/", "./src/", "./", "core/", "lib/", "main/"] + for prefix in common_prefixes: + if normalized.startswith(prefix): + normalized = normalized[len(prefix) :] + break + + return normalized + + +def _paths_match( + normalized_target: str, + normalized_summary: str, + original_target: str, + original_summary: str, +) -> bool: + """Check if two file paths match using multiple strategies""" + + # Strategy 1: Exact normalized match + if normalized_target == normalized_summary: + return True + + # Strategy 2: Basename match (filename only) + target_basename = os.path.basename(original_target) + summary_basename = os.path.basename(original_summary) + if target_basename == summary_basename and len(target_basename) > 4: + return True + + # Strategy 3: Suffix match (remove common prefixes and compare) + target_suffix = _remove_common_prefixes(normalized_target) + summary_suffix = _remove_common_prefixes(normalized_summary) + if target_suffix == summary_suffix: + return True + + # Strategy 4: Ends with match + if normalized_target.endswith(normalized_summary) or normalized_summary.endswith( + normalized_target + ): + return True + + # Strategy 5: Contains match for longer paths + if len(normalized_target) > 10 and normalized_target in normalized_summary: + return True + if len(normalized_summary) > 10 and normalized_summary in normalized_target: + return True + + return False + + +def _remove_common_prefixes(file_path: str) -> str: + """Remove common prefixes from file path""" + prefixes_to_remove = ["src/", "core/", "./", "lib/", "main/"] + path = file_path + + for prefix in prefixes_to_remove: + if path.startswith(prefix): + path = path[len(prefix) :] + + return path + + +def _extract_file_section_alternative( + summary_content: str, target_file_path: str +) -> str: + """Alternative method to extract file section using simpler pattern matching""" + + # Get the basename for fallback matching + target_basename = os.path.basename(target_file_path) + + # Split by separator lines to get individual sections + sections = summary_content.split("=" * 80) + + for i, section in enumerate(sections): + if "## IMPLEMENTATION File" in section: + # Extract the file path from the header + lines = section.strip().split("\n") + for line in lines: + if "## IMPLEMENTATION File" in line: + # Extract file path between "File " and "; ROUND" + try: + file_part = line.split("File ")[1].split("; ROUND")[0].strip() + + # Check if this matches our target + if ( + _normalize_file_path(target_file_path) + == _normalize_file_path(file_part) + or target_basename == os.path.basename(file_part) + or target_file_path in file_part + or file_part.endswith(target_file_path) + ): + # Get the next section which contains the content + if i + 1 < len(sections): + content_section = sections[i + 1].strip() + return f"""================================================================================ +## IMPLEMENTATION File {file_part} +================================================================================ + +{content_section} + +--- +*Extracted from implement_code_summary.md using alternative method*""" + except (IndexError, AttributeError): + continue + + return None + + +# ==================== Code Search Tools ==================== + + +@mcp.tool() +async def search_code( + pattern: str, + file_pattern: str = "*.json", + use_regex: bool = False, + search_directory: str = None, +) -> str: + """ + Search patterns in code files + + Args: + pattern: Search pattern + file_pattern: File pattern (e.g., '*.py') + use_regex: Whether to use regular expressions + search_directory: Specify search directory (optional, uses WORKSPACE_DIR if not specified) + + Returns: + JSON string of search results + """ + try: + # Determine search directory + if search_directory: + # If search directory is specified, use the specified directory + if os.path.isabs(search_directory): + search_path = Path(search_directory) + else: + # Relative path, relative to current working directory + search_path = Path.cwd() / search_directory + else: + # ๅฆ‚ๆžœๆฒกๆœ‰ๆŒ‡ๅฎšSearch directory๏ผŒไฝฟ็”จ้ป˜่ฎค็š„WORKSPACE_DIR + ensure_workspace_exists() + search_path = WORKSPACE_DIR + + # ๆฃ€ๆŸฅSearch directoryๆ˜ฏๅฆๅญ˜ๅœจ + if not search_path.exists(): + result = { + "status": "error", + "message": f"Search directoryไธๅญ˜ๅœจ: {search_path}", + "pattern": pattern, + } + return json.dumps(result, ensure_ascii=False, indent=2) + + import glob + + # Get matching files + file_paths = glob.glob(str(search_path / "**" / file_pattern), recursive=True) + + matches = [] + total_files_searched = 0 + + for file_path in file_paths: + try: + with open(file_path, "r", encoding="utf-8") as f: + lines = f.readlines() + + total_files_searched += 1 + relative_path = os.path.relpath(file_path, search_path) + + for line_num, line in enumerate(lines, 1): + if use_regex: + if re.search(pattern, line): + matches.append( + { + "file": relative_path, + "line_number": line_num, + "line_content": line.strip(), + "match_type": "regex", + } + ) + else: + if pattern.lower() in line.lower(): + matches.append( + { + "file": relative_path, + "line_number": line_num, + "line_content": line.strip(), + "match_type": "substring", + } + ) + + except Exception as e: + logger.warning(f"Error searching file {file_path}: {e}") + continue + + result = { + "status": "success", + "pattern": pattern, + "file_pattern": file_pattern, + "use_regex": use_regex, + "search_directory": str(search_path), + "total_matches": len(matches), + "total_files_searched": total_files_searched, + "matches": matches[:50], # ้™ๅˆถ่ฟ”ๅ›žๅ‰50ไธชๅŒน้… + } + + if len(matches) > 50: + result["note"] = f"ๆ˜พ็คบๅ‰50ไธชๅŒน้…๏ผŒๆ€ปๅ…ฑๆ‰พๅˆฐ{len(matches)}ไธชๅŒน้…" + + log_operation( + "search_code", + { + "pattern": pattern, + "file_pattern": file_pattern, + "use_regex": use_regex, + "search_directory": str(search_path), + "total_matches": len(matches), + "files_searched": total_files_searched, + }, + ) + + return json.dumps(result, ensure_ascii=False, indent=2) + + except Exception as e: + result = { + "status": "error", + "message": f"Code search failed: {str(e)}", + "pattern": pattern, + } + log_operation("search_code_error", {"pattern": pattern, "error": str(e)}) + return json.dumps(result, ensure_ascii=False, indent=2) + + +# ==================== File Structure Tools ==================== + + +@mcp.tool() +async def get_file_structure(directory: str = ".", max_depth: int = 5) -> str: + """ + Get directory file structure + + Args: + directory: Directory path, relative to workspace + max_depth: ๆœ€ๅคง้ๅŽ†ๆทฑๅบฆ + + Returns: + JSON string of file structure + """ + try: + ensure_workspace_exists() + + if directory == ".": + target_dir = WORKSPACE_DIR + else: + target_dir = validate_path(directory) + + if not target_dir.exists(): + result = { + "status": "error", + "message": f"Directory does not exist: {directory}", + } + return json.dumps(result, ensure_ascii=False, indent=2) + + def scan_directory(path: Path, current_depth: int = 0) -> Dict[str, Any]: + """Recursively scan directory""" + if current_depth >= max_depth: + return {"type": "directory", "name": path.name, "truncated": True} + + items = [] + try: + for item in sorted(path.iterdir()): + relative_path = os.path.relpath(item, WORKSPACE_DIR) + + if item.is_file(): + file_info = { + "type": "file", + "name": item.name, + "path": relative_path, + "size_bytes": item.stat().st_size, + "extension": item.suffix, + } + items.append(file_info) + elif item.is_dir() and not item.name.startswith("."): + dir_info = scan_directory(item, current_depth + 1) + dir_info["path"] = relative_path + items.append(dir_info) + except PermissionError: + pass + + return { + "type": "directory", + "name": path.name, + "items": items, + "item_count": len(items), + } + + structure = scan_directory(target_dir) + + # ็ปŸ่ฎกไฟกๆฏ + def count_items(node): + if node["type"] == "file": + return {"files": 1, "directories": 0} + else: + counts = {"files": 0, "directories": 1} + for item in node.get("items", []): + item_counts = count_items(item) + counts["files"] += item_counts["files"] + counts["directories"] += item_counts["directories"] + return counts + + counts = count_items(structure) + + result = { + "status": "success", + "directory": directory, + "max_depth": max_depth, + "structure": structure, + "summary": { + "total_files": counts["files"], + "total_directories": counts["directories"] + - 1, # Exclude root directory + }, + } + + log_operation( + "get_file_structure", + { + "directory": directory, + "max_depth": max_depth, + "total_files": counts["files"], + "total_directories": counts["directories"] - 1, + }, + ) + + return json.dumps(result, ensure_ascii=False, indent=2) + + except Exception as e: + result = { + "status": "error", + "message": f"Failed to get file structure: {str(e)}", + "directory": directory, + } + log_operation( + "get_file_structure_error", {"directory": directory, "error": str(e)} + ) + return json.dumps(result, ensure_ascii=False, indent=2) + + +# ==================== Workspace Management Tools ==================== + + +@mcp.tool() +async def set_workspace(workspace_path: str) -> str: + """ + Set workspace directory + + Called by workflow to set workspace to: {plan_file_parent}/generate_code + This ensures all file operations are executed relative to the correct project directory + + Args: + workspace_path: Workspace path (Usually {plan_file_parent}/generate_code) + + Returns: + JSON string of operation result + """ + try: + global WORKSPACE_DIR + new_workspace = Path(workspace_path).resolve() + + # Create directory (if it does not exist) + new_workspace.mkdir(parents=True, exist_ok=True) + + old_workspace = WORKSPACE_DIR + WORKSPACE_DIR = new_workspace + + logger.info(f"New Workspace: {WORKSPACE_DIR}") + + result = { + "status": "success", + "message": f"Workspace setup successful: {workspace_path}", + "new_workspace": str(WORKSPACE_DIR), + } + + log_operation( + "set_workspace", + { + "old_workspace": str(old_workspace) if old_workspace else None, + "new_workspace": str(WORKSPACE_DIR), + "workspace_alignment": "plan_file_parent/generate_code", + }, + ) + + return json.dumps(result, ensure_ascii=False, indent=2) + + except Exception as e: + result = { + "status": "error", + "message": f"Failed to set workspace: {str(e)}", + "workspace_path": workspace_path, + } + log_operation( + "set_workspace_error", {"workspace_path": workspace_path, "error": str(e)} + ) + return json.dumps(result, ensure_ascii=False, indent=2) + + +@mcp.tool() +async def get_operation_history(last_n: int = 10) -> str: + """ + Get operation history + + Args: + last_n: Return the last N operations + + Returns: + JSON string of operation history + """ + try: + recent_history = ( + OPERATION_HISTORY[-last_n:] if last_n > 0 else OPERATION_HISTORY + ) + + result = { + "status": "success", + "total_operations": len(OPERATION_HISTORY), + "returned_operations": len(recent_history), + "workspace": str(WORKSPACE_DIR) if WORKSPACE_DIR else None, + "history": recent_history, + } + + return json.dumps(result, ensure_ascii=False, indent=2) + + except Exception as e: + result = { + "status": "error", + "message": f"Failed to get operation history: {str(e)}", + } + return json.dumps(result, ensure_ascii=False, indent=2) + + +# ==================== Server Initialization ==================== + + +def main(): + """Start MCP server""" + print("๐Ÿš€ Code Implementation MCP Server") + print( + "๐Ÿ“ Paper Code Implementation Tool Server / Paper Code Implementation Tool Server" + ) + print("") + print("Available tools / Available tools:") + # print(" โ€ข read_file - Read file contents / Read file contents") + print( + " โ€ข read_code_mem - Read code summary from implement_code_summary.md / Read code summary from implement_code_summary.md" + ) + print(" โ€ข write_file - Write file contents / Write file contents") + print(" โ€ข execute_python - Execute Python code / Execute Python code") + print(" โ€ข execute_bash - Execute bash command / Execute bash commands") + print(" โ€ข search_code - Search code patterns / Search code patterns") + print(" โ€ข get_file_structure - Get file structure / Get file structure") + print(" โ€ข set_workspace - Set workspace / Set workspace") + print(" โ€ข get_operation_history - Get operation history / Get operation history") + print("") + print("๐Ÿ”ง Server starting...") + + # Initialize default workspace + initialize_workspace() + + # Start server + mcp.run() + + +if __name__ == "__main__": + main() diff --git a/DeepCode/tools/code_indexer.py b/DeepCode/tools/code_indexer.py new file mode 100644 index 00000000..01bf06e6 --- /dev/null +++ b/DeepCode/tools/code_indexer.py @@ -0,0 +1,1646 @@ +""" +Code Indexer for Repository Analysis + +Analyzes code repositories to build comprehensive indexes for each subdirectory, +identifying file relationships and reusable components for implementation. + +Features: +- Recursive file traversal +- LLM-powered code similarity analysis using augmented LLM classes +- JSON-based relationship storage +- Configurable matching strategies +- Progress tracking and error handling +- Automatic LLM provider selection based on API key availability +""" + +import asyncio +import json +import logging +import os +import re +from datetime import datetime +from pathlib import Path +from dataclasses import dataclass, asdict +from typing import List, Dict, Any + +# MCP Agent imports for LLM +from utils.llm_utils import get_preferred_llm_class, get_default_models + + +@dataclass +class FileRelationship: + """Represents a relationship between a repo file and target structure file""" + + repo_file_path: str + target_file_path: str + relationship_type: str # 'direct_match', 'partial_match', 'reference', 'utility' + confidence_score: float # 0.0 to 1.0 + helpful_aspects: List[str] + potential_contributions: List[str] + usage_suggestions: str + + +@dataclass +class FileSummary: + """Summary information for a repository file""" + + file_path: str + file_type: str + main_functions: List[str] + key_concepts: List[str] + dependencies: List[str] + summary: str + lines_of_code: int + last_modified: str + + +@dataclass +class RepoIndex: + """Complete index for a repository""" + + repo_name: str + total_files: int + file_summaries: List[FileSummary] + relationships: List[FileRelationship] + analysis_metadata: Dict[str, Any] + + +class CodeIndexer: + """Main class for building code repository indexes""" + + def __init__( + self, + code_base_path: str = None, + target_structure: str = None, + output_dir: str = None, + config_path: str = "mcp_agent.secrets.yaml", + indexer_config_path: str = None, + enable_pre_filtering: bool = True, + ): + # Load configurations first + self.config_path = config_path + self.indexer_config_path = indexer_config_path + self.api_config = self._load_api_config() + self.indexer_config = self._load_indexer_config() + self.default_models = get_default_models("mcp_agent.config.yaml") + + # Use config paths if not provided as parameters + paths_config = self.indexer_config.get("paths", {}) + self.code_base_path = Path( + code_base_path or paths_config.get("code_base_path", "code_base") + ) + self.output_dir = Path(output_dir or paths_config.get("output_dir", "indexes")) + self.target_structure = ( + target_structure # This must be provided as it's project-specific + ) + self.enable_pre_filtering = enable_pre_filtering + + # LLM clients + self.llm_client = None + self.llm_client_type = None + + # Initialize logger early + self.logger = self._setup_logger() + + # Create output directory if it doesn't exist + self.output_dir.mkdir(parents=True, exist_ok=True) + + # Load file analysis configuration + file_analysis_config = self.indexer_config.get("file_analysis", {}) + self.supported_extensions = set( + file_analysis_config.get( + "supported_extensions", + [ + ".py", + ".js", + ".ts", + ".java", + ".cpp", + ".c", + ".h", + ".hpp", + ".cs", + ".php", + ".rb", + ".go", + ".rs", + ".scala", + ".kt", + ".swift", + ".m", + ".mm", + ".r", + ".matlab", + ".sql", + ".sh", + ".bat", + ".ps1", + ".yaml", + ".yml", + ".json", + ".xml", + ".toml", + ], + ) + ) + + self.skip_directories = set( + file_analysis_config.get( + "skip_directories", + [ + "__pycache__", + "node_modules", + "target", + "build", + "dist", + "venv", + "env", + ], + ) + ) + + self.max_file_size = file_analysis_config.get("max_file_size", 1048576) # 1MB + self.max_content_length = file_analysis_config.get("max_content_length", 3000) + + # Load LLM configuration + llm_config = self.indexer_config.get("llm", {}) + self.model_provider = llm_config.get("model_provider", "anthropic") + self.llm_max_tokens = llm_config.get("max_tokens", 4000) + self.llm_temperature = llm_config.get("temperature", 0.3) + self.llm_system_prompt = llm_config.get( + "system_prompt", + "You are a code analysis expert. Provide precise, structured analysis of code relationships and similarities.", + ) + self.request_delay = llm_config.get("request_delay", 0.1) + self.max_retries = llm_config.get("max_retries", 3) + self.retry_delay = llm_config.get("retry_delay", 1.0) + + # Load relationship configuration + relationship_config = self.indexer_config.get("relationships", {}) + self.min_confidence_score = relationship_config.get("min_confidence_score", 0.3) + self.high_confidence_threshold = relationship_config.get( + "high_confidence_threshold", 0.7 + ) + self.relationship_types = relationship_config.get( + "relationship_types", + { + "direct_match": 1.0, + "partial_match": 0.8, + "reference": 0.6, + "utility": 0.4, + }, + ) + + # Load performance configuration + performance_config = self.indexer_config.get("performance", {}) + self.enable_concurrent_analysis = performance_config.get( + "enable_concurrent_analysis", False + ) + self.max_concurrent_files = performance_config.get("max_concurrent_files", 5) + self.enable_content_caching = performance_config.get( + "enable_content_caching", False + ) + self.max_cache_size = performance_config.get("max_cache_size", 100) + + # Load debug configuration + debug_config = self.indexer_config.get("debug", {}) + self.save_raw_responses = debug_config.get("save_raw_responses", False) + self.raw_responses_dir = debug_config.get( + "raw_responses_dir", "debug_responses" + ) + self.verbose_output = debug_config.get("verbose_output", False) + self.mock_llm_responses = debug_config.get("mock_llm_responses", False) + + # Load output configuration + output_config = self.indexer_config.get("output", {}) + self.generate_summary = output_config.get("generate_summary", True) + self.generate_statistics = output_config.get("generate_statistics", True) + self.include_metadata = output_config.get("include_metadata", True) + self.index_filename_pattern = output_config.get( + "index_filename_pattern", "{repo_name}_index.json" + ) + self.summary_filename = output_config.get( + "summary_filename", "indexing_summary.json" + ) + self.stats_filename = output_config.get( + "stats_filename", "indexing_statistics.json" + ) + + # Initialize caching if enabled + self.content_cache = {} if self.enable_content_caching else None + + # Create debug directory if needed + if self.save_raw_responses: + Path(self.raw_responses_dir).mkdir(parents=True, exist_ok=True) + + # Debug logging + if self.verbose_output: + self.logger.info( + f"Initialized CodeIndexer with config: {self.indexer_config_path}" + ) + self.logger.info(f"Code base path: {self.code_base_path}") + self.logger.info(f"Output directory: {self.output_dir}") + self.logger.info(f"Model provider: {self.model_provider}") + self.logger.info(f"Concurrent analysis: {self.enable_concurrent_analysis}") + self.logger.info(f"Content caching: {self.enable_content_caching}") + self.logger.info(f"Mock LLM responses: {self.mock_llm_responses}") + + def _setup_logger(self) -> logging.Logger: + """Setup logging configuration from config file""" + logger = logging.getLogger("CodeIndexer") + + # Get logging config + logging_config = self.indexer_config.get("logging", {}) + log_level = logging_config.get("level", "INFO") + log_format = logging_config.get( + "log_format", "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + + logger.setLevel(getattr(logging, log_level.upper(), logging.INFO)) + + # Clear existing handlers + logger.handlers.clear() + + # Console handler + handler = logging.StreamHandler() + formatter = logging.Formatter(log_format) + handler.setFormatter(formatter) + logger.addHandler(handler) + + # File handler if enabled + if logging_config.get("log_to_file", False): + log_file = logging_config.get("log_file", "indexer.log") + file_handler = logging.FileHandler(log_file, encoding="utf-8") + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + + return logger + + def _load_api_config(self) -> Dict[str, Any]: + """Load API configuration from YAML file""" + try: + import yaml + + with open(self.config_path, "r", encoding="utf-8") as f: + return yaml.safe_load(f) + except Exception as e: + # Create a basic logger for this error since self.logger doesn't exist yet + print(f"Warning: Failed to load API config from {self.config_path}: {e}") + return {} + + def _load_indexer_config(self) -> Dict[str, Any]: + """Load indexer configuration from YAML file""" + try: + import yaml + + with open(self.indexer_config_path, "r", encoding="utf-8") as f: + config = yaml.safe_load(f) + if config is None: + config = {} + return config + except Exception as e: + print( + f"Warning: Failed to load indexer config from {self.indexer_config_path}: {e}" + ) + print("Using default configuration values") + return {} + + async def _initialize_llm_client(self): + """Initialize LLM client (Anthropic or OpenAI) based on API key availability""" + if self.llm_client is not None: + return self.llm_client, self.llm_client_type + + # Check if mock responses are enabled + if self.mock_llm_responses: + self.logger.info("Using mock LLM responses for testing") + self.llm_client = "mock" + self.llm_client_type = "mock" + return "mock", "mock" + + # Check which API has available key and try that first + anthropic_key = self.api_config.get("anthropic", {}).get("api_key", "") + openai_key = self.api_config.get("openai", {}).get("api_key", "") + + # Try Anthropic API first if key is available + if anthropic_key and anthropic_key.strip(): + try: + from anthropic import AsyncAnthropic + + client = AsyncAnthropic(api_key=anthropic_key) + # Test connection with default model from config + await client.messages.create( + model=self.default_models["anthropic"], + max_tokens=10, + messages=[{"role": "user", "content": "test"}], + ) + self.logger.info( + f"Using Anthropic API with model: {self.default_models['anthropic']}" + ) + self.llm_client = client + self.llm_client_type = "anthropic" + return client, "anthropic" + except Exception as e: + self.logger.warning(f"Anthropic API unavailable: {e}") + + # Try OpenAI API if Anthropic failed or key not available + if openai_key and openai_key.strip(): + try: + from openai import AsyncOpenAI + + # Handle custom base_url if specified + openai_config = self.api_config.get("openai", {}) + base_url = openai_config.get("base_url") + + if base_url: + client = AsyncOpenAI(api_key=openai_key, base_url=base_url) + else: + client = AsyncOpenAI(api_key=openai_key) + + # Test connection with default model from config + await client.chat.completions.create( + model=self.default_models["openai"], + max_tokens=10, + messages=[{"role": "user", "content": "test"}], + ) + self.logger.info( + f"Using OpenAI API with model: {self.default_models['openai']}" + ) + if base_url: + self.logger.info(f"Using custom base URL: {base_url}") + self.llm_client = client + self.llm_client_type = "openai" + return client, "openai" + except Exception as e: + self.logger.warning(f"OpenAI API unavailable: {e}") + + raise ValueError( + "No available LLM API - please check your API keys in configuration" + ) + + async def _call_llm( + self, prompt: str, system_prompt: str = None, max_tokens: int = None + ) -> str: + """Call LLM for code analysis with retry mechanism and debugging support""" + if system_prompt is None: + system_prompt = self.llm_system_prompt + if max_tokens is None: + max_tokens = self.llm_max_tokens + + # Mock response for testing + if self.mock_llm_responses: + mock_response = self._generate_mock_response(prompt) + if self.save_raw_responses: + self._save_debug_response("mock", prompt, mock_response) + return mock_response + + last_error = None + + # Retry mechanism + for attempt in range(self.max_retries): + try: + if self.verbose_output and attempt > 0: + self.logger.info( + f"LLM call attempt {attempt + 1}/{self.max_retries}" + ) + + client, client_type = await self._initialize_llm_client() + + if client_type == "anthropic": + response = await client.messages.create( + model=self.default_models["anthropic"], + system=system_prompt, + messages=[{"role": "user", "content": prompt}], + max_tokens=max_tokens, + temperature=self.llm_temperature, + ) + + content = "" + for block in response.content: + if block.type == "text": + content += block.text + + # Save debug response if enabled + if self.save_raw_responses: + self._save_debug_response("anthropic", prompt, content) + + return content + + elif client_type == "openai": + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": prompt}, + ] + + response = await client.chat.completions.create( + model=self.default_models["openai"], + messages=messages, + max_tokens=max_tokens, + temperature=self.llm_temperature, + ) + + content = response.choices[0].message.content or "" + + # Save debug response if enabled + if self.save_raw_responses: + self._save_debug_response("openai", prompt, content) + + return content + else: + raise ValueError(f"Unsupported client type: {client_type}") + + except Exception as e: + last_error = e + self.logger.warning(f"LLM call attempt {attempt + 1} failed: {e}") + + if attempt < self.max_retries - 1: + await asyncio.sleep( + self.retry_delay * (attempt + 1) + ) # Exponential backoff + + # All retries failed + error_msg = f"LLM call failed after {self.max_retries} attempts. Last error: {str(last_error)}" + self.logger.error(error_msg) + return f"Error in LLM analysis: {error_msg}" + + def _generate_mock_response(self, prompt: str) -> str: + """Generate mock LLM response for testing""" + if "JSON format" in prompt and "file_type" in prompt: + # File analysis mock + return """ + { + "file_type": "Python module", + "main_functions": ["main_function", "helper_function"], + "key_concepts": ["data_processing", "algorithm"], + "dependencies": ["numpy", "pandas"], + "summary": "Mock analysis of code file functionality." + } + """ + elif "relationships" in prompt: + # Relationship analysis mock + return """ + { + "relationships": [ + { + "target_file_path": "src/core/mock.py", + "relationship_type": "partial_match", + "confidence_score": 0.8, + "helpful_aspects": ["algorithm implementation", "data structures"], + "potential_contributions": ["core functionality", "utility methods"], + "usage_suggestions": "Mock relationship suggestion for testing." + } + ] + } + """ + elif "relevant_files" in prompt: + # File filtering mock + return """ + { + "relevant_files": [ + { + "file_path": "mock_file.py", + "relevance_reason": "Mock relevance reason", + "confidence": 0.9, + "expected_contribution": "Mock contribution" + } + ], + "summary": { + "total_files_analyzed": "10", + "relevant_files_count": "1", + "filtering_strategy": "Mock filtering strategy" + } + } + """ + else: + return "Mock LLM response for testing purposes." + + def _save_debug_response(self, provider: str, prompt: str, response: str): + """Save LLM response for debugging""" + try: + import hashlib + from datetime import datetime + + # Create a hash of the prompt for filename + prompt_hash = hashlib.md5(prompt.encode()).hexdigest()[:8] + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"{provider}_{timestamp}_{prompt_hash}.json" + + debug_data = { + "timestamp": datetime.now().isoformat(), + "provider": provider, + "prompt": prompt[:500] + "..." if len(prompt) > 500 else prompt, + "response": response, + "full_prompt_length": len(prompt), + } + + debug_file = Path(self.raw_responses_dir) / filename + with open(debug_file, "w", encoding="utf-8") as f: + json.dump(debug_data, f, indent=2, ensure_ascii=False) + + except Exception as e: + self.logger.warning(f"Failed to save debug response: {e}") + + def get_all_repo_files(self, repo_path: Path) -> List[Path]: + """Recursively get all supported files in a repository""" + files = [] + + try: + for root, dirs, filenames in os.walk(repo_path): + # Skip common non-code directories + dirs[:] = [ + d + for d in dirs + if not d.startswith(".") and d not in self.skip_directories + ] + + for filename in filenames: + file_path = Path(root) / filename + if file_path.suffix.lower() in self.supported_extensions: + files.append(file_path) + + except Exception as e: + self.logger.error(f"Error traversing {repo_path}: {e}") + + return files + + def generate_file_tree(self, repo_path: Path, max_depth: int = 5) -> str: + """Generate file tree structure string for the repository""" + tree_lines = [] + + def add_to_tree(current_path: Path, prefix: str = "", depth: int = 0): + if depth > max_depth: + return + + try: + items = sorted( + current_path.iterdir(), key=lambda x: (x.is_file(), x.name.lower()) + ) + # Filter out irrelevant directories and files + items = [ + item + for item in items + if not item.name.startswith(".") + and item.name not in self.skip_directories + ] + + for i, item in enumerate(items): + is_last = i == len(items) - 1 + current_prefix = "โ””โ”€โ”€ " if is_last else "โ”œโ”€โ”€ " + tree_lines.append(f"{prefix}{current_prefix}{item.name}") + + if item.is_dir(): + extension_prefix = " " if is_last else "โ”‚ " + add_to_tree(item, prefix + extension_prefix, depth + 1) + elif item.suffix.lower() in self.supported_extensions: + # Add file size information + try: + size = item.stat().st_size + if size > 1024: + size_str = f" ({size // 1024}KB)" + else: + size_str = f" ({size}B)" + tree_lines[-1] += size_str + except (OSError, PermissionError): + pass + + except PermissionError: + tree_lines.append(f"{prefix}โ”œโ”€โ”€ [Permission Denied]") + except Exception as e: + tree_lines.append(f"{prefix}โ”œโ”€โ”€ [Error: {str(e)}]") + + tree_lines.append(f"{repo_path.name}/") + add_to_tree(repo_path) + return "\n".join(tree_lines) + + async def pre_filter_files(self, repo_path: Path, file_tree: str) -> List[str]: + """Use LLM to pre-filter relevant files based on target structure""" + filter_prompt = f""" + You are a code analysis expert. Please analyze the following code repository file tree based on the target project structure and filter out files that may be relevant to the target project. + + Target Project Structure: + {self.target_structure} + + Code Repository File Tree: + {file_tree} + + Please analyze which files might be helpful for implementing the target project structure, including: + - Core algorithm implementation files (such as GCN, recommendation systems, graph neural networks, etc.) + - Data processing and preprocessing files + - Loss functions and evaluation metric files + - Configuration and utility files + - Test files + - Documentation files + + Please return the filtering results in JSON format: + {{ + "relevant_files": [ + {{ + "file_path": "file path relative to repository root", + "relevance_reason": "why this file is relevant", + "confidence": 0.0-1.0, + "expected_contribution": "expected contribution to the target project" + }} + ], + "summary": {{ + "total_files_analyzed": "total number of files analyzed", + "relevant_files_count": "number of relevant files", + "filtering_strategy": "explanation of filtering strategy" + }} + }} + + Only return files with confidence > {self.min_confidence_score}. Focus on files related to recommendation systems, graph neural networks, and diffusion models. + """ + + try: + self.logger.info("Starting LLM pre-filtering of files...") + llm_response = await self._call_llm( + filter_prompt, + system_prompt="You are a professional code analysis and project architecture expert, skilled at identifying code file functionality and relevance.", + max_tokens=2000, + ) + + # Parse JSON response + match = re.search(r"\{.*\}", llm_response, re.DOTALL) + if not match: + self.logger.warning( + "Unable to parse LLM filtering response, will use all files" + ) + return [] + + filter_data = json.loads(match.group(0)) + relevant_files = filter_data.get("relevant_files", []) + + # Extract file paths + selected_files = [] + for file_info in relevant_files: + file_path = file_info.get("file_path", "") + confidence = file_info.get("confidence", 0.0) + # Use configured minimum confidence threshold + if file_path and confidence > self.min_confidence_score: + selected_files.append(file_path) + + summary = filter_data.get("summary", {}) + self.logger.info( + f"LLM filtering completed: {summary.get('relevant_files_count', len(selected_files))} relevant files selected" + ) + self.logger.info( + f"Filtering strategy: {summary.get('filtering_strategy', 'Not provided')}" + ) + + return selected_files + + except Exception as e: + self.logger.error(f"LLM pre-filtering failed: {e}") + self.logger.info("Will fallback to analyzing all files") + return [] + + def filter_files_by_paths( + self, all_files: List[Path], selected_paths: List[str], repo_path: Path + ) -> List[Path]: + """Filter file list based on LLM-selected paths""" + if not selected_paths: + return all_files + + filtered_files = [] + + for file_path in all_files: + # Get path relative to repository root + relative_path = str(file_path.relative_to(repo_path)) + + # Check if it's in the selected list + for selected_path in selected_paths: + # Normalize path comparison + if ( + relative_path == selected_path + or relative_path.replace("\\", "/") + == selected_path.replace("\\", "/") + or selected_path in relative_path + or relative_path in selected_path + ): + filtered_files.append(file_path) + break + + return filtered_files + + def _get_cache_key(self, file_path: Path) -> str: + """Generate cache key for file content""" + try: + stats = file_path.stat() + return f"{file_path}:{stats.st_mtime}:{stats.st_size}" + except (OSError, PermissionError): + return str(file_path) + + def _manage_cache_size(self): + """Manage cache size to stay within limits""" + if not self.enable_content_caching or not self.content_cache: + return + + if len(self.content_cache) > self.max_cache_size: + # Remove oldest entries (simple FIFO strategy) + excess_count = len(self.content_cache) - self.max_cache_size + 10 + keys_to_remove = list(self.content_cache.keys())[:excess_count] + + for key in keys_to_remove: + del self.content_cache[key] + + if self.verbose_output: + self.logger.info( + f"Cache cleaned: removed {excess_count} entries, {len(self.content_cache)} entries remaining" + ) + + async def analyze_file_content(self, file_path: Path) -> FileSummary: + """Analyze a single file and create summary with caching support""" + try: + # Check file size before reading + file_size = file_path.stat().st_size + if file_size > self.max_file_size: + self.logger.warning( + f"Skipping file {file_path} - size {file_size} bytes exceeds limit {self.max_file_size}" + ) + return FileSummary( + file_path=str(file_path.relative_to(self.code_base_path)), + file_type="skipped - too large", + main_functions=[], + key_concepts=[], + dependencies=[], + summary=f"File skipped - size {file_size} bytes exceeds {self.max_file_size} byte limit", + lines_of_code=0, + last_modified=datetime.fromtimestamp( + file_path.stat().st_mtime + ).isoformat(), + ) + + # Check cache if enabled + cache_key = None + if self.enable_content_caching: + cache_key = self._get_cache_key(file_path) + if cache_key in self.content_cache: + if self.verbose_output: + self.logger.info(f"Using cached analysis for {file_path.name}") + return self.content_cache[cache_key] + + with open(file_path, "r", encoding="utf-8", errors="ignore") as f: + content = f.read() + + # Get file stats + stats = file_path.stat() + lines_of_code = len([line for line in content.split("\n") if line.strip()]) + + # Truncate content based on config + content_for_analysis = content[: self.max_content_length] + content_suffix = "..." if len(content) > self.max_content_length else "" + + # Create analysis prompt + analysis_prompt = f""" + Analyze this code file and provide a structured summary: + + File: {file_path.name} + Content: + ``` + {content_for_analysis}{content_suffix} + ``` + + Please provide analysis in this JSON format: + {{ + "file_type": "description of what type of file this is", + "main_functions": ["list", "of", "main", "functions", "or", "classes"], + "key_concepts": ["important", "concepts", "algorithms", "patterns"], + "dependencies": ["external", "libraries", "or", "imports"], + "summary": "2-3 sentence summary of what this file does" + }} + + Focus on the core functionality and potential reusability. + """ + + # Get LLM analysis with configured parameters + llm_response = await self._call_llm(analysis_prompt, max_tokens=1000) + + try: + # Try to parse JSON response + match = re.search(r"\{.*\}", llm_response, re.DOTALL) + analysis_data = json.loads(match.group(0)) + except json.JSONDecodeError: + # Fallback to basic analysis if JSON parsing fails + analysis_data = { + "file_type": f"{file_path.suffix} file", + "main_functions": [], + "key_concepts": [], + "dependencies": [], + "summary": "File analysis failed - JSON parsing error", + } + + file_summary = FileSummary( + file_path=str(file_path.relative_to(self.code_base_path)), + file_type=analysis_data.get("file_type", "unknown"), + main_functions=analysis_data.get("main_functions", []), + key_concepts=analysis_data.get("key_concepts", []), + dependencies=analysis_data.get("dependencies", []), + summary=analysis_data.get("summary", "No summary available"), + lines_of_code=lines_of_code, + last_modified=datetime.fromtimestamp(stats.st_mtime).isoformat(), + ) + + # Cache the result if caching is enabled + if self.enable_content_caching and cache_key: + self.content_cache[cache_key] = file_summary + self._manage_cache_size() + + return file_summary + + except Exception as e: + self.logger.error(f"Error analyzing file {file_path}: {e}") + return FileSummary( + file_path=str(file_path.relative_to(self.code_base_path)), + file_type="error", + main_functions=[], + key_concepts=[], + dependencies=[], + summary=f"Analysis failed: {str(e)}", + lines_of_code=0, + last_modified="", + ) + + async def find_relationships( + self, file_summary: FileSummary + ) -> List[FileRelationship]: + """Find relationships between a repo file and target structure""" + + # Build relationship type description from config + relationship_type_desc = [] + for rel_type, weight in self.relationship_types.items(): + relationship_type_desc.append(f"- {rel_type} (priority: {weight})") + + relationship_prompt = f""" + Analyze the relationship between this existing code file and the target project structure. + + Existing File Analysis: + - Path: {file_summary.file_path} + - Type: {file_summary.file_type} + - Functions: {', '.join(file_summary.main_functions)} + - Concepts: {', '.join(file_summary.key_concepts)} + - Summary: {file_summary.summary} + + Target Project Structure: + {self.target_structure} + + Available relationship types (with priority weights): + {chr(10).join(relationship_type_desc)} + + Identify potential relationships and provide analysis in this JSON format: + {{ + "relationships": [ + {{ + "target_file_path": "path/in/target/structure", + "relationship_type": "direct_match|partial_match|reference|utility", + "confidence_score": 0.0-1.0, + "helpful_aspects": ["specific", "aspects", "that", "could", "help"], + "potential_contributions": ["how", "this", "could", "contribute"], + "usage_suggestions": "detailed suggestion on how to use this file" + }} + ] + }} + + Consider the priority weights when determining relationship types. Higher weight types should be preferred when multiple types apply. + Only include relationships with confidence > {self.min_confidence_score}. Focus on concrete, actionable connections. + """ + + try: + llm_response = await self._call_llm(relationship_prompt, max_tokens=1500) + + match = re.search(r"\{.*\}", llm_response, re.DOTALL) + relationship_data = json.loads(match.group(0)) + + relationships = [] + for rel_data in relationship_data.get("relationships", []): + confidence_score = float(rel_data.get("confidence_score", 0.0)) + relationship_type = rel_data.get("relationship_type", "reference") + + # Validate relationship type is in config + if relationship_type not in self.relationship_types: + if self.verbose_output: + self.logger.warning( + f"Unknown relationship type '{relationship_type}', using 'reference'" + ) + relationship_type = "reference" + + # Apply configured minimum confidence filter + if confidence_score > self.min_confidence_score: + relationship = FileRelationship( + repo_file_path=file_summary.file_path, + target_file_path=rel_data.get("target_file_path", ""), + relationship_type=relationship_type, + confidence_score=confidence_score, + helpful_aspects=rel_data.get("helpful_aspects", []), + potential_contributions=rel_data.get( + "potential_contributions", [] + ), + usage_suggestions=rel_data.get("usage_suggestions", ""), + ) + relationships.append(relationship) + + return relationships + + except Exception as e: + self.logger.error( + f"Error finding relationships for {file_summary.file_path}: {e}" + ) + return [] + + async def _analyze_single_file_with_relationships( + self, file_path: Path, index: int, total: int + ) -> tuple: + """Analyze a single file and its relationships (for concurrent processing)""" + if self.verbose_output: + self.logger.info(f"Analyzing file {index}/{total}: {file_path.name}") + + # Get file summary + file_summary = await self.analyze_file_content(file_path) + + # Find relationships + relationships = await self.find_relationships(file_summary) + + return file_summary, relationships + + async def process_repository(self, repo_path: Path) -> RepoIndex: + """Process a single repository and create complete index with optional concurrent processing""" + repo_name = repo_path.name + self.logger.info(f"Processing repository: {repo_name}") + + # Step 1: Generate file tree + self.logger.info("Generating file tree structure...") + file_tree = self.generate_file_tree(repo_path) + + # Step 2: Get all files + all_files = self.get_all_repo_files(repo_path) + self.logger.info(f"Found {len(all_files)} files in {repo_name}") + + # Step 3: LLM pre-filtering of relevant files + if self.enable_pre_filtering: + self.logger.info("Using LLM for file pre-filtering...") + selected_file_paths = await self.pre_filter_files(repo_path, file_tree) + else: + self.logger.info("Pre-filtering is disabled, will analyze all files") + selected_file_paths = [] + + # Step 4: Filter file list based on filtering results + if selected_file_paths: + files_to_analyze = self.filter_files_by_paths( + all_files, selected_file_paths, repo_path + ) + self.logger.info( + f"After LLM filtering, will analyze {len(files_to_analyze)} relevant files (from {len(all_files)} total)" + ) + else: + files_to_analyze = all_files + self.logger.info("LLM filtering failed, will analyze all files") + + # Step 5: Analyze filtered files (concurrent or sequential) + if self.enable_concurrent_analysis and len(files_to_analyze) > 1: + self.logger.info( + f"Using concurrent analysis with max {self.max_concurrent_files} parallel files" + ) + file_summaries, all_relationships = await self._process_files_concurrently( + files_to_analyze + ) + else: + self.logger.info("Using sequential file analysis") + file_summaries, all_relationships = await self._process_files_sequentially( + files_to_analyze + ) + + # Step 6: Create repository index + repo_index = RepoIndex( + repo_name=repo_name, + total_files=len(all_files), # Record original file count + file_summaries=file_summaries, + relationships=all_relationships, + analysis_metadata={ + "analysis_date": datetime.now().isoformat(), + "target_structure_analyzed": self.target_structure[:200] + "...", + "total_relationships_found": len(all_relationships), + "high_confidence_relationships": len( + [ + r + for r in all_relationships + if r.confidence_score > self.high_confidence_threshold + ] + ), + "analyzer_version": "1.4.0", # Updated version to reflect augmented LLM support + "pre_filtering_enabled": self.enable_pre_filtering, + "files_before_filtering": len(all_files), + "files_after_filtering": len(files_to_analyze), + "filtering_efficiency": round( + (1 - len(files_to_analyze) / len(all_files)) * 100, 2 + ) + if all_files + else 0, + "config_file_used": self.indexer_config_path, + "min_confidence_score": self.min_confidence_score, + "high_confidence_threshold": self.high_confidence_threshold, + "concurrent_analysis_used": self.enable_concurrent_analysis, + "content_caching_enabled": self.enable_content_caching, + "cache_hits": len(self.content_cache) if self.content_cache else 0, + }, + ) + + return repo_index + + async def _process_files_sequentially(self, files_to_analyze: list) -> tuple: + """Process files sequentially (original method)""" + file_summaries = [] + all_relationships = [] + + for i, file_path in enumerate(files_to_analyze, 1): + ( + file_summary, + relationships, + ) = await self._analyze_single_file_with_relationships( + file_path, i, len(files_to_analyze) + ) + file_summaries.append(file_summary) + all_relationships.extend(relationships) + + # Add configured delay to avoid overwhelming the LLM API + await asyncio.sleep(self.request_delay) + + return file_summaries, all_relationships + + async def _process_files_concurrently(self, files_to_analyze: list) -> tuple: + """Process files concurrently with semaphore limiting""" + file_summaries = [] + all_relationships = [] + + # Create semaphore to limit concurrent tasks + semaphore = asyncio.Semaphore(self.max_concurrent_files) + tasks = [] + + async def _process_with_semaphore(file_path: Path, index: int, total: int): + async with semaphore: + # Add a small delay to space out concurrent requests + if index > 1: + await asyncio.sleep( + self.request_delay * 0.5 + ) # Reduced delay for concurrent processing + return await self._analyze_single_file_with_relationships( + file_path, index, total + ) + + try: + # Create tasks for all files + tasks = [ + _process_with_semaphore(file_path, i, len(files_to_analyze)) + for i, file_path in enumerate(files_to_analyze, 1) + ] + + # Process tasks and collect results + if self.verbose_output: + self.logger.info( + f"Starting concurrent analysis of {len(tasks)} files..." + ) + + try: + results = await asyncio.gather(*tasks, return_exceptions=True) + + for i, result in enumerate(results): + if isinstance(result, Exception): + self.logger.error( + f"Failed to analyze file {files_to_analyze[i]}: {result}" + ) + # Create error summary + error_summary = FileSummary( + file_path=str( + files_to_analyze[i].relative_to(self.code_base_path) + ), + file_type="error", + main_functions=[], + key_concepts=[], + dependencies=[], + summary=f"Concurrent analysis failed: {str(result)}", + lines_of_code=0, + last_modified="", + ) + file_summaries.append(error_summary) + else: + file_summary, relationships = result + file_summaries.append(file_summary) + all_relationships.extend(relationships) + + except Exception as e: + self.logger.error(f"Concurrent processing failed: {e}") + # Cancel any remaining tasks + for task in tasks: + if not task.done() and not task.cancelled(): + task.cancel() + + # Wait for cancelled tasks to complete + try: + await asyncio.sleep(0.1) # Brief wait for cancellation + except Exception: + pass + + # Fallback to sequential processing + self.logger.info("Falling back to sequential processing...") + return await self._process_files_sequentially(files_to_analyze) + + if self.verbose_output: + self.logger.info( + f"Concurrent analysis completed: {len(file_summaries)} files processed" + ) + + return file_summaries, all_relationships + + except Exception as e: + # Ensure all tasks are cancelled in case of unexpected errors + if tasks: + for task in tasks: + if not task.done() and not task.cancelled(): + task.cancel() + + # Wait briefly for cancellation to complete + try: + await asyncio.sleep(0.1) + except Exception: + pass + + self.logger.error(f"Critical error in concurrent processing: {e}") + # Fallback to sequential processing + self.logger.info( + "Falling back to sequential processing due to critical error..." + ) + return await self._process_files_sequentially(files_to_analyze) + + finally: + # Final cleanup: ensure all tasks are properly finished + if tasks: + for task in tasks: + if not task.done() and not task.cancelled(): + task.cancel() + + # Clear task references to help with garbage collection + tasks.clear() + + # Force garbage collection to help clean up semaphore and related resources + import gc + + gc.collect() + + async def build_all_indexes(self) -> Dict[str, str]: + """Build indexes for all repositories in code_base""" + if not self.code_base_path.exists(): + raise FileNotFoundError( + f"Code base path does not exist: {self.code_base_path}" + ) + + # Get all repository directories + repo_dirs = [ + d + for d in self.code_base_path.iterdir() + if d.is_dir() and not d.name.startswith(".") + ] + + if not repo_dirs: + raise ValueError(f"No repositories found in {self.code_base_path}") + + self.logger.info(f"Found {len(repo_dirs)} repositories to process") + + # Process each repository + output_files = {} + statistics_data = [] + + for repo_dir in repo_dirs: + try: + # Process repository + repo_index = await self.process_repository(repo_dir) + + # Generate output filename using configured pattern + output_filename = self.index_filename_pattern.format( + repo_name=repo_index.repo_name + ) + output_file = self.output_dir / output_filename + + # Get output configuration + output_config = self.indexer_config.get("output", {}) + json_indent = output_config.get("json_indent", 2) + ensure_ascii = not output_config.get("ensure_ascii", False) + + # Save to JSON file + with open(output_file, "w", encoding="utf-8") as f: + if self.include_metadata: + json.dump( + asdict(repo_index), + f, + indent=json_indent, + ensure_ascii=ensure_ascii, + ) + else: + # Save without metadata if disabled + index_data = asdict(repo_index) + index_data.pop("analysis_metadata", None) + json.dump( + index_data, f, indent=json_indent, ensure_ascii=ensure_ascii + ) + + output_files[repo_index.repo_name] = str(output_file) + self.logger.info( + f"Saved index for {repo_index.repo_name} to {output_file}" + ) + + # Collect statistics for report + if self.generate_statistics: + stats = self._extract_repository_statistics(repo_index) + statistics_data.append(stats) + + except Exception as e: + self.logger.error(f"Failed to process repository {repo_dir.name}: {e}") + continue + + # Generate additional reports if configured + if self.generate_summary: + summary_path = self.generate_summary_report(output_files) + self.logger.info(f"Generated summary report: {summary_path}") + + if self.generate_statistics: + stats_path = self.generate_statistics_report(statistics_data) + self.logger.info(f"Generated statistics report: {stats_path}") + + return output_files + + def _extract_repository_statistics(self, repo_index: RepoIndex) -> Dict[str, Any]: + """Extract statistical information from a repository index""" + metadata = repo_index.analysis_metadata + + # Count relationship types + relationship_type_counts = {} + for rel in repo_index.relationships: + rel_type = rel.relationship_type + relationship_type_counts[rel_type] = ( + relationship_type_counts.get(rel_type, 0) + 1 + ) + + # Count file types + file_type_counts = {} + for file_summary in repo_index.file_summaries: + file_type = file_summary.file_type + file_type_counts[file_type] = file_type_counts.get(file_type, 0) + 1 + + # Calculate statistics + total_lines = sum(fs.lines_of_code for fs in repo_index.file_summaries) + avg_lines = ( + total_lines / len(repo_index.file_summaries) + if repo_index.file_summaries + else 0 + ) + + avg_confidence = ( + sum(r.confidence_score for r in repo_index.relationships) + / len(repo_index.relationships) + if repo_index.relationships + else 0 + ) + + return { + "repo_name": repo_index.repo_name, + "total_files": repo_index.total_files, + "analyzed_files": len(repo_index.file_summaries), + "total_relationships": len(repo_index.relationships), + "high_confidence_relationships": metadata.get( + "high_confidence_relationships", 0 + ), + "relationship_type_counts": relationship_type_counts, + "file_type_counts": file_type_counts, + "total_lines_of_code": total_lines, + "average_lines_per_file": round(avg_lines, 2), + "average_confidence_score": round(avg_confidence, 3), + "filtering_efficiency": metadata.get("filtering_efficiency", 0), + "concurrent_analysis_used": metadata.get("concurrent_analysis_used", False), + "cache_hits": metadata.get("cache_hits", 0), + "analysis_date": metadata.get("analysis_date", "unknown"), + } + + def generate_statistics_report(self, statistics_data: List[Dict[str, Any]]) -> str: + """Generate a detailed statistics report""" + stats_path = self.output_dir / self.stats_filename + + # Calculate aggregate statistics + total_repos = len(statistics_data) + total_files_analyzed = sum(stat["analyzed_files"] for stat in statistics_data) + total_relationships = sum( + stat["total_relationships"] for stat in statistics_data + ) + total_lines = sum(stat["total_lines_of_code"] for stat in statistics_data) + + # Aggregate relationship types + aggregated_rel_types = {} + for stat in statistics_data: + for rel_type, count in stat["relationship_type_counts"].items(): + aggregated_rel_types[rel_type] = ( + aggregated_rel_types.get(rel_type, 0) + count + ) + + # Aggregate file types + aggregated_file_types = {} + for stat in statistics_data: + for file_type, count in stat["file_type_counts"].items(): + aggregated_file_types[file_type] = ( + aggregated_file_types.get(file_type, 0) + count + ) + + # Calculate averages + avg_files_per_repo = total_files_analyzed / total_repos if total_repos else 0 + avg_relationships_per_repo = ( + total_relationships / total_repos if total_repos else 0 + ) + avg_lines_per_repo = total_lines / total_repos if total_repos else 0 + + # Build statistics report + statistics_report = { + "report_generation_time": datetime.now().isoformat(), + "analyzer_version": "1.4.0", + "configuration_used": { + "config_file": self.indexer_config_path, + "concurrent_analysis_enabled": self.enable_concurrent_analysis, + "content_caching_enabled": self.enable_content_caching, + "pre_filtering_enabled": self.enable_pre_filtering, + "min_confidence_score": self.min_confidence_score, + "high_confidence_threshold": self.high_confidence_threshold, + }, + "aggregate_statistics": { + "total_repositories_processed": total_repos, + "total_files_analyzed": total_files_analyzed, + "total_relationships_found": total_relationships, + "total_lines_of_code": total_lines, + "average_files_per_repository": round(avg_files_per_repo, 2), + "average_relationships_per_repository": round( + avg_relationships_per_repo, 2 + ), + "average_lines_per_repository": round(avg_lines_per_repo, 2), + }, + "relationship_type_distribution": aggregated_rel_types, + "file_type_distribution": aggregated_file_types, + "repository_details": statistics_data, + "performance_metrics": { + "concurrent_processing_repos": sum( + 1 + for s in statistics_data + if s.get("concurrent_analysis_used", False) + ), + "cache_efficiency": { + "total_cache_hits": sum( + s.get("cache_hits", 0) for s in statistics_data + ), + "repositories_with_caching": sum( + 1 for s in statistics_data if s.get("cache_hits", 0) > 0 + ), + }, + "filtering_efficiency": { + "average_filtering_efficiency": round( + sum(s.get("filtering_efficiency", 0) for s in statistics_data) + / total_repos, + 2, + ) + if total_repos + else 0, + "max_filtering_efficiency": max( + (s.get("filtering_efficiency", 0) for s in statistics_data), + default=0, + ), + "min_filtering_efficiency": min( + (s.get("filtering_efficiency", 0) for s in statistics_data), + default=0, + ), + }, + }, + } + + # Get output configuration + output_config = self.indexer_config.get("output", {}) + json_indent = output_config.get("json_indent", 2) + ensure_ascii = not output_config.get("ensure_ascii", False) + + with open(stats_path, "w", encoding="utf-8") as f: + json.dump( + statistics_report, f, indent=json_indent, ensure_ascii=ensure_ascii + ) + + return str(stats_path) + + def generate_summary_report(self, output_files: Dict[str, str]) -> str: + """Generate a summary report of all indexes created""" + report_path = self.output_dir / "indexing_summary.json" + + # Get output configuration from config file + output_config = self.indexer_config.get("output", {}) + json_indent = output_config.get("json_indent", 2) + ensure_ascii = not output_config.get("ensure_ascii", False) + + summary_data = { + "indexing_completion_time": datetime.now().isoformat(), + "total_repositories_processed": len(output_files), + "output_files": output_files, + "target_structure": self.target_structure, + "code_base_path": str(self.code_base_path), + "configuration": { + "config_file_used": self.indexer_config_path, + "api_config_file": self.config_path, + "pre_filtering_enabled": self.enable_pre_filtering, + "min_confidence_score": self.min_confidence_score, + "high_confidence_threshold": self.high_confidence_threshold, + "max_file_size": self.max_file_size, + "max_content_length": self.max_content_length, + "request_delay": self.request_delay, + "supported_extensions_count": len(self.supported_extensions), + "skip_directories_count": len(self.skip_directories), + }, + } + + with open(report_path, "w", encoding="utf-8") as f: + json.dump(summary_data, f, indent=json_indent, ensure_ascii=ensure_ascii) + + return str(report_path) + + +async def main(): + """Main function to run the code indexer with full configuration support""" + + # Configuration - can be overridden by config file + config_file = "DeepCode/tools/indexer_config.yaml" + api_config_file = "DeepCode/mcp_agent.secrets.yaml" + + # You can override these parameters or let them be read from config + code_base_path = "DeepCode/deepcode_lab/papers/1/code_base/" # Will use config file value if None + output_dir = ( + "DeepCode/deepcode_lab/papers/1/indexes/" # Will use config file value if None + ) + + # Target structure - this should be customized for your specific project + target_structure = """ + project/ + โ”œโ”€โ”€ src/ + โ”‚ โ”œโ”€โ”€ core/ + โ”‚ โ”‚ โ”œโ”€โ”€ gcn.py # GCN encoder + โ”‚ โ”‚ โ”œโ”€โ”€ diffusion.py # forward/reverse processes + โ”‚ โ”‚ โ”œโ”€โ”€ denoiser.py # denoising MLP + โ”‚ โ”‚ โ””โ”€โ”€ fusion.py # fusion combiner + โ”‚ โ”œโ”€โ”€ models/ # model wrapper classes + โ”‚ โ”‚ โ””โ”€โ”€ recdiff.py + โ”‚ โ”œโ”€โ”€ utils/ + โ”‚ โ”‚ โ”œโ”€โ”€ data.py # loading & preprocessing + โ”‚ โ”‚ โ”œโ”€โ”€ predictor.py # scoring functions + โ”‚ โ”‚ โ”œโ”€โ”€ loss.py # loss functions + โ”‚ โ”‚ โ”œโ”€โ”€ metrics.py # NDCG, Recall etc. + โ”‚ โ”‚ โ””โ”€โ”€ sched.py # beta/alpha schedule utils + โ”‚ โ””โ”€โ”€ configs/ + โ”‚ โ””โ”€โ”€ default.yaml # hyperparameters, paths + โ”œโ”€โ”€ tests/ + โ”‚ โ”œโ”€โ”€ test_gcn.py + โ”‚ โ”œโ”€โ”€ test_diffusion.py + โ”‚ โ”œโ”€โ”€ test_denoiser.py + โ”‚ โ”œโ”€โ”€ test_loss.py + โ”‚ โ””โ”€โ”€ test_pipeline.py + โ”œโ”€โ”€ docs/ + โ”‚ โ”œโ”€โ”€ architecture.md + โ”‚ โ”œโ”€โ”€ api_reference.md + โ”‚ โ””โ”€โ”€ README.md + โ”œโ”€โ”€ experiments/ + โ”‚ โ”œโ”€โ”€ run_experiment.py + โ”‚ โ””โ”€โ”€ notebooks/ + โ”‚ โ””โ”€โ”€ analysis.ipynb + โ”œโ”€โ”€ requirements.txt + โ””โ”€โ”€ setup.py + """ + + print("๐Ÿš€ Starting Code Indexer with Enhanced Configuration Support") + print(f"๐Ÿ“‹ Configuration file: {config_file}") + print(f"๐Ÿ”‘ API configuration file: {api_config_file}") + + # Create indexer with full configuration support + try: + indexer = CodeIndexer( + code_base_path=code_base_path, # None = read from config + target_structure=target_structure, # Required - project specific + output_dir=output_dir, # None = read from config + config_path=api_config_file, # API configuration file + indexer_config_path=config_file, # Configuration file + enable_pre_filtering=True, # Can be overridden in config + ) + + # Display configuration information + print(f"๐Ÿ“ Code base path: {indexer.code_base_path}") + print(f"๐Ÿ“‚ Output directory: {indexer.output_dir}") + print( + f"๐Ÿค– Default models: Anthropic={indexer.default_models['anthropic']}, OpenAI={indexer.default_models['openai']}" + ) + print(f"๐Ÿ”ง Preferred LLM: {get_preferred_llm_class(api_config_file).__name__}") + print( + f"โšก Concurrent analysis: {'enabled' if indexer.enable_concurrent_analysis else 'disabled'}" + ) + print( + f"๐Ÿ—„๏ธ Content caching: {'enabled' if indexer.enable_content_caching else 'disabled'}" + ) + print( + f"๐Ÿ” Pre-filtering: {'enabled' if indexer.enable_pre_filtering else 'disabled'}" + ) + print(f"๐Ÿ› Debug mode: {'enabled' if indexer.verbose_output else 'disabled'}") + print( + f"๐ŸŽญ Mock responses: {'enabled' if indexer.mock_llm_responses else 'disabled'}" + ) + + # Validate configuration + if not indexer.code_base_path.exists(): + raise FileNotFoundError( + f"Code base path does not exist: {indexer.code_base_path}" + ) + + if not target_structure: + raise ValueError("Target structure is required for analysis") + + print("\n๐Ÿ”ง Starting indexing process...") + + # Build all indexes + output_files = await indexer.build_all_indexes() + + # Display results + print("\nโœ… Indexing completed successfully!") + print(f"๐Ÿ“Š Processed {len(output_files)} repositories") + print("๐Ÿ“ Output files:") + for repo_name, file_path in output_files.items(): + print(f" - {repo_name}: {file_path}") + + # Display additional reports generated + if indexer.generate_summary: + summary_file = indexer.output_dir / indexer.summary_filename + if summary_file.exists(): + print(f"๐Ÿ“‹ Summary report: {summary_file}") + + if indexer.generate_statistics: + stats_file = indexer.output_dir / indexer.stats_filename + if stats_file.exists(): + print(f"๐Ÿ“ˆ Statistics report: {stats_file}") + + # Performance information + if indexer.enable_content_caching and indexer.content_cache: + print(f"๐Ÿ—„๏ธ Cache performance: {len(indexer.content_cache)} items cached") + + print("\n๐ŸŽ‰ Code indexing process completed successfully!") + + except FileNotFoundError as e: + print(f"โŒ File not found error: {e}") + print("๐Ÿ’ก Please check your configuration file paths") + except ValueError as e: + print(f"โŒ Configuration error: {e}") + print("๐Ÿ’ก Please check your configuration file settings") + except Exception as e: + print(f"โŒ Indexing failed: {e}") + print("๐Ÿ’ก Check the logs for more details") + + # Print debug information if available + try: + indexer + if indexer.verbose_output: + import traceback + + print("\n๐Ÿ› Debug information:") + traceback.print_exc() + except NameError: + pass + + +def print_usage_example(): + """Print usage examples for different scenarios""" + print(""" + ๐Ÿ“– Code Indexer Usage Examples: + + 1. Basic usage with config file: + - Update paths in indexer_config.yaml + - Run: python code_indexer.py + + 2. Enable debugging: + - Set debug.verbose_output: true in config + - Set debug.save_raw_responses: true to save LLM responses + + 3. Enable concurrent processing: + - Set performance.enable_concurrent_analysis: true + - Adjust performance.max_concurrent_files as needed + + 4. Enable caching: + - Set performance.enable_content_caching: true + - Adjust performance.max_cache_size as needed + + 5. Mock mode for testing: + - Set debug.mock_llm_responses: true + - No API calls will be made + + 6. Custom output: + - Modify output.index_filename_pattern + - Set output.generate_statistics: true for detailed reports + + ๐Ÿ“‹ Configuration file location: tools/indexer_config.yaml + """) + + +if __name__ == "__main__": + import sys + + if len(sys.argv) > 1 and sys.argv[1] in ["--help", "-h", "help"]: + print_usage_example() + else: + asyncio.run(main()) diff --git a/DeepCode/tools/code_reference_indexer.py b/DeepCode/tools/code_reference_indexer.py new file mode 100644 index 00000000..c4abc015 --- /dev/null +++ b/DeepCode/tools/code_reference_indexer.py @@ -0,0 +1,495 @@ +#!/usr/bin/env python3 +""" +Code Reference Indexer MCP Tool - Unified Version + +Specialized MCP tool for searching relevant index content in indexes folder +and formatting it for LLM code implementation reference. + +Core Features: +1. **UNIFIED TOOL**: Combined search_code_references that handles directory setup, loading, and searching in one call +2. Match relevant reference code based on target file path and functionality requirements +3. Format output of relevant code examples, functions and concepts +4. Provide structured reference information for LLM use + +Key Improvement: +- Single tool call that handles all steps internally +- Agent only needs to provide indexes_path and target_file +- No dependency on calling order or global state management +""" + +import json +from pathlib import Path +from typing import Dict, List, Tuple +from dataclasses import dataclass +import logging + +# Import MCP modules +from mcp.server.fastmcp import FastMCP + +# Setup logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Create FastMCP server instance +mcp = FastMCP("code-reference-indexer") + + +@dataclass +class CodeReference: + """Code reference information structure""" + + file_path: str + file_type: str + main_functions: List[str] + key_concepts: List[str] + dependencies: List[str] + summary: str + lines_of_code: int + repo_name: str + confidence_score: float = 0.0 + + +@dataclass +class RelationshipInfo: + """Relationship information structure""" + + repo_file_path: str + target_file_path: str + relationship_type: str + confidence_score: float + helpful_aspects: List[str] + potential_contributions: List[str] + usage_suggestions: str + + +def load_index_files_from_directory(indexes_directory: str) -> Dict[str, Dict]: + """Load all index files from specified directory""" + indexes_path = Path(indexes_directory).resolve() + + if not indexes_path.exists(): + logger.warning(f"Indexes directory does not exist: {indexes_path}") + return {} + + index_cache = {} + + for index_file in indexes_path.glob("*.json"): + try: + with open(index_file, "r", encoding="utf-8") as f: + index_data = json.load(f) + index_cache[index_file.stem] = index_data + logger.info(f"Loaded index file: {index_file.name}") + except Exception as e: + logger.error(f"Failed to load index file {index_file.name}: {e}") + + logger.info(f"Loaded {len(index_cache)} index files from {indexes_path}") + return index_cache + + +def extract_code_references(index_data: Dict) -> List[CodeReference]: + """Extract code reference information from index data""" + references = [] + + repo_name = index_data.get("repo_name", "Unknown") + file_summaries = index_data.get("file_summaries", []) + + for file_summary in file_summaries: + reference = CodeReference( + file_path=file_summary.get("file_path", ""), + file_type=file_summary.get("file_type", ""), + main_functions=file_summary.get("main_functions", []), + key_concepts=file_summary.get("key_concepts", []), + dependencies=file_summary.get("dependencies", []), + summary=file_summary.get("summary", ""), + lines_of_code=file_summary.get("lines_of_code", 0), + repo_name=repo_name, + ) + references.append(reference) + + return references + + +def extract_relationships(index_data: Dict) -> List[RelationshipInfo]: + """Extract relationship information from index data""" + relationships = [] + + relationship_list = index_data.get("relationships", []) + + for rel in relationship_list: + relationship = RelationshipInfo( + repo_file_path=rel.get("repo_file_path", ""), + target_file_path=rel.get("target_file_path", ""), + relationship_type=rel.get("relationship_type", ""), + confidence_score=rel.get("confidence_score", 0.0), + helpful_aspects=rel.get("helpful_aspects", []), + potential_contributions=rel.get("potential_contributions", []), + usage_suggestions=rel.get("usage_suggestions", ""), + ) + relationships.append(relationship) + + return relationships + + +def calculate_relevance_score( + target_file: str, reference: CodeReference, keywords: List[str] = None +) -> float: + """Calculate relevance score between reference code and target file""" + score = 0.0 + + # File name similarity + target_name = Path(target_file).stem.lower() + ref_name = Path(reference.file_path).stem.lower() + + if target_name in ref_name or ref_name in target_name: + score += 0.3 + + # File type matching + target_extension = Path(target_file).suffix + ref_extension = Path(reference.file_path).suffix + + if target_extension == ref_extension: + score += 0.2 + + # Keyword matching + if keywords: + keyword_matches = 0 + total_searchable_text = ( + " ".join(reference.key_concepts) + + " " + + " ".join(reference.main_functions) + + " " + + reference.summary + + " " + + reference.file_type + ).lower() + + for keyword in keywords: + if keyword.lower() in total_searchable_text: + keyword_matches += 1 + + if keywords: + score += (keyword_matches / len(keywords)) * 0.5 + + return min(score, 1.0) + + +def find_relevant_references_in_cache( + target_file: str, + index_cache: Dict[str, Dict], + keywords: List[str] = None, + max_results: int = 10, +) -> List[Tuple[CodeReference, float]]: + """Find reference code relevant to target file from provided cache""" + all_references = [] + + # Collect reference information from all index files + for repo_name, index_data in index_cache.items(): + references = extract_code_references(index_data) + for ref in references: + relevance_score = calculate_relevance_score(target_file, ref, keywords) + if relevance_score > 0.1: # Only keep results with certain relevance + all_references.append((ref, relevance_score)) + + # Sort by relevance score + all_references.sort(key=lambda x: x[1], reverse=True) + + return all_references[:max_results] + + +def find_direct_relationships_in_cache( + target_file: str, index_cache: Dict[str, Dict] +) -> List[RelationshipInfo]: + """Find direct relationships with target file from provided cache""" + relationships = [] + + # Normalize target file path (remove common prefixes if exists) + common_prefixes = ["src/", "core/", "lib/", "main/", "./"] + normalized_target = target_file.strip("/") + for prefix in common_prefixes: + if normalized_target.startswith(prefix): + normalized_target = normalized_target[len(prefix) :] + break + + # Collect relationship information from all index files + for repo_name, index_data in index_cache.items(): + repo_relationships = extract_relationships(index_data) + for rel in repo_relationships: + # Normalize target file path in relationship + normalized_rel_target = rel.target_file_path.strip("/") + for prefix in common_prefixes: + if normalized_rel_target.startswith(prefix): + normalized_rel_target = normalized_rel_target[len(prefix) :] + break + + # Check target file path matching (support multiple matching methods) + if ( + normalized_target == normalized_rel_target + or normalized_target in normalized_rel_target + or normalized_rel_target in normalized_target + or target_file in rel.target_file_path + or rel.target_file_path in target_file + ): + relationships.append(rel) + + # Sort by confidence score + relationships.sort(key=lambda x: x.confidence_score, reverse=True) + + return relationships + + +def format_reference_output( + target_file: str, + relevant_refs: List[Tuple[CodeReference, float]], + relationships: List[RelationshipInfo], +) -> str: + """Format reference information output""" + output_lines = [] + + output_lines.append(f"# Code Reference Information - {target_file}") + output_lines.append("=" * 80) + output_lines.append("") + + # Direct relationship information + if relationships: + output_lines.append("## ๐ŸŽฏ Direct Relationships") + output_lines.append("") + + for i, rel in enumerate(relationships[:5], 1): + output_lines.append(f"### {i}. {rel.repo_file_path}") + output_lines.append(f"**Relationship Type**: {rel.relationship_type}") + output_lines.append(f"**Confidence Score**: {rel.confidence_score:.2f}") + output_lines.append( + f"**Helpful Aspects**: {', '.join(rel.helpful_aspects)}" + ) + output_lines.append( + f"**Potential Contributions**: {', '.join(rel.potential_contributions)}" + ) + output_lines.append(f"**Usage Suggestions**: {rel.usage_suggestions}") + output_lines.append("") + + # Relevant code references + if relevant_refs: + output_lines.append("## ๐Ÿ“š Relevant Code References") + output_lines.append("") + + for i, (ref, score) in enumerate(relevant_refs[:8], 1): + output_lines.append(f"### {i}. {ref.file_path} (Relevance: {score:.2f})") + output_lines.append(f"**Repository**: {ref.repo_name}") + output_lines.append(f"**File Type**: {ref.file_type}") + output_lines.append( + f"**Main Functions**: {', '.join(ref.main_functions[:5])}" + ) + output_lines.append(f"**Key Concepts**: {', '.join(ref.key_concepts[:8])}") + output_lines.append(f"**Dependencies**: {', '.join(ref.dependencies[:6])}") + output_lines.append(f"**Lines of Code**: {ref.lines_of_code}") + output_lines.append(f"**Summary**: {ref.summary[:300]}...") + output_lines.append("") + + # Implementation suggestions + output_lines.append("## ๐Ÿ’ก Implementation Suggestions") + output_lines.append("") + + if relevant_refs: + # Collect all function names and concepts + all_functions = set() + all_concepts = set() + all_dependencies = set() + + for ref, _ in relevant_refs[:5]: + all_functions.update(ref.main_functions) + all_concepts.update(ref.key_concepts) + all_dependencies.update(ref.dependencies) + + output_lines.append("**Reference Function Name Patterns**:") + for func in sorted(list(all_functions))[:10]: + output_lines.append(f"- {func}") + output_lines.append("") + + output_lines.append("**Important Concepts and Patterns**:") + for concept in sorted(list(all_concepts))[:15]: + output_lines.append(f"- {concept}") + output_lines.append("") + + output_lines.append("**Potential Dependencies Needed**:") + for dep in sorted(list(all_dependencies))[:10]: + output_lines.append(f"- {dep}") + output_lines.append("") + + output_lines.append("## ๐Ÿš€ Next Actions") + output_lines.append( + "1. Analyze design patterns and architectural styles from the above reference code" + ) + output_lines.append("2. Determine core functionalities and interfaces to implement") + output_lines.append("3. Choose appropriate dependency libraries and tools") + output_lines.append( + "4. Design implementation solution consistent with existing code style" + ) + output_lines.append("5. Start writing specific code implementation") + + return "\n".join(output_lines) + + +# ==================== MCP Tool Definitions ==================== + + +@mcp.tool() +async def search_code_references( + indexes_path: str, target_file: str, keywords: str = "", max_results: int = 10 +) -> str: + """ + **UNIFIED TOOL**: Search relevant reference code from index files for target file implementation. + This tool combines directory setup, index loading, and searching in a single call. + + Args: + indexes_path: Path to the indexes directory containing JSON index files + target_file: Target file path (file to be implemented) + keywords: Search keywords, comma-separated + max_results: Maximum number of results to return + + Returns: + Formatted reference code information JSON string + """ + try: + # Step 1: Load index files from specified directory + logger.info(f"Loading index files from: {indexes_path}") + index_cache = load_index_files_from_directory(indexes_path) + + if not index_cache: + result = { + "status": "error", + "message": f"No index files found or failed to load from: {indexes_path}", + "target_file": target_file, + "indexes_path": indexes_path, + } + return json.dumps(result, ensure_ascii=False, indent=2) + + # Step 2: Parse keywords + keyword_list = ( + [kw.strip() for kw in keywords.split(",") if kw.strip()] if keywords else [] + ) + + # Step 3: Find relevant reference code + relevant_refs = find_relevant_references_in_cache( + target_file, index_cache, keyword_list, max_results + ) + + # Step 4: Find direct relationships + relationships = find_direct_relationships_in_cache(target_file, index_cache) + + # Step 5: Format output + formatted_output = format_reference_output( + target_file, relevant_refs, relationships + ) + + result = { + "status": "success", + "target_file": target_file, + "indexes_path": indexes_path, + "keywords_used": keyword_list, + "total_references_found": len(relevant_refs), + "total_relationships_found": len(relationships), + "formatted_content": formatted_output, + "indexes_loaded": list(index_cache.keys()), + "total_indexes_loaded": len(index_cache), + } + + logger.info( + f"Successfully found {len(relevant_refs)} references and {len(relationships)} relationships for {target_file}" + ) + return json.dumps(result, ensure_ascii=False, indent=2) + + except Exception as e: + logger.error(f"Error in search_code_references: {str(e)}") + result = { + "status": "error", + "message": f"Failed to search reference code: {str(e)}", + "target_file": target_file, + "indexes_path": indexes_path, + } + return json.dumps(result, ensure_ascii=False, indent=2) + + +@mcp.tool() +async def get_indexes_overview(indexes_path: str) -> str: + """ + Get overview of all available reference code index information from specified directory + + Args: + indexes_path: Path to the indexes directory containing JSON index files + + Returns: + Overview information of all available reference code JSON string + """ + try: + # Load index files from specified directory + index_cache = load_index_files_from_directory(indexes_path) + + if not index_cache: + result = { + "status": "error", + "message": f"No index files found in: {indexes_path}", + "indexes_path": indexes_path, + } + return json.dumps(result, ensure_ascii=False, indent=2) + + overview = {"total_repos": len(index_cache), "repositories": {}} + + for repo_name, index_data in index_cache.items(): + repo_info = { + "repo_name": index_data.get("repo_name", repo_name), + "total_files": index_data.get("total_files", 0), + "file_types": [], + "main_concepts": [], + "total_relationships": len(index_data.get("relationships", [])), + } + + # Collect file types and concepts + file_summaries = index_data.get("file_summaries", []) + file_types = set() + concepts = set() + + for file_summary in file_summaries: + file_types.add(file_summary.get("file_type", "Unknown")) + concepts.update(file_summary.get("key_concepts", [])) + + repo_info["file_types"] = sorted(list(file_types)) + repo_info["main_concepts"] = sorted(list(concepts))[ + :20 + ] # Limit concept count + + overview["repositories"][repo_name] = repo_info + + result = { + "status": "success", + "overview": overview, + "indexes_directory": str(Path(indexes_path).resolve()), + "total_indexes_loaded": len(index_cache), + } + + return json.dumps(result, ensure_ascii=False, indent=2) + + except Exception as e: + result = { + "status": "error", + "message": f"Failed to get indexes overview: {str(e)}", + "indexes_path": indexes_path, + } + return json.dumps(result, ensure_ascii=False, indent=2) + + +def main(): + """Main function""" + logger.info("Starting unified Code Reference Indexer MCP server") + logger.info("Available tools:") + logger.info( + "1. search_code_references(indexes_path, target_file, keywords, max_results) - UNIFIED TOOL" + ) + logger.info( + "2. get_indexes_overview(indexes_path) - Get overview of available indexes" + ) + + # Run MCP server + mcp.run() + + +if __name__ == "__main__": + main() diff --git a/DeepCode/tools/command_executor.py b/DeepCode/tools/command_executor.py new file mode 100644 index 00000000..4bdc4711 --- /dev/null +++ b/DeepCode/tools/command_executor.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +""" +Command Executor MCP Tool / ๅ‘ฝไปคๆ‰ง่กŒๅ™จ MCP ๅทฅๅ…ท + +ไธ“้—จ่ดŸ่ดฃๆ‰ง่กŒLLM็”Ÿๆˆ็š„shellๅ‘ฝไปคๆฅๅˆ›ๅปบๆ–‡ไปถๆ ‘็ป“ๆž„ +Specialized in executing LLM-generated shell commands to create file tree structures +""" + +import subprocess +from pathlib import Path +from typing import List, Dict +from mcp.server.models import InitializationOptions +import mcp.types as types +from mcp.server import NotificationOptions, Server +import mcp.server.stdio + +# ๅˆ›ๅปบMCPๆœๅŠกๅ™จๅฎžไพ‹ / Create MCP server instance +app = Server("command-executor") + + +@app.list_tools() +async def handle_list_tools() -> list[types.Tool]: + """ + ๅˆ—ๅ‡บๅฏ็”จๅทฅๅ…ท / List available tools + """ + return [ + types.Tool( + name="execute_commands", + description=""" + ๆ‰ง่กŒshellๅ‘ฝไปคๅˆ—่กจๆฅๅˆ›ๅปบๆ–‡ไปถๆ ‘็ป“ๆž„ + Execute shell command list to create file tree structure + + Args: + commands: ่ฆๆ‰ง่กŒ็š„shellๅ‘ฝไปคๅˆ—่กจ๏ผˆๆฏ่กŒไธ€ไธชๅ‘ฝไปค๏ผ‰ + working_directory: ๆ‰ง่กŒๅ‘ฝไปค็š„ๅทฅไฝœ็›ฎๅฝ• + + Returns: + ๅ‘ฝไปคๆ‰ง่กŒ็ป“ๆžœๅ’Œ่ฏฆ็ป†ๆŠฅๅ‘Š + """, + inputSchema={ + "type": "object", + "properties": { + "commands": { + "type": "string", + "title": "Commands", + "description": "่ฆๆ‰ง่กŒ็š„shellๅ‘ฝไปคๅˆ—่กจ๏ผŒๆฏ่กŒไธ€ไธชๅ‘ฝไปค", + }, + "working_directory": { + "type": "string", + "title": "Working Directory", + "description": "ๆ‰ง่กŒๅ‘ฝไปค็š„ๅทฅไฝœ็›ฎๅฝ•", + }, + }, + "required": ["commands", "working_directory"], + }, + ), + types.Tool( + name="execute_single_command", + description=""" + ๆ‰ง่กŒๅ•ไธชshellๅ‘ฝไปค + Execute single shell command + + Args: + command: ่ฆๆ‰ง่กŒ็š„ๅ•ไธชๅ‘ฝไปค + working_directory: ๆ‰ง่กŒๅ‘ฝไปค็š„ๅทฅไฝœ็›ฎๅฝ• + + Returns: + ๅ‘ฝไปคๆ‰ง่กŒ็ป“ๆžœ + """, + inputSchema={ + "type": "object", + "properties": { + "command": { + "type": "string", + "title": "Command", + "description": "่ฆๆ‰ง่กŒ็š„ๅ•ไธชshellๅ‘ฝไปค", + }, + "working_directory": { + "type": "string", + "title": "Working Directory", + "description": "ๆ‰ง่กŒๅ‘ฝไปค็š„ๅทฅไฝœ็›ฎๅฝ•", + }, + }, + "required": ["command", "working_directory"], + }, + ), + ] + + +@app.call_tool() +async def handle_call_tool(name: str, arguments: dict) -> list[types.TextContent]: + """ + ๅค„็†ๅทฅๅ…ท่ฐƒ็”จ / Handle tool calls + """ + try: + if name == "execute_commands": + return await execute_command_batch( + arguments.get("commands", ""), arguments.get("working_directory", ".") + ) + elif name == "execute_single_command": + return await execute_single_command( + arguments.get("command", ""), arguments.get("working_directory", ".") + ) + else: + raise ValueError(f"ๆœช็Ÿฅๅทฅๅ…ท / Unknown tool: {name}") + + except Exception as e: + return [ + types.TextContent( + type="text", + text=f"ๅทฅๅ…ทๆ‰ง่กŒ้”™่ฏฏ / Error executing tool {name}: {str(e)}", + ) + ] + + +async def execute_command_batch( + commands: str, working_directory: str +) -> list[types.TextContent]: + """ + ๆ‰ง่กŒๅคšไธชshellๅ‘ฝไปค / Execute multiple shell commands + + Args: + commands: ๅ‘ฝไปคๅˆ—่กจ๏ผŒๆฏ่กŒไธ€ไธชๅ‘ฝไปค / Command list, one command per line + working_directory: ๅทฅไฝœ็›ฎๅฝ• / Working directory + + Returns: + ๆ‰ง่กŒ็ป“ๆžœ / Execution results + """ + try: + # ็กฎไฟๅทฅไฝœ็›ฎๅฝ•ๅญ˜ๅœจ / Ensure working directory exists + Path(working_directory).mkdir(parents=True, exist_ok=True) + + # ๅˆ†ๅ‰ฒๅ‘ฝไปค่กŒ / Split command lines + command_lines = [ + cmd.strip() for cmd in commands.strip().split("\n") if cmd.strip() + ] + + if not command_lines: + return [ + types.TextContent( + type="text", text="ๆฒกๆœ‰ๆไพ›ๆœ‰ๆ•ˆๅ‘ฝไปค / No valid commands provided" + ) + ] + + results = [] + stats = {"successful": 0, "failed": 0, "timeout": 0} + + for i, command in enumerate(command_lines, 1): + try: + # ๆ‰ง่กŒๅ‘ฝไปค / Execute command + result = subprocess.run( + command, + shell=True, + cwd=working_directory, + capture_output=True, + text=True, + timeout=30, # 30็ง’่ถ…ๆ—ถ + ) + + if result.returncode == 0: + results.append(f"โœ… Command {i}: {command}") + if result.stdout.strip(): + results.append(f" ่พ“ๅ‡บ / Output: {result.stdout.strip()}") + stats["successful"] += 1 + else: + results.append(f"โŒ Command {i}: {command}") + if result.stderr.strip(): + results.append(f" ้”™่ฏฏ / Error: {result.stderr.strip()}") + stats["failed"] += 1 + + except subprocess.TimeoutExpired: + results.append(f"โฑ๏ธ Command {i} ่ถ…ๆ—ถ / timeout: {command}") + stats["timeout"] += 1 + except Exception as e: + results.append(f"๐Ÿ’ฅ Command {i} ๅผ‚ๅธธ / exception: {command} - {str(e)}") + stats["failed"] += 1 + + # ็”Ÿๆˆๆ‰ง่กŒๆŠฅๅ‘Š / Generate execution report + summary = generate_execution_summary(working_directory, command_lines, stats) + final_result = summary + "\n" + "\n".join(results) + + return [types.TextContent(type="text", text=final_result)] + + except Exception as e: + return [ + types.TextContent( + type="text", + text=f"ๆ‰น้‡ๅ‘ฝไปคๆ‰ง่กŒๅคฑ่ดฅ / Failed to execute command batch: {str(e)}", + ) + ] + + +async def execute_single_command( + command: str, working_directory: str +) -> list[types.TextContent]: + """ + ๆ‰ง่กŒๅ•ไธชshellๅ‘ฝไปค / Execute single shell command + + Args: + command: ่ฆๆ‰ง่กŒ็š„ๅ‘ฝไปค / Command to execute + working_directory: ๅทฅไฝœ็›ฎๅฝ• / Working directory + + Returns: + ๆ‰ง่กŒ็ป“ๆžœ / Execution result + """ + try: + # ็กฎไฟๅทฅไฝœ็›ฎๅฝ•ๅญ˜ๅœจ / Ensure working directory exists + Path(working_directory).mkdir(parents=True, exist_ok=True) + + # ๆ‰ง่กŒๅ‘ฝไปค / Execute command + result = subprocess.run( + command, + shell=True, + cwd=working_directory, + capture_output=True, + text=True, + timeout=30, + ) + + # ๆ ผๅผๅŒ–่พ“ๅ‡บ / Format output + output = format_single_command_result(command, working_directory, result) + + return [types.TextContent(type="text", text=output)] + + except subprocess.TimeoutExpired: + return [ + types.TextContent( + type="text", text=f"โฑ๏ธ ๅ‘ฝไปค่ถ…ๆ—ถ / Command timeout: {command}" + ) + ] + except Exception as e: + return [ + types.TextContent( + type="text", text=f"๐Ÿ’ฅ ๅ‘ฝไปคๆ‰ง่กŒ้”™่ฏฏ / Command execution error: {str(e)}" + ) + ] + + +def generate_execution_summary( + working_directory: str, command_lines: List[str], stats: Dict[str, int] +) -> str: + """ + ็”Ÿๆˆๆ‰ง่กŒๆ€ป็ป“ / Generate execution summary + + Args: + working_directory: ๅทฅไฝœ็›ฎๅฝ• / Working directory + command_lines: ๅ‘ฝไปคๅˆ—่กจ / Command list + stats: ็ปŸ่ฎกไฟกๆฏ / Statistics + + Returns: + ๆ ผๅผๅŒ–็š„ๆ€ป็ป“ / Formatted summary + """ + return f""" +ๅ‘ฝไปคๆ‰ง่กŒๆ€ป็ป“ / Command Execution Summary: +{'='*50} +ๅทฅไฝœ็›ฎๅฝ• / Working Directory: {working_directory} +ๆ€ปๅ‘ฝไปคๆ•ฐ / Total Commands: {len(command_lines)} +ๆˆๅŠŸ / Successful: {stats['successful']} +ๅคฑ่ดฅ / Failed: {stats['failed']} +่ถ…ๆ—ถ / Timeout: {stats['timeout']} + +่ฏฆ็ป†็ป“ๆžœ / Detailed Results: +{'-'*50}""" + + +def format_single_command_result( + command: str, working_directory: str, result: subprocess.CompletedProcess +) -> str: + """ + ๆ ผๅผๅŒ–ๅ•ๅ‘ฝไปคๆ‰ง่กŒ็ป“ๆžœ / Format single command execution result + + Args: + command: ๆ‰ง่กŒ็š„ๅ‘ฝไปค / Executed command + working_directory: ๅทฅไฝœ็›ฎๅฝ• / Working directory + result: ๆ‰ง่กŒ็ป“ๆžœ / Execution result + + Returns: + ๆ ผๅผๅŒ–็š„็ป“ๆžœ / Formatted result + """ + output = f""" +ๅ•ๅ‘ฝไปคๆ‰ง่กŒ / Single Command Execution: +{'='*40} +ๅทฅไฝœ็›ฎๅฝ• / Working Directory: {working_directory} +ๅ‘ฝไปค / Command: {command} +่ฟ”ๅ›ž็  / Return Code: {result.returncode} + +""" + + if result.returncode == 0: + output += "โœ… ็Šถๆ€ / Status: SUCCESS / ๆˆๅŠŸ\n" + if result.stdout.strip(): + output += f"่พ“ๅ‡บ / Output:\n{result.stdout.strip()}\n" + else: + output += "โŒ ็Šถๆ€ / Status: FAILED / ๅคฑ่ดฅ\n" + if result.stderr.strip(): + output += f"้”™่ฏฏ / Error:\n{result.stderr.strip()}\n" + + return output + + +async def main(): + """ + ่ฟ่กŒMCPๆœๅŠกๅ™จ / Run MCP server + """ + # ้€š่ฟ‡stdio่ฟ่กŒๆœๅŠกๅ™จ / Run server via stdio + async with mcp.server.stdio.stdio_server() as (read_stream, write_stream): + await app.run( + read_stream, + write_stream, + InitializationOptions( + server_name="command-executor", + server_version="1.0.0", + capabilities=app.get_capabilities( + notification_options=NotificationOptions(), + experimental_capabilities={}, + ), + ), + ) + + +if __name__ == "__main__": + import asyncio + + asyncio.run(main()) diff --git a/DeepCode/tools/document_segmentation_server.py b/DeepCode/tools/document_segmentation_server.py new file mode 100644 index 00000000..c090632e --- /dev/null +++ b/DeepCode/tools/document_segmentation_server.py @@ -0,0 +1,1937 @@ +#!/usr/bin/env python3 +""" +Document Segmentation MCP Server + +This MCP server provides intelligent document segmentation and retrieval functions for handling +large research papers and technical documents that exceed LLM token limits. + +==== CORE FUNCTIONALITY ==== +1. Analyze document structure and type using semantic content analysis +2. Create intelligent segments based on content semantics, not just structure +3. Provide query-aware segment retrieval with relevance scoring +4. Support both structured (papers with headers) and unstructured documents +5. Configurable segmentation strategies based on document complexity + +==== MCP TOOLS PROVIDED ==== + +๐Ÿ“„ analyze_and_segment_document(paper_dir: str, force_refresh: bool = False) + Purpose: Analyzes document structure and creates intelligent segments + - Detects document type (research paper, technical doc, algorithm-focused, etc.) + - Selects optimal segmentation strategy based on content analysis + - Creates semantic segments preserving algorithm and concept integrity + - Stores segmentation index for efficient retrieval + - Returns: JSON with segmentation status, strategy used, and segment count + +๐Ÿ“– read_document_segments(paper_dir: str, query_type: str, keywords: List[str] = None, + max_segments: int = 3, max_total_chars: int = None) + Purpose: Intelligently retrieves relevant document segments based on query context + - query_type: "concept_analysis", "algorithm_extraction", or "code_planning" + - Uses semantic relevance scoring to rank segments + - Applies query-specific filtering and keyword matching + - Dynamically calculates optimal character limits based on content complexity + - Returns: JSON with selected segments optimized for the specific query type + +๐Ÿ“‹ get_document_overview(paper_dir: str) + Purpose: Provides high-level overview of document structure and available segments + - Shows document type and segmentation strategy used + - Lists all segments with titles, content types, and relevance scores + - Displays segment statistics (character counts, keyword summaries) + - Returns: JSON with complete document analysis metadata + +==== SEGMENTATION STRATEGIES ==== +- semantic_research_focused: For academic papers with complex algorithmic content +- algorithm_preserve_integrity: Maintains algorithm blocks and formula chains intact +- concept_implementation_hybrid: Merges related concepts with implementation details +- semantic_chunking_enhanced: Advanced boundary detection for long documents +- content_aware_segmentation: Adaptive chunking based on content density + +==== INTELLIGENT FEATURES ==== +- Semantic boundary detection (not just structural) +- Algorithm block identification and preservation +- Formula chain recognition and grouping +- Concept-implementation relationship mapping +- Multi-level relevance scoring (content type, importance, keyword matching) +- Backward compatibility with existing document indexes +- Configurable via mcp_agent.config.yaml (enabled/disabled, size thresholds) + +Usage: +python tools/document_segmentation_server.py +""" + +import os +import re +import json +import sys +import io +from typing import Dict, List, Tuple +import hashlib +import logging +from datetime import datetime +from dataclasses import dataclass, asdict + +# Set standard output encoding to UTF-8 +if sys.stdout.encoding != "utf-8": + try: + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8") + sys.stderr.reconfigure(encoding="utf-8") + else: + sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding="utf-8") + sys.stderr = io.TextIOWrapper(sys.stderr.detach(), encoding="utf-8") + except Exception as e: + print(f"Warning: Could not set UTF-8 encoding: {e}") + +# Import MCP related modules +from mcp.server.fastmcp import FastMCP + +# Setup logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Create FastMCP server instance +mcp = FastMCP("document-segmentation-server") + + +@dataclass +class DocumentSegment: + """Represents a document segment with metadata""" + + id: str + title: str + content: str + content_type: str # "introduction", "methodology", "algorithm", "results", etc. + keywords: List[str] + char_start: int + char_end: int + char_count: int + relevance_scores: Dict[str, float] # Scores for different query types + section_path: str # e.g., "3.2.1" for nested sections + + +@dataclass +class DocumentIndex: + """Document index containing all segments and metadata""" + + document_path: str + document_type: str # "academic_paper", "technical_doc", "code_doc", "general" + segmentation_strategy: str + total_segments: int + total_chars: int + segments: List[DocumentSegment] + created_at: str + + +class DocumentAnalyzer: + """Enhanced document analyzer using semantic content analysis instead of mechanical structure detection""" + + # More precise semantic indicators, weighted by importance + ALGORITHM_INDICATORS = { + "high": [ + "algorithm", + "procedure", + "method", + "approach", + "technique", + "framework", + ], + "medium": ["step", "process", "implementation", "computation", "calculation"], + "low": ["example", "illustration", "demonstration"], + } + + TECHNICAL_CONCEPT_INDICATORS = { + "high": ["formula", "equation", "theorem", "lemma", "proof", "definition"], + "medium": ["parameter", "variable", "function", "model", "architecture"], + "low": ["notation", "symbol", "term"], + } + + IMPLEMENTATION_INDICATORS = { + "high": ["code", "implementation", "programming", "software", "system"], + "medium": ["design", "structure", "module", "component", "interface"], + "low": ["tool", "library", "package"], + } + + # Semantic features of document types (not just based on titles) + RESEARCH_PAPER_PATTERNS = [ + r"(?i)\babstract\b.*?\n.*?(introduction|motivation|background)", + r"(?i)(methodology|method).*?(experiment|evaluation|result)", + r"(?i)(conclusion|future work|limitation).*?(reference|bibliography)", + r"(?i)(related work|literature review|prior art)", + ] + + TECHNICAL_DOC_PATTERNS = [ + r"(?i)(getting started|installation|setup).*?(usage|example)", + r"(?i)(api|interface|specification).*?(parameter|endpoint)", + r"(?i)(tutorial|guide|walkthrough).*?(step|instruction)", + r"(?i)(troubleshooting|faq|common issues)", + ] + + def analyze_document_type(self, content: str) -> Tuple[str, float]: + """ + Enhanced document type analysis based on semantic content patterns + + Returns: + Tuple[str, float]: (document_type, confidence_score) + """ + content_lower = content.lower() + + # Calculate weighted semantic indicator scores + algorithm_score = self._calculate_weighted_score( + content_lower, self.ALGORITHM_INDICATORS + ) + concept_score = self._calculate_weighted_score( + content_lower, self.TECHNICAL_CONCEPT_INDICATORS + ) + implementation_score = self._calculate_weighted_score( + content_lower, self.IMPLEMENTATION_INDICATORS + ) + + # Detect semantic patterns of document types + research_pattern_score = self._detect_pattern_score( + content, self.RESEARCH_PAPER_PATTERNS + ) + technical_pattern_score = self._detect_pattern_score( + content, self.TECHNICAL_DOC_PATTERNS + ) + + # Comprehensive evaluation of document type + total_research_score = ( + algorithm_score + concept_score + research_pattern_score * 2 + ) + total_technical_score = implementation_score + technical_pattern_score * 2 + + # Determine document type based on content density and pattern matching + if research_pattern_score > 0.5 and total_research_score > 3.0: + return "research_paper", min(0.95, 0.6 + research_pattern_score * 0.35) + elif algorithm_score > 2.0 and concept_score > 1.5: + return "algorithm_focused", 0.85 + elif total_technical_score > 2.5: + return "technical_doc", 0.8 + elif implementation_score > 1.5: + return "implementation_guide", 0.75 + else: + return "general_document", 0.5 + + def _calculate_weighted_score( + self, content: str, indicators: Dict[str, List[str]] + ) -> float: + """Calculate weighted semantic indicator scores""" + score = 0.0 + for weight_level, terms in indicators.items(): + weight = {"high": 3.0, "medium": 2.0, "low": 1.0}[weight_level] + for term in terms: + if term in content: + score += weight * ( + content.count(term) * 0.5 + 1 + ) # Consider term frequency + return score + + def _detect_pattern_score(self, content: str, patterns: List[str]) -> float: + """Detect semantic pattern matching scores""" + matches = 0 + for pattern in patterns: + if re.search(pattern, content, re.DOTALL): + matches += 1 + return matches / len(patterns) + + def detect_segmentation_strategy(self, content: str, doc_type: str) -> str: + """ + Intelligently determine the best segmentation strategy based on content semantics rather than mechanical structure + """ + # Analyze content characteristics + algorithm_density = self._calculate_algorithm_density(content) + concept_complexity = self._calculate_concept_complexity(content) + implementation_detail_level = self._calculate_implementation_detail_level( + content + ) + + # Select strategy based on document type and content characteristics + if doc_type == "research_paper" and algorithm_density > 0.3: + return "semantic_research_focused" + elif doc_type == "algorithm_focused" or algorithm_density > 0.5: + return "algorithm_preserve_integrity" + elif concept_complexity > 0.4 and implementation_detail_level > 0.3: + return "concept_implementation_hybrid" + elif len(content) > 15000: # Long documents + return "semantic_chunking_enhanced" + else: + return "content_aware_segmentation" + + def _calculate_algorithm_density(self, content: str) -> float: + """Calculate algorithm content density""" + total_chars = len(content) + algorithm_chars = 0 + + # Identify algorithm blocks + algorithm_patterns = [ + r"(?i)(algorithm\s+\d+|procedure\s+\d+)", + r"(?i)(step\s+\d+|phase\s+\d+)", + r"(?i)(input:|output:|return:|initialize:)", + r"(?i)(for\s+each|while|if.*then|else)", + r"(?i)(function|method|procedure).*\(", + ] + + for pattern in algorithm_patterns: + matches = re.finditer(pattern, content) + for match in matches: + # Estimate algorithm block size (expand forward and backward from match point) + start = max(0, match.start() - 200) + end = min(len(content), match.end() + 800) + algorithm_chars += end - start + + return min(1.0, algorithm_chars / total_chars) + + def _calculate_concept_complexity(self, content: str) -> float: + """Calculate concept complexity""" + concept_indicators = self.TECHNICAL_CONCEPT_INDICATORS + complexity_score = 0.0 + + for level, terms in concept_indicators.items(): + weight = {"high": 3.0, "medium": 2.0, "low": 1.0}[level] + for term in terms: + complexity_score += content.lower().count(term) * weight + + # Normalize to 0-1 range + return min(1.0, complexity_score / 100) + + def _calculate_implementation_detail_level(self, content: str) -> float: + """Calculate implementation detail level""" + implementation_patterns = [ + r"(?i)(code|implementation|programming)", + r"(?i)(class|function|method|variable)", + r"(?i)(import|include|library)", + r"(?i)(parameter|argument|return)", + r"(?i)(example|demo|tutorial)", + ] + + detail_score = 0 + for pattern in implementation_patterns: + detail_score += len(re.findall(pattern, content)) + + return min(1.0, detail_score / 50) + + +class DocumentSegmenter: + """Creates intelligent segments from documents""" + + def __init__(self): + self.analyzer = DocumentAnalyzer() + + def segment_document(self, content: str, strategy: str) -> List[DocumentSegment]: + """ + Perform intelligent segmentation using the specified strategy + """ + if strategy == "semantic_research_focused": + return self._segment_research_paper_semantically(content) + elif strategy == "algorithm_preserve_integrity": + return self._segment_preserve_algorithm_integrity(content) + elif strategy == "concept_implementation_hybrid": + return self._segment_concept_implementation_hybrid(content) + elif strategy == "semantic_chunking_enhanced": + return self._segment_by_enhanced_semantic_chunks(content) + elif strategy == "content_aware_segmentation": + return self._segment_content_aware(content) + else: + # Compatibility with legacy strategies + return self._segment_by_enhanced_semantic_chunks(content) + + def _segment_by_headers(self, content: str) -> List[DocumentSegment]: + """Segment document based on markdown headers""" + segments = [] + lines = content.split("\n") + current_segment = [] + current_header = None + char_pos = 0 + + for line in lines: + line_with_newline = line + "\n" + + # Check if line is a header + header_match = re.match(r"^(#{1,6})\s+(.+)$", line) + + if header_match: + # Save previous segment if exists + if current_segment and current_header: + segment_content = "\n".join(current_segment).strip() + if segment_content: + # Analyze content type and importance + content_type = self._classify_content_type( + current_header, segment_content + ) + importance_score = ( + 0.8 if content_type in ["algorithm", "formula"] else 0.7 + ) + + segment = self._create_enhanced_segment( + segment_content, + current_header, + char_pos - len(segment_content.encode("utf-8")), + char_pos, + importance_score, + content_type, + ) + segments.append(segment) + + # Start new segment + current_header = header_match.group(2).strip() + current_segment = [line] + else: + if current_segment is not None: + current_segment.append(line) + + char_pos += len(line_with_newline.encode("utf-8")) + + # Add final segment + if current_segment and current_header: + segment_content = "\n".join(current_segment).strip() + if segment_content: + # Analyze content type and importance + content_type = self._classify_content_type( + current_header, segment_content + ) + importance_score = ( + 0.8 if content_type in ["algorithm", "formula"] else 0.7 + ) + + segment = self._create_enhanced_segment( + segment_content, + current_header, + char_pos - len(segment_content.encode("utf-8")), + char_pos, + importance_score, + content_type, + ) + segments.append(segment) + + return segments + + def _segment_preserve_algorithm_integrity( + self, content: str + ) -> List[DocumentSegment]: + """Smart segmentation strategy that preserves algorithm integrity""" + segments = [] + + # 1. Identify algorithm blocks and related descriptions + algorithm_blocks = self._identify_algorithm_blocks(content) + + # 2. Identify concept definition groups + concept_groups = self._identify_concept_groups(content) + + # 3. Identify formula derivation chains + formula_chains = self._identify_formula_chains(content) + + # 4. Merge related content blocks to ensure integrity + content_blocks = self._merge_related_content_blocks( + algorithm_blocks, concept_groups, formula_chains, content + ) + + # 5. Convert to DocumentSegment + for i, block in enumerate(content_blocks): + segment = self._create_enhanced_segment( + block["content"], + block["title"], + block["start_pos"], + block["end_pos"], + block["importance_score"], + block["content_type"], + ) + segments.append(segment) + + return segments + + def _segment_research_paper_semantically( + self, content: str + ) -> List[DocumentSegment]: + """Semantic segmentation specifically for research papers""" + segments = [] + + # Identify semantic structure of research papers + paper_sections = self._identify_research_paper_sections(content) + + for section in paper_sections: + # Ensure each section contains sufficient context + enhanced_content = self._enhance_section_with_context(section, content) + + segment = self._create_enhanced_segment( + enhanced_content["content"], + enhanced_content["title"], + enhanced_content["start_pos"], + enhanced_content["end_pos"], + enhanced_content["importance_score"], + enhanced_content["content_type"], + ) + segments.append(segment) + + return segments + + def _segment_concept_implementation_hybrid( + self, content: str + ) -> List[DocumentSegment]: + """Intelligent segmentation combining concepts and implementation""" + segments = [] + + # Identify concept-implementation correspondence + concept_impl_pairs = self._identify_concept_implementation_pairs(content) + + for pair in concept_impl_pairs: + # Merge related concepts and implementations into one segment + merged_content = self._merge_concept_with_implementation(pair, content) + + segment = self._create_enhanced_segment( + merged_content["content"], + merged_content["title"], + merged_content["start_pos"], + merged_content["end_pos"], + merged_content["importance_score"], + merged_content["content_type"], + ) + segments.append(segment) + + return segments + + def _segment_by_enhanced_semantic_chunks( + self, content: str + ) -> List[DocumentSegment]: + """Enhanced semantic chunk segmentation""" + segments = [] + + # Use improved semantic boundary detection + semantic_boundaries = self._detect_semantic_boundaries(content) + + current_start = 0 + for i, boundary in enumerate(semantic_boundaries): + chunk_content = content[current_start : boundary["position"]] + + if len(chunk_content.strip()) > 200: # Minimum content threshold + segment = self._create_enhanced_segment( + chunk_content, + boundary["suggested_title"], + current_start, + boundary["position"], + boundary["importance_score"], + boundary["content_type"], + ) + segments.append(segment) + + current_start = boundary["position"] + + # Handle the final segment + if current_start < len(content): + final_content = content[current_start:] + if len(final_content.strip()) > 200: + segment = self._create_enhanced_segment( + final_content, + "Final Section", + current_start, + len(content), + 0.7, + "general", + ) + segments.append(segment) + + return segments + + def _segment_content_aware(self, content: str) -> List[DocumentSegment]: + """Content-aware intelligent segmentation""" + segments = [] + + # Adaptive segmentation size + optimal_chunk_size = self._calculate_optimal_chunk_size(content) + + # Segment based on content density + content_chunks = self._create_content_aware_chunks(content, optimal_chunk_size) + + for chunk in content_chunks: + segment = self._create_enhanced_segment( + chunk["content"], + chunk["title"], + chunk["start_pos"], + chunk["end_pos"], + chunk["importance_score"], + chunk["content_type"], + ) + segments.append(segment) + + return segments + + def _segment_academic_paper(self, content: str) -> List[DocumentSegment]: + """Segment academic paper using semantic understanding""" + # First try header-based segmentation + headers = re.findall(r"^(#{1,6})\s+(.+)$", content, re.MULTILINE) + if len(headers) >= 2: + return self._segment_by_headers(content) + + # Fallback to semantic detection of academic sections + sections = self._detect_academic_sections(content) + segments = [] + + for section in sections: + # Determine importance based on section type + section_type = section.get("type", "general") + content_type = ( + section_type + if section_type + in ["algorithm", "formula", "introduction", "conclusion"] + else "general" + ) + importance_score = { + "algorithm": 0.95, + "formula": 0.9, + "introduction": 0.85, + "conclusion": 0.8, + }.get(content_type, 0.7) + + segment = self._create_enhanced_segment( + section["content"], + section["title"], + section["start_pos"], + section["end_pos"], + importance_score, + content_type, + ) + segments.append(segment) + + return segments + + def _detect_academic_sections(self, content: str) -> List[Dict]: + """Detect academic paper sections even without clear headers""" + sections = [] + + # Common academic section patterns + section_patterns = [ + (r"(?i)(abstract|ๆ‘˜่ฆ)", "introduction"), + (r"(?i)(introduction|ๅผ•่จ€|็ฎ€ไป‹)", "introduction"), + (r"(?i)(related work|็›ธๅ…ณๅทฅไฝœ|่ƒŒๆ™ฏ)", "background"), + (r"(?i)(method|methodology|approach|ๆ–นๆณ•)", "methodology"), + (r"(?i)(algorithm|็ฎ—ๆณ•)", "algorithm"), + (r"(?i)(experiment|ๅฎž้ชŒ|evaluation|่ฏ„ไผฐ)", "experiment"), + (r"(?i)(result|็ป“ๆžœ|finding)", "results"), + (r"(?i)(conclusion|็ป“่ฎบ|ๆ€ป็ป“)", "conclusion"), + (r"(?i)(reference|ๅ‚่€ƒๆ–‡็Œฎ|bibliography)", "references"), + ] + + current_pos = 0 + for i, (pattern, section_type) in enumerate(section_patterns): + match = re.search(pattern, content[current_pos:], re.IGNORECASE) + if match: + start_pos = current_pos + match.start() + + # Find end position (next section or end of document) + next_pos = len(content) + for next_pattern, _ in section_patterns[i + 1 :]: + next_match = re.search( + next_pattern, content[start_pos + 100 :], re.IGNORECASE + ) + if next_match: + next_pos = start_pos + 100 + next_match.start() + break + + section_content = content[start_pos:next_pos].strip() + if len(section_content) > 50: # Minimum content length + # Calculate importance score and content type + importance_score = self._calculate_paragraph_importance( + section_content, section_type + ) + content_type = self._classify_content_type( + match.group(1), section_content + ) + + sections.append( + { + "title": match.group(1), + "content": section_content, + "start_pos": start_pos, + "end_pos": next_pos, + "type": section_type, + "importance_score": importance_score, + "content_type": content_type, + } + ) + + current_pos = next_pos + + return sections + + def _segment_by_semantic_chunks(self, content: str) -> List[DocumentSegment]: + """Segment long documents into semantic chunks""" + # Split into paragraphs first + paragraphs = [p.strip() for p in content.split("\n\n") if p.strip()] + + segments = [] + current_chunk = [] + current_chunk_size = 0 + chunk_size_limit = 3000 # characters + overlap_size = 200 + + char_pos = 0 + + for para in paragraphs: + para_size = len(para) + + # If adding this paragraph exceeds limit, create a segment + if current_chunk_size + para_size > chunk_size_limit and current_chunk: + chunk_content = "\n\n".join(current_chunk) + # Analyze semantic chunk content type + content_type = self._classify_paragraph_type(chunk_content) + importance_score = self._calculate_paragraph_importance( + chunk_content, content_type + ) + + segment = self._create_enhanced_segment( + chunk_content, + f"Section {len(segments) + 1}", + char_pos - len(chunk_content.encode("utf-8")), + char_pos, + importance_score, + content_type, + ) + segments.append(segment) + + # Keep last part for overlap + overlap_content = ( + chunk_content[-overlap_size:] + if len(chunk_content) > overlap_size + else "" + ) + current_chunk = [overlap_content, para] if overlap_content else [para] + current_chunk_size = len(overlap_content) + para_size + else: + current_chunk.append(para) + current_chunk_size += para_size + + char_pos += para_size + 2 # +2 for \n\n + + # Add final chunk + if current_chunk: + chunk_content = "\n\n".join(current_chunk) + # Analyze final chunk content type + content_type = self._classify_paragraph_type(chunk_content) + importance_score = self._calculate_paragraph_importance( + chunk_content, content_type + ) + + segment = self._create_enhanced_segment( + chunk_content, + f"Section {len(segments) + 1}", + char_pos - len(chunk_content.encode("utf-8")), + char_pos, + importance_score, + content_type, + ) + segments.append(segment) + + return segments + + def _segment_by_paragraphs(self, content: str) -> List[DocumentSegment]: + """Simple paragraph-based segmentation for short documents""" + paragraphs = [p.strip() for p in content.split("\n\n") if p.strip()] + segments = [] + char_pos = 0 + + for i, para in enumerate(paragraphs): + if len(para) > 100: # Only include substantial paragraphs + # Analyze paragraph type and importance + content_type = self._classify_paragraph_type(para) + importance_score = self._calculate_paragraph_importance( + para, content_type + ) + + segment = self._create_enhanced_segment( + para, + f"Paragraph {i + 1}", + char_pos, + char_pos + len(para.encode("utf-8")), + importance_score, + content_type, + ) + segments.append(segment) + char_pos += len(para.encode("utf-8")) + 2 + + return segments + + # =============== Enhanced intelligent segmentation helper methods =============== + + def _identify_algorithm_blocks(self, content: str) -> List[Dict]: + """Identify algorithm blocks and related descriptions""" + algorithm_blocks = [] + + # Algorithm block identification patterns + algorithm_patterns = [ + r"(?i)(algorithm\s+\d+|procedure\s+\d+|method\s+\d+).*?(?=algorithm\s+\d+|procedure\s+\d+|method\s+\d+|$)", + r"(?i)(input:|output:|returns?:|require:|ensure:).*?(?=\n\s*\n|\n\s*(?:input:|output:|returns?:|require:|ensure:)|$)", + r"(?i)(for\s+each|while|if.*then|repeat.*until).*?(?=\n\s*\n|$)", + r"(?i)(step\s+\d+|phase\s+\d+).*?(?=step\s+\d+|phase\s+\d+|\n\s*\n|$)", + ] + + for pattern in algorithm_patterns: + matches = re.finditer(pattern, content, re.DOTALL) + for match in matches: + # Expand context to include complete descriptions + start = max(0, match.start() - 300) + end = min(len(content), match.end() + 500) + + # Find natural boundaries + while start > 0 and content[start] not in "\n.!?": + start -= 1 + while end < len(content) and content[end] not in "\n.!?": + end += 1 + + algorithm_blocks.append( + { + "start_pos": start, + "end_pos": end, + "content": content[start:end].strip(), + "title": self._extract_algorithm_title( + content[match.start() : match.end()] + ), + "importance_score": 0.95, # High importance for algorithm blocks + "content_type": "algorithm", + } + ) + + return algorithm_blocks + + def _identify_concept_groups(self, content: str) -> List[Dict]: + """Identify concept definition groups""" + concept_groups = [] + + # Concept definition patterns + concept_patterns = [ + r"(?i)(definition|define|let|denote|given).*?(?=\n\s*\n|definition|define|let|denote|$)", + r"(?i)(theorem|lemma|proposition|corollary).*?(?=\n\s*\n|theorem|lemma|proposition|corollary|$)", + r"(?i)(notation|symbol|parameter).*?(?=\n\s*\n|notation|symbol|parameter|$)", + ] + + for pattern in concept_patterns: + matches = re.finditer(pattern, content, re.DOTALL) + for match in matches: + # Expand context + start = max(0, match.start() - 200) + end = min(len(content), match.end() + 300) + + concept_groups.append( + { + "start_pos": start, + "end_pos": end, + "content": content[start:end].strip(), + "title": self._extract_concept_title( + content[match.start() : match.end()] + ), + "importance_score": 0.85, + "content_type": "concept", + } + ) + + return concept_groups + + def _identify_formula_chains(self, content: str) -> List[Dict]: + """Identify formula derivation chains""" + formula_chains = [] + + # Formula patterns + formula_patterns = [ + r"\$\$.*?\$\$", # Block-level mathematical formulas + r"\$[^$]+\$", # Inline mathematical formulas + r"(?i)(equation|formula).*?(?=\n\s*\n|equation|formula|$)", + r"(?i)(where|such that|given that).*?(?=\n\s*\n|where|such that|given that|$)", + ] + + # Find dense formula regions + formula_positions = [] + for pattern in formula_patterns: + matches = re.finditer(pattern, content, re.DOTALL) + for match in matches: + formula_positions.append((match.start(), match.end())) + + # Merge nearby formulas into formula chains + formula_positions.sort() + if formula_positions: + current_chain_start = formula_positions[0][0] + current_chain_end = formula_positions[0][1] + + for start, end in formula_positions[1:]: + if ( + start - current_chain_end < 500 + ): # Merge formulas within 500 characters + current_chain_end = end + else: + # Save current chain + formula_chains.append( + { + "start_pos": max(0, current_chain_start - 200), + "end_pos": min(len(content), current_chain_end + 200), + "content": content[ + max(0, current_chain_start - 200) : min( + len(content), current_chain_end + 200 + ) + ].strip(), + "title": "Mathematical Formulation", + "importance_score": 0.9, + "content_type": "formula", + } + ) + current_chain_start = start + current_chain_end = end + + # Add the last chain + formula_chains.append( + { + "start_pos": max(0, current_chain_start - 200), + "end_pos": min(len(content), current_chain_end + 200), + "content": content[ + max(0, current_chain_start - 200) : min( + len(content), current_chain_end + 200 + ) + ].strip(), + "title": "Mathematical Formulation", + "importance_score": 0.9, + "content_type": "formula", + } + ) + + return formula_chains + + def _merge_related_content_blocks( + self, + algorithm_blocks: List[Dict], + concept_groups: List[Dict], + formula_chains: List[Dict], + content: str, + ) -> List[Dict]: + """Merge related content blocks to ensure integrity""" + all_blocks = algorithm_blocks + concept_groups + formula_chains + all_blocks.sort(key=lambda x: x["start_pos"]) + + merged_blocks = [] + i = 0 + + while i < len(all_blocks): + current_block = all_blocks[i] + + # Check if can merge with the next block + while i + 1 < len(all_blocks): + next_block = all_blocks[i + 1] + + # If blocks are close or content related, merge them + if next_block["start_pos"] - current_block[ + "end_pos" + ] < 300 or self._are_blocks_related(current_block, next_block): + # Merge blocks + merged_content = content[ + current_block["start_pos"] : next_block["end_pos"] + ] + current_block = { + "start_pos": current_block["start_pos"], + "end_pos": next_block["end_pos"], + "content": merged_content.strip(), + "title": f"{current_block['title']} & {next_block['title']}", + "importance_score": max( + current_block["importance_score"], + next_block["importance_score"], + ), + "content_type": "merged", + } + i += 1 + else: + break + + merged_blocks.append(current_block) + i += 1 + + return merged_blocks + + def _are_blocks_related(self, block1: Dict, block2: Dict) -> bool: + """Determine if two content blocks are related""" + # Check content type associations + related_types = [ + ("algorithm", "formula"), + ("concept", "algorithm"), + ("formula", "concept"), + ] + + for type1, type2 in related_types: + if ( + block1["content_type"] == type1 and block2["content_type"] == type2 + ) or (block1["content_type"] == type2 and block2["content_type"] == type1): + return True + + return False + + def _extract_algorithm_title(self, text: str) -> str: + """Extract title from algorithm text""" + lines = text.split("\n")[:3] # First 3 lines + for line in lines: + line = line.strip() + if line and len(line) < 100: # Reasonable title length + # Clean title + title = re.sub(r"[^\w\s-]", "", line) + if title: + return title[:50] # Limit title length + return "Algorithm Block" + + def _extract_concept_title(self, text: str) -> str: + """Extract title from concept text""" + lines = text.split("\n")[:2] + for line in lines: + line = line.strip() + if line and len(line) < 80: + title = re.sub(r"[^\w\s-]", "", line) + if title: + return title[:50] + return "Concept Definition" + + def _create_enhanced_segment( + self, + content: str, + title: str, + start_pos: int, + end_pos: int, + importance_score: float, + content_type: str, + ) -> DocumentSegment: + """Create enhanced document segment""" + # Generate unique ID + segment_id = hashlib.md5( + f"{title}_{start_pos}_{end_pos}_{importance_score}".encode() + ).hexdigest()[:8] + + # Extract keywords + keywords = self._extract_enhanced_keywords(content, content_type) + + # Calculate enhanced relevance scores + relevance_scores = self._calculate_enhanced_relevance_scores( + content, content_type, importance_score + ) + + return DocumentSegment( + id=segment_id, + title=title, + content=content, + content_type=content_type, + keywords=keywords, + char_start=start_pos, + char_end=end_pos, + char_count=len(content), + relevance_scores=relevance_scores, + section_path=title, + ) + + def _extract_enhanced_keywords(self, content: str, content_type: str) -> List[str]: + """Extract enhanced keywords based on content type""" + words = re.findall(r"\b[a-zA-Z]{3,}\b", content.lower()) + + # Adjust stopwords based on content type + if content_type == "algorithm": + algorithm_stopwords = { + "step", + "then", + "else", + "end", + "begin", + "start", + "stop", + } + words = [w for w in words if w not in algorithm_stopwords] + elif content_type == "formula": + formula_keywords = ["equation", "formula", "where", "given", "such", "that"] + words.extend(formula_keywords) + + # General stopwords + general_stopwords = { + "the", + "and", + "for", + "are", + "but", + "not", + "you", + "all", + "can", + "her", + "was", + "one", + "our", + "had", + "but", + "have", + "this", + "that", + "with", + "from", + "they", + "she", + "been", + "were", + "said", + "each", + "which", + "their", + } + + keywords = [w for w in set(words) if w not in general_stopwords and len(w) > 3] + return keywords[:25] # Increase keyword count + + def _calculate_enhanced_relevance_scores( + self, content: str, content_type: str, importance_score: float + ) -> Dict[str, float]: + """Calculate enhanced relevance scores""" + content_lower = content.lower() + + base_scores = { + "concept_analysis": 0.5, + "algorithm_extraction": 0.5, + "code_planning": 0.5, + } + + # Adjust base scores based on content type and importance + if content_type == "algorithm": + base_scores["algorithm_extraction"] = importance_score + base_scores["code_planning"] = importance_score * 0.9 + base_scores["concept_analysis"] = importance_score * 0.7 + elif content_type == "concept": + base_scores["concept_analysis"] = importance_score + base_scores["algorithm_extraction"] = importance_score * 0.8 + base_scores["code_planning"] = importance_score * 0.6 + elif content_type == "formula": + base_scores["algorithm_extraction"] = importance_score + base_scores["concept_analysis"] = importance_score * 0.8 + base_scores["code_planning"] = importance_score * 0.9 + elif content_type == "merged": + # Merged content is usually important + base_scores = {k: importance_score * 0.95 for k in base_scores} + + # Additional bonus based on content density + algorithm_indicators = ["algorithm", "method", "procedure", "step", "process"] + concept_indicators = ["definition", "concept", "framework", "approach"] + implementation_indicators = ["implementation", "code", "function", "design"] + + for query_type, indicators in [ + ("algorithm_extraction", algorithm_indicators), + ("concept_analysis", concept_indicators), + ("code_planning", implementation_indicators), + ]: + density_bonus = ( + sum(1 for indicator in indicators if indicator in content_lower) * 0.1 + ) + base_scores[query_type] = min(1.0, base_scores[query_type] + density_bonus) + + return base_scores + + # Placeholder methods - can be further implemented later + def _identify_research_paper_sections(self, content: str) -> List[Dict]: + """Identify research paper sections - simplified implementation""" + # Temporarily use improved semantic detection + return self._detect_academic_sections(content) + + def _enhance_section_with_context(self, section: Dict, content: str) -> Dict: + """Add context to sections - simplified implementation""" + return section + + def _identify_concept_implementation_pairs(self, content: str) -> List[Dict]: + """Identify concept-implementation pairs - simplified implementation""" + return [] + + def _merge_concept_with_implementation(self, pair: Dict, content: str) -> Dict: + """Merge concepts with implementation - simplified implementation""" + return pair + + def _detect_semantic_boundaries(self, content: str) -> List[Dict]: + """Detect semantic boundaries - based on paragraphs and logical separators""" + boundaries = [] + + # Split paragraphs by double line breaks + paragraphs = content.split("\n\n") + current_pos = 0 + + for i, para in enumerate(paragraphs): + if len(para.strip()) > 100: # Valid paragraph + # Analyze paragraph type + content_type = self._classify_paragraph_type(para) + importance_score = self._calculate_paragraph_importance( + para, content_type + ) + + boundaries.append( + { + "position": current_pos + len(para), + "suggested_title": self._extract_paragraph_title(para, i + 1), + "importance_score": importance_score, + "content_type": content_type, + } + ) + + current_pos += len(para) + 2 # +2 for \n\n + + return boundaries + + def _classify_paragraph_type(self, paragraph: str) -> str: + """Classify paragraph type""" + para_lower = paragraph.lower() + + if "algorithm" in para_lower or "procedure" in para_lower: + return "algorithm" + elif "formula" in para_lower or "$$" in paragraph: + return "formula" + elif any( + word in para_lower for word in ["introduction", "overview", "abstract"] + ): + return "introduction" + elif any(word in para_lower for word in ["conclusion", "summary", "result"]): + return "conclusion" + else: + return "general" + + def _calculate_paragraph_importance( + self, paragraph: str, content_type: str + ) -> float: + """Calculate paragraph importance""" + if content_type == "algorithm": + return 0.95 + elif content_type == "formula": + return 0.9 + elif content_type == "introduction": + return 0.85 + elif content_type == "conclusion": + return 0.8 + else: + return 0.7 + + def _extract_paragraph_title(self, paragraph: str, index: int) -> str: + """Extract paragraph title""" + lines = paragraph.split("\n") + for line in lines[:2]: + if line.startswith("#"): + return line.strip("# ") + elif len(line) < 80 and line.strip(): + return line.strip() + return f"Section {index}" + + def _calculate_optimal_chunk_size(self, content: str) -> int: + """Calculate optimal chunk size""" + # Dynamically adjust based on content complexity + complexity = self.analyzer._calculate_concept_complexity(content) + if complexity > 0.7: + return 4000 # Complex content needs larger chunks + elif complexity > 0.4: + return 3000 + else: + return 2000 + + def _create_content_aware_chunks(self, content: str, chunk_size: int) -> List[Dict]: + """Create content-aware chunks - simplified implementation""" + chunks = [] + paragraphs = [p.strip() for p in content.split("\n\n") if p.strip()] + + current_chunk = [] + current_size = 0 + start_pos = 0 + + for para in paragraphs: + para_size = len(para) + + if current_size + para_size > chunk_size and current_chunk: + chunk_content = "\n\n".join(current_chunk) + chunks.append( + { + "content": chunk_content, + "title": f"Section {len(chunks) + 1}", + "start_pos": start_pos, + "end_pos": start_pos + len(chunk_content), + "importance_score": 0.7, + "content_type": "general", + } + ) + + current_chunk = [para] + current_size = para_size + start_pos += len(chunk_content) + 2 + else: + current_chunk.append(para) + current_size += para_size + + # Add the last chunk + if current_chunk: + chunk_content = "\n\n".join(current_chunk) + chunks.append( + { + "content": chunk_content, + "title": f"Section {len(chunks) + 1}", + "start_pos": start_pos, + "end_pos": start_pos + len(chunk_content), + "importance_score": 0.7, + "content_type": "general", + } + ) + + return chunks + + def _create_segment( + self, content: str, title: str, start_pos: int, end_pos: int + ) -> DocumentSegment: + """Create a DocumentSegment with metadata""" + # Generate unique ID + segment_id = hashlib.md5(f"{title}_{start_pos}_{end_pos}".encode()).hexdigest()[ + :8 + ] + + # Extract keywords from content + keywords = self._extract_keywords(content) + + # Determine content type + content_type = self._classify_content_type(title, content) + + # Calculate relevance scores for different query types + relevance_scores = self._calculate_relevance_scores(content, content_type) + + return DocumentSegment( + id=segment_id, + title=title, + content=content, + content_type=content_type, + keywords=keywords, + char_start=start_pos, + char_end=end_pos, + char_count=len(content), + relevance_scores=relevance_scores, + section_path=title, # Simplified for now + ) + + def _extract_keywords(self, content: str) -> List[str]: + """Extract relevant keywords from content""" + # Simple keyword extraction - could be enhanced with NLP + words = re.findall(r"\b[a-zA-Z]{3,}\b", content.lower()) + + # Remove common words + stopwords = { + "the", + "and", + "for", + "are", + "but", + "not", + "you", + "all", + "can", + "her", + "was", + "one", + "our", + "had", + "but", + "have", + "this", + "that", + "with", + "from", + "they", + "she", + "been", + "were", + "said", + "each", + "which", + "their", + } + + keywords = [w for w in set(words) if w not in stopwords and len(w) > 3] + return keywords[:20] # Top 20 keywords + + def _classify_content_type(self, title: str, content: str) -> str: + """Classify the type of content based on title and content""" + title_lower = title.lower() + content_lower = content.lower() + + if any( + word in title_lower for word in ["introduction", "abstract", "overview"] + ): + return "introduction" + elif any(word in title_lower for word in ["method", "approach", "algorithm"]): + return "methodology" + elif any( + word in title_lower for word in ["experiment", "evaluation", "result"] + ): + return "experiment" + elif any( + word in title_lower for word in ["conclusion", "discussion", "summary"] + ): + return "conclusion" + elif any(word in title_lower for word in ["reference", "bibliography"]): + return "references" + elif "algorithm" in content_lower or "procedure" in content_lower: + return "algorithm" + else: + return "general" + + def _calculate_relevance_scores( + self, content: str, content_type: str + ) -> Dict[str, float]: + """Calculate relevance scores for different query types""" + content_lower = content.lower() + + scores = { + "concept_analysis": 0.5, + "algorithm_extraction": 0.5, + "code_planning": 0.5, + } + + # Concept analysis relevance + concept_indicators = [ + "introduction", + "overview", + "architecture", + "system", + "framework", + "concept", + "approach", + ] + concept_score = sum( + 1 for indicator in concept_indicators if indicator in content_lower + ) / len(concept_indicators) + scores["concept_analysis"] = min( + 1.0, concept_score + (0.8 if content_type == "introduction" else 0) + ) + + # Algorithm extraction relevance + algorithm_indicators = [ + "algorithm", + "method", + "procedure", + "formula", + "equation", + "step", + "process", + ] + algorithm_score = sum( + 1 for indicator in algorithm_indicators if indicator in content_lower + ) / len(algorithm_indicators) + scores["algorithm_extraction"] = min( + 1.0, algorithm_score + (0.9 if content_type == "methodology" else 0) + ) + + # Code planning relevance + code_indicators = [ + "implementation", + "code", + "function", + "class", + "module", + "structure", + "design", + ] + code_score = sum( + 1 for indicator in code_indicators if indicator in content_lower + ) / len(code_indicators) + scores["code_planning"] = min( + 1.0, + code_score + (0.7 if content_type in ["methodology", "algorithm"] else 0), + ) + + return scores + + +# Global variables +DOCUMENT_INDEXES: Dict[str, DocumentIndex] = {} +segmenter = DocumentSegmenter() + + +def get_segments_dir(paper_dir: str) -> str: + """Get the segments directory path""" + return os.path.join(paper_dir, "document_segments") + + +def ensure_segments_dir_exists(segments_dir: str): + """Ensure segments directory exists""" + os.makedirs(segments_dir, exist_ok=True) + + +@mcp.tool() +async def analyze_and_segment_document( + paper_dir: str, force_refresh: bool = False +) -> str: + """ + Analyze document structure and create intelligent segments + + Args: + paper_dir: Path to the paper directory + force_refresh: Whether to force re-analysis even if segments exist + + Returns: + JSON string with segmentation results + """ + try: + # Find markdown file in paper directory + md_files = [f for f in os.listdir(paper_dir) if f.endswith(".md")] + if not md_files: + return json.dumps( + { + "status": "error", + "message": f"No markdown file found in {paper_dir}", + }, + ensure_ascii=False, + indent=2, + ) + + md_file_path = os.path.join(paper_dir, md_files[0]) + segments_dir = get_segments_dir(paper_dir) + index_file_path = os.path.join(segments_dir, "document_index.json") + + # Check if analysis already exists and is recent + if not force_refresh and os.path.exists(index_file_path): + try: + with open(index_file_path, "r", encoding="utf-8") as f: + existing_index = json.load(f) + + # Compatibility handling: ensure segments data structure is correct + if "segments" in existing_index: + segments_data = [] + for seg_data in existing_index["segments"]: + # Ensure all required fields exist + segment_dict = dict(seg_data) + + if "content_type" not in segment_dict: + segment_dict["content_type"] = "general" + if "keywords" not in segment_dict: + segment_dict["keywords"] = [] + if "relevance_scores" not in segment_dict: + segment_dict["relevance_scores"] = { + "concept_analysis": 0.5, + "algorithm_extraction": 0.5, + "code_planning": 0.5, + } + if "section_path" not in segment_dict: + segment_dict["section_path"] = segment_dict.get( + "title", "Unknown" + ) + + segments_data.append(DocumentSegment(**segment_dict)) + + existing_index["segments"] = segments_data + + DOCUMENT_INDEXES[paper_dir] = DocumentIndex(**existing_index) + return json.dumps( + { + "status": "success", + "message": "Using existing document analysis", + "segments_dir": segments_dir, + "total_segments": existing_index["total_segments"], + }, + ensure_ascii=False, + indent=2, + ) + + except Exception as e: + logger.error(f"Failed to load existing index: {e}") + logger.info("Will perform fresh analysis instead") + # Remove corrupted index file and continue with new analysis + try: + os.remove(index_file_path) + except Exception as e: + pass + + # Read document content + with open(md_file_path, "r", encoding="utf-8") as f: + content = f.read() + + # Analyze document + analyzer = DocumentAnalyzer() + doc_type, confidence = analyzer.analyze_document_type(content) + strategy = analyzer.detect_segmentation_strategy(content, doc_type) + + # Create segments + segments = segmenter.segment_document(content, strategy) + + # Create document index + document_index = DocumentIndex( + document_path=md_file_path, + document_type=doc_type, + segmentation_strategy=strategy, + total_segments=len(segments), + total_chars=len(content), + segments=segments, + created_at=datetime.now().isoformat(), + ) + + # Save segments + ensure_segments_dir_exists(segments_dir) + + # Save document index + with open(index_file_path, "w", encoding="utf-8") as f: + json.dump( + asdict(document_index), f, ensure_ascii=False, indent=2, default=str + ) + + # Save individual segment files for fallback + for segment in segments: + segment_file_path = os.path.join(segments_dir, f"segment_{segment.id}.md") + with open(segment_file_path, "w", encoding="utf-8") as f: + f.write(f"# {segment.title}\n\n") + f.write(f"**Content Type:** {segment.content_type}\n") + f.write(f"**Keywords:** {', '.join(segment.keywords[:10])}\n\n") + f.write(segment.content) + + # Store in memory + DOCUMENT_INDEXES[paper_dir] = document_index + + logger.info( + f"Document segmentation completed: {len(segments)} segments created" + ) + + return json.dumps( + { + "status": "success", + "message": f"Document analysis completed with {strategy} strategy", + "document_type": doc_type, + "segmentation_strategy": strategy, + "segments_dir": segments_dir, + "total_segments": len(segments), + "total_chars": len(content), + }, + ensure_ascii=False, + indent=2, + ) + + except Exception as e: + logger.error(f"Error in analyze_and_segment_document: {e}") + return json.dumps( + {"status": "error", "message": f"Failed to analyze document: {str(e)}"}, + ensure_ascii=False, + indent=2, + ) + + +@mcp.tool() +async def read_document_segments( + paper_dir: str, + query_type: str, + keywords: List[str] = None, + max_segments: int = 3, + max_total_chars: int = None, +) -> str: + """ + Intelligently retrieve relevant document segments based on query type + + Args: + paper_dir: Path to the paper directory + query_type: Type of query - "concept_analysis", "algorithm_extraction", or "code_planning" + keywords: Optional list of keywords to search for + max_segments: Maximum number of segments to return + max_total_chars: Maximum total characters to return + + Returns: + JSON string with selected segments + """ + try: + # Ensure document is analyzed + if paper_dir not in DOCUMENT_INDEXES: + segments_dir = get_segments_dir(paper_dir) + index_file_path = os.path.join(segments_dir, "document_index.json") + + if os.path.exists(index_file_path): + with open(index_file_path, "r", encoding="utf-8") as f: + index_data = json.load(f) + # Convert dict back to DocumentIndex with backward compatibility + segments_data = [] + for seg_data in index_data.get("segments", []): + # Ensure all required fields exist, provide default values + segment_dict = dict(seg_data) + + # Compatibility handling: add missing fields + if "content_type" not in segment_dict: + segment_dict["content_type"] = "general" + if "keywords" not in segment_dict: + segment_dict["keywords"] = [] + if "relevance_scores" not in segment_dict: + segment_dict["relevance_scores"] = { + "concept_analysis": 0.5, + "algorithm_extraction": 0.5, + "code_planning": 0.5, + } + if "section_path" not in segment_dict: + segment_dict["section_path"] = segment_dict.get( + "title", "Unknown" + ) + + segment = DocumentSegment(**segment_dict) + segments_data.append(segment) + + index_data["segments"] = segments_data + DOCUMENT_INDEXES[paper_dir] = DocumentIndex(**index_data) + else: + # Auto-analyze if not found + await analyze_and_segment_document(paper_dir) + + document_index = DOCUMENT_INDEXES[paper_dir] + + # Dynamically calculate character limit + if max_total_chars is None: + max_total_chars = _calculate_adaptive_char_limit(document_index, query_type) + + # Score and rank segments with enhanced algorithm + scored_segments = [] + for segment in document_index.segments: + # Base relevance score (already enhanced in new system) + relevance_score = segment.relevance_scores.get(query_type, 0.5) + + # Enhanced keyword matching with position weighting + if keywords: + keyword_score = _calculate_enhanced_keyword_score(segment, keywords) + relevance_score += keyword_score + + # Content completeness bonus + completeness_bonus = _calculate_completeness_bonus(segment, document_index) + relevance_score += completeness_bonus + + scored_segments.append((segment, relevance_score)) + + # Sort by enhanced relevance score + scored_segments.sort(key=lambda x: x[1], reverse=True) + + # Intelligent segment selection with integrity preservation + selected_segments = _select_segments_with_integrity( + scored_segments, max_segments, max_total_chars, query_type + ) + + total_chars = sum(seg["char_count"] for seg in selected_segments) + + logger.info( + f"Selected {len(selected_segments)} segments for {query_type} query" + ) + + return json.dumps( + { + "status": "success", + "query_type": query_type, + "keywords": keywords or [], + "total_segments_available": len(document_index.segments), + "segments_selected": len(selected_segments), + "total_chars": total_chars, + "max_chars_used": max_total_chars, + "segments": selected_segments, + }, + ensure_ascii=False, + indent=2, + ) + + except Exception as e: + logger.error(f"Error in read_document_segments: {e}") + return json.dumps( + { + "status": "error", + "message": f"Failed to read document segments: {str(e)}", + }, + ensure_ascii=False, + indent=2, + ) + + +@mcp.tool() +async def get_document_overview(paper_dir: str) -> str: + """ + Get overview of document structure and available segments + + Args: + paper_dir: Path to the paper directory + + Returns: + JSON string with document overview + """ + try: + # Ensure document is analyzed + if paper_dir not in DOCUMENT_INDEXES: + await analyze_and_segment_document(paper_dir) + + document_index = DOCUMENT_INDEXES[paper_dir] + + # Create overview + segment_summaries = [] + for segment in document_index.segments: + segment_summaries.append( + { + "id": segment.id, + "title": segment.title, + "content_type": segment.content_type, + "char_count": segment.char_count, + "keywords": segment.keywords[:5], # Top 5 keywords + "relevance_scores": segment.relevance_scores, + } + ) + + return json.dumps( + { + "status": "success", + "document_path": document_index.document_path, + "document_type": document_index.document_type, + "segmentation_strategy": document_index.segmentation_strategy, + "total_segments": document_index.total_segments, + "total_chars": document_index.total_chars, + "created_at": document_index.created_at, + "segments_overview": segment_summaries, + }, + ensure_ascii=False, + indent=2, + ) + + except Exception as e: + logger.error(f"Error in get_document_overview: {e}") + return json.dumps( + { + "status": "error", + "message": f"Failed to get document overview: {str(e)}", + }, + ensure_ascii=False, + indent=2, + ) + + +# =============== Enhanced retrieval system helper methods =============== + + +def _calculate_adaptive_char_limit( + document_index: DocumentIndex, query_type: str +) -> int: + """Dynamically calculate character limit based on document complexity and query type""" + base_limit = 6000 + + # Adjust based on document type + if document_index.document_type == "research_paper": + base_limit = 10000 + elif document_index.document_type == "algorithm_focused": + base_limit = 12000 + elif document_index.segmentation_strategy == "algorithm_preserve_integrity": + base_limit = 15000 + + # Adjust based on query type + query_multipliers = { + "algorithm_extraction": 1.5, # Algorithms need more context + "concept_analysis": 1.2, + "code_planning": 1.3, + } + + multiplier = query_multipliers.get(query_type, 1.0) + return int(base_limit * multiplier) + + +def _calculate_enhanced_keyword_score( + segment: DocumentSegment, keywords: List[str] +) -> float: + """Calculate enhanced keyword matching score""" + score = 0.0 + content_lower = segment.content.lower() + title_lower = segment.title.lower() + + for keyword in keywords: + keyword_lower = keyword.lower() + + # Title matching has higher weight + if keyword_lower in title_lower: + score += 0.3 + + # Content matching + content_matches = content_lower.count(keyword_lower) + if content_matches > 0: + # Consider term frequency and position + frequency_score = min(0.2, content_matches * 0.05) + + # Check if in important position (first 25% of content) + early_content = content_lower[: len(content_lower) // 4] + if keyword_lower in early_content: + frequency_score += 0.1 + + score += frequency_score + + return min(0.6, score) # Limit maximum bonus + + +def _calculate_completeness_bonus( + segment: DocumentSegment, document_index: DocumentIndex +) -> float: + """Calculate content completeness bonus""" + bonus = 0.0 + + # Completeness bonus for algorithm and formula content + if segment.content_type in ["algorithm", "formula", "merged"]: + bonus += 0.2 + + # Long paragraphs usually contain more complete information + if segment.char_count > 2000: + bonus += 0.1 + elif segment.char_count > 4000: + bonus += 0.15 + + # High importance paragraph bonus + if segment.relevance_scores.get("algorithm_extraction", 0) > 0.8: + bonus += 0.1 + + return min(0.3, bonus) + + +def _select_segments_with_integrity( + scored_segments: List[Tuple], + max_segments: int, + max_total_chars: int, + query_type: str, +) -> List[Dict]: + """Intelligently select segments while maintaining content integrity""" + selected_segments = [] + total_chars = 0 + + # First select the highest scoring segments + for segment, score in scored_segments: + if len(selected_segments) >= max_segments: + break + + if total_chars + segment.char_count <= max_total_chars: + selected_segments.append( + { + "id": segment.id, + "title": segment.title, + "content": segment.content, + "content_type": segment.content_type, + "relevance_score": score, + "char_count": segment.char_count, + } + ) + total_chars += segment.char_count + elif len(selected_segments) == 0: + # If the first segment exceeds the limit, truncate but preserve it + truncated_content = ( + segment.content[: max_total_chars - 200] + + "\n\n[Content truncated for length...]" + ) + selected_segments.append( + { + "id": segment.id, + "title": segment.title, + "content": truncated_content, + "content_type": segment.content_type, + "relevance_score": score, + "char_count": len(truncated_content), + } + ) + break + + # If there's remaining space, try to add relevant small segments + remaining_chars = max_total_chars - total_chars + if remaining_chars > 500 and len(selected_segments) < max_segments: + for segment, score in scored_segments[len(selected_segments) :]: + if ( + segment.char_count <= remaining_chars + and len(selected_segments) < max_segments + ): + selected_segments.append( + { + "id": segment.id, + "title": segment.title, + "content": segment.content, + "content_type": segment.content_type, + "relevance_score": score, + "char_count": segment.char_count, + } + ) + remaining_chars -= segment.char_count + + return selected_segments + + +if __name__ == "__main__": + # Run the MCP server + mcp.run() diff --git a/DeepCode/tools/git_command.py b/DeepCode/tools/git_command.py new file mode 100644 index 00000000..8070734d --- /dev/null +++ b/DeepCode/tools/git_command.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +""" +GitHub Repository Downloader MCP Tool using FastMCP +""" + +import asyncio +import os +import re +from typing import Dict, List, Optional +from pathlib import Path + +from mcp.server import FastMCP + +# ๅˆ›ๅปบ FastMCP ๅฎžไพ‹ +mcp = FastMCP("github-downloader") + + +class GitHubURLExtractor: + """ๆๅ–GitHub URL็š„ๅทฅๅ…ท็ฑป""" + + @staticmethod + def extract_github_urls(text: str) -> List[str]: + """ไปŽๆ–‡ๆœฌไธญๆๅ–GitHub URLs""" + patterns = [ + # ๆ ‡ๅ‡†HTTPS URL + r"https?://github\.com/[\w\-\.]+/[\w\-\.]+(?:\.git)?", + # SSH URL + r"git@github\.com:[\w\-\.]+/[\w\-\.]+(?:\.git)?", + # ็Ÿญๆ ผๅผ owner/repo - ๆ›ดไธฅๆ ผ็š„ๅŒน้… + r"(? Optional[str]: + """ไปŽๆ–‡ๆœฌไธญๆๅ–็›ฎๆ ‡่ทฏๅพ„""" + # ่ทฏๅพ„ๆŒ‡็คบ่ฏๆจกๅผ + patterns = [ + r'(?:to|into|in|at)\s+(?:folder|directory|path)?\s*["\']?([^\s"\']+)["\']?', + r'(?:save|download|clone)\s+(?:to|into|at)\s+["\']?([^\s"\']+)["\']?', + # ไธญๆ–‡ๆ”ฏๆŒ + r'(?:ๅˆฐ|ๅœจ|ไฟๅญ˜ๅˆฐ|ไธ‹่ฝฝๅˆฐ|ๅ…‹้š†ๅˆฐ)\s*["\']?([^\s"\']+)["\']?', + ] + + for pattern in patterns: + match = re.search(pattern, text, re.IGNORECASE) + if match: + path = match.group(1).strip("ใ€‚๏ผŒ,.") + # ่ฟ‡ๆปคๆŽ‰้€š็”จ่ฏ + if path and path.lower() not in [ + "here", + "there", + "current", + "local", + "่ฟ™้‡Œ", + "ๅฝ“ๅ‰", + "ๆœฌๅœฐ", + ]: + return path + + return None + + @staticmethod + def infer_repo_name(url: str) -> str: + """ไปŽURLๆŽจๆ–ญไป“ๅบ“ๅ็งฐ""" + url = url.rstrip(".git") + if "github.com" in url: + parts = url.split("/") + if len(parts) >= 2: + return parts[-1] + return "repository" + + +async def check_git_installed() -> bool: + """ๆฃ€ๆŸฅGitๆ˜ฏๅฆๅฎ‰่ฃ…""" + try: + proc = await asyncio.create_subprocess_exec( + "git", + "--version", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + await proc.wait() + return proc.returncode == 0 + except Exception: + return False + + +async def clone_repository(repo_url: str, target_path: str) -> Dict[str, any]: + """ๆ‰ง่กŒgit cloneๅ‘ฝไปค""" + try: + proc = await asyncio.create_subprocess_exec( + "git", + "clone", + repo_url, + target_path, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + stdout, stderr = await proc.communicate() + + return { + "success": proc.returncode == 0, + "stdout": stdout.decode("utf-8", errors="replace"), + "stderr": stderr.decode("utf-8", errors="replace"), + "returncode": proc.returncode, + } + except Exception as e: + return {"success": False, "error": str(e)} + + +@mcp.tool() +async def download_github_repo(instruction: str) -> str: + """ + Download GitHub repositories from natural language instructions. + + Args: + instruction: Natural language text containing GitHub URLs and optional target paths + + Returns: + Status message about the download operation + + Examples: + - "Download https://github.com/openai/gpt-3" + - "Clone microsoft/vscode to my-projects folder" + - "Get https://github.com/facebook/react" + """ + # ๆฃ€ๆŸฅGitๆ˜ฏๅฆๅฎ‰่ฃ… + if not await check_git_installed(): + return "โŒ Error: Git is not installed or not in system PATH" + + extractor = GitHubURLExtractor() + + # ๆๅ–GitHub URLs + urls = extractor.extract_github_urls(instruction) + if not urls: + return "โŒ No GitHub URLs found in the instruction" + + # ๆๅ–็›ฎๆ ‡่ทฏๅพ„ + target_path = extractor.extract_target_path(instruction) + + # ไธ‹่ฝฝไป“ๅบ“ + results = [] + for url in urls: + try: + # ๅ‡†ๅค‡็›ฎๆ ‡่ทฏๅพ„ + if target_path: + # ๅˆคๆ–ญๆ˜ฏๅฆไธบ็ปๅฏน่ทฏๅพ„ + if os.path.isabs(target_path): + # ๅฆ‚ๆžœๆ˜ฏ็ปๅฏน่ทฏๅพ„๏ผŒ็›ดๆŽฅไฝฟ็”จ + final_path = target_path + # ๅฆ‚ๆžœ็›ฎๆ ‡่ทฏๅพ„ๆ˜ฏ็›ฎๅฝ•๏ผŒๆทปๅŠ ไป“ๅบ“ๅ + if os.path.basename(target_path) == "" or target_path.endswith("/"): + final_path = os.path.join( + target_path, extractor.infer_repo_name(url) + ) + else: + # ๅฆ‚ๆžœๆ˜ฏ็›ธๅฏน่ทฏๅพ„๏ผŒไฟๆŒ็›ธๅฏน่ทฏๅพ„ + final_path = target_path + # ๅฆ‚ๆžœ็›ฎๆ ‡่ทฏๅพ„ๆ˜ฏ็›ฎๅฝ•๏ผŒๆทปๅŠ ไป“ๅบ“ๅ + if os.path.basename(target_path) == "" or target_path.endswith("/"): + final_path = os.path.join( + target_path, extractor.infer_repo_name(url) + ) + else: + final_path = extractor.infer_repo_name(url) + + # ๅฆ‚ๆžœๆ˜ฏ็›ธๅฏน่ทฏๅพ„๏ผŒ็กฎไฟไฝฟ็”จ็›ธๅฏน่ทฏๅพ„ๆ ผๅผ + if not os.path.isabs(final_path): + final_path = os.path.normpath(final_path) + if final_path.startswith("/"): + final_path = final_path.lstrip("/") + + # ็กฎไฟ็ˆถ็›ฎๅฝ•ๅญ˜ๅœจ + parent_dir = os.path.dirname(final_path) + if parent_dir: + os.makedirs(parent_dir, exist_ok=True) + + # ๆฃ€ๆŸฅ็›ฎๆ ‡่ทฏๅพ„ๆ˜ฏๅฆๅทฒๅญ˜ๅœจ + if os.path.exists(final_path): + results.append( + f"โŒ Failed to download {url}: Target path already exists: {final_path}" + ) + continue + + # ๆ‰ง่กŒๅ…‹้š† + result = await clone_repository(url, final_path) + + if result["success"]: + msg = f"โœ… Successfully downloaded: {url}\n" + msg += f" Location: {final_path}" + if result.get("stdout"): + msg += f"\n {result['stdout'].strip()}" + else: + msg = f"โŒ Failed to download: {url}\n" + msg += f" Error: {result.get('error', result.get('stderr', 'Unknown error'))}" + + except Exception as e: + msg = f"โŒ Failed to download: {url}\n" + msg += f" Error: {str(e)}" + + results.append(msg) + + return "\n\n".join(results) + + +@mcp.tool() +async def parse_github_urls(text: str) -> str: + """ + Extract GitHub URLs and target paths from text. + + Args: + text: Text containing GitHub URLs + + Returns: + Parsed GitHub URLs and target path information + """ + extractor = GitHubURLExtractor() + + urls = extractor.extract_github_urls(text) + target_path = extractor.extract_target_path(text) + + content = "๐Ÿ“ Parsed information:\n\n" + + if urls: + content += "GitHub URLs found:\n" + for url in urls: + content += f" โ€ข {url}\n" + else: + content += "No GitHub URLs found\n" + + if target_path: + content += f"\nTarget path: {target_path}" + else: + content += "\nTarget path: Not specified (will use repository name)" + + return content + + +@mcp.tool() +async def git_clone( + repo_url: str, target_path: Optional[str] = None, branch: Optional[str] = None +) -> str: + """ + Clone a specific GitHub repository. + + Args: + repo_url: GitHub repository URL + target_path: Optional target directory path + branch: Optional branch name to clone + + Returns: + Status message about the clone operation + """ + # ๆฃ€ๆŸฅGitๆ˜ฏๅฆๅฎ‰่ฃ… + if not await check_git_installed(): + return "โŒ Error: Git is not installed or not in system PATH" + + # ๅ‡†ๅค‡็›ฎๆ ‡่ทฏๅพ„ + if not target_path: + extractor = GitHubURLExtractor() + target_path = extractor.infer_repo_name(repo_url) + + # ่ฝฌๆขไธบ็ปๅฏน่ทฏๅพ„ + if not os.path.isabs(target_path): + target_path = str(Path.cwd() / target_path) + + # ๆฃ€ๆŸฅ็›ฎๆ ‡่ทฏๅพ„ + if os.path.exists(target_path): + return f"โŒ Error: Target path already exists: {target_path}" + + # ๆž„ๅปบๅ‘ฝไปค + cmd = ["git", "clone"] + if branch: + cmd.extend(["-b", branch]) + cmd.extend([repo_url, target_path]) + + # ๆ‰ง่กŒๅ…‹้š† + try: + proc = await asyncio.create_subprocess_exec( + *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + + stdout, stderr = await proc.communicate() + + if proc.returncode == 0: + result = "โœ… Successfully cloned repository\n" + result += f"Repository: {repo_url}\n" + result += f"Location: {target_path}" + if branch: + result += f"\nBranch: {branch}" + return result + else: + return f"โŒ Clone failed\nError: {stderr.decode('utf-8', errors='replace')}" + + except Exception as e: + return f"โŒ Clone failed\nError: {str(e)}" + + +# ไธป็จ‹ๅบๅ…ฅๅฃ +if __name__ == "__main__": + print("๐Ÿš€ GitHub Repository Downloader MCP Tool") + print("๐Ÿ“ Starting server with FastMCP...") + print("\nAvailable tools:") + print(" โ€ข download_github_repo - Download repos from natural language") + print(" โ€ข parse_github_urls - Extract GitHub URLs from text") + print(" โ€ข git_clone - Clone a specific repository") + print("") + + # ่ฟ่กŒๆœๅŠกๅ™จ + mcp.run() diff --git a/DeepCode/tools/indexer_config.yaml b/DeepCode/tools/indexer_config.yaml new file mode 100644 index 00000000..fcc642f4 --- /dev/null +++ b/DeepCode/tools/indexer_config.yaml @@ -0,0 +1,141 @@ +# Code Indexer Configuration File +# Configure various aspects of the code indexing process + +# Paths Configuration +paths: + code_base_path: "D:/Documents/GitHub/Code-Agent/examples/input/paper1/code_base" + output_dir: "D:/Documents/GitHub/Code-Agent/examples/input/paper1/indexes" + +# File Analysis Settings +file_analysis: + # Supported file extensions for analysis + supported_extensions: + - ".py" # Python + - ".js" # JavaScript + - ".ts" # TypeScript + - ".java" # Java + - ".cpp" # C++ + - ".c" # C + - ".h" # C Header + - ".hpp" # C++ Header + - ".cs" # C# + - ".php" # PHP + - ".rb" # Ruby + - ".go" # Go + - ".rs" # Rust + - ".scala" # Scala + - ".kt" # Kotlin + - ".swift" # Swift + - ".r" # R + - ".sql" # SQL + - ".sh" # Shell Script + - ".bat" # Batch File + - ".ps1" # PowerShell + - ".yaml" # YAML + - ".yml" # YAML + - ".json" # JSON + - ".xml" # XML + - ".toml" # TOML + + # Directories to skip during traversal + skip_directories: + - "__pycache__" + - "node_modules" + - "target" + - "build" + - "dist" + - "venv" + - "env" + - ".git" + - ".svn" + - ".hg" + - "coverage" + - ".pytest_cache" + - ".mypy_cache" + + # Maximum file size to analyze (in bytes) + max_file_size: 1048576 # 1MB + + # Maximum content length to send to LLM (in characters) + max_content_length: 3000 + +# LLM Configuration +llm: + # Model selection: "anthropic" or "openai" + model_provider: "openai" + + # Request parameters + max_tokens: 4000 + temperature: 0.3 + + # System prompt for analysis + system_prompt: "You are a code analysis expert. Provide precise, structured analysis of code relationships and similarities." + + # Rate limiting (seconds between requests) + request_delay: 0.1 + + # Retry configuration + max_retries: 3 + retry_delay: 1.0 + +# Relationship Analysis Settings +relationships: + # Minimum confidence score to include a relationship + min_confidence_score: 0.3 + + # High confidence threshold for reporting + high_confidence_threshold: 0.7 + + # Relationship types and their priorities + relationship_types: + direct_match: 1.0 # Direct implementation match + partial_match: 0.8 # Partial functionality match + reference: 0.6 # Reference or utility function + utility: 0.4 # General utility or helper + +# Output Configuration +output: + # JSON formatting options + json_indent: 2 + ensure_ascii: false + + # Generate additional report files + generate_summary: true + generate_statistics: true + + # Include metadata in output + include_metadata: true + + # File naming pattern (use {repo_name} placeholder) + index_filename_pattern: "{repo_name}_index.json" + summary_filename: "indexing_summary.json" + stats_filename: "indexing_statistics.json" + +# Logging Configuration +logging: + level: "INFO" # DEBUG, INFO, WARNING, ERROR + log_to_file: true + log_file: "indexer.log" + log_format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + +# Performance Settings +performance: + # Enable concurrent processing of files within a repository + enable_concurrent_analysis: true + max_concurrent_files: 5 + + # Memory optimization + enable_content_caching: false + max_cache_size: 100 + +# Debug and Development Settings +debug: + # Save raw LLM responses for debugging + save_raw_responses: false + raw_responses_dir: "debug_responses" + + # Verbose output during processing + verbose_output: false + + # Skip LLM calls for testing (uses mock responses) + mock_llm_responses: false diff --git a/DeepCode/tools/pdf_converter.py b/DeepCode/tools/pdf_converter.py new file mode 100644 index 00000000..c9d01e8d --- /dev/null +++ b/DeepCode/tools/pdf_converter.py @@ -0,0 +1,640 @@ +#!/usr/bin/env python3 +""" +PDF Converter Utility + +This module provides functionality for converting various document formats to PDF, +including Office documents (.doc, .docx, .ppt, .pptx, .xls, .xlsx) and text files (.txt, .md). + +Requirements: +- LibreOffice for Office document conversion +- ReportLab for text-to-PDF conversion +""" + +from __future__ import annotations + +import argparse +import logging +import subprocess +import tempfile +import shutil +import platform +from pathlib import Path +from typing import Union, Optional, Dict, Any + + +class PDFConverter: + """ + PDF conversion utility class. + + Provides methods to convert Office documents and text files to PDF format. + """ + + # Define supported file formats + OFFICE_FORMATS = {".doc", ".docx", ".ppt", ".pptx", ".xls", ".xlsx"} + TEXT_FORMATS = {".txt", ".md"} + + # Class-level logger + logger = logging.getLogger(__name__) + + def __init__(self) -> None: + """Initialize the PDF converter.""" + pass + + @staticmethod + def convert_office_to_pdf( + doc_path: Union[str, Path], output_dir: Optional[str] = None + ) -> Path: + """ + Convert Office document (.doc, .docx, .ppt, .pptx, .xls, .xlsx) to PDF. + Requires LibreOffice to be installed. + + Args: + doc_path: Path to the Office document file + output_dir: Output directory for the PDF file + + Returns: + Path to the generated PDF file + """ + try: + # Convert to Path object for easier handling + doc_path = Path(doc_path) + if not doc_path.exists(): + raise FileNotFoundError(f"Office document does not exist: {doc_path}") + + name_without_suff = doc_path.stem + + # Prepare output directory + if output_dir: + base_output_dir = Path(output_dir) + else: + base_output_dir = doc_path.parent / "pdf_output" + + base_output_dir.mkdir(parents=True, exist_ok=True) + + # Check if LibreOffice is available + libreoffice_available = False + working_libreoffice_cmd: Optional[str] = None + + # Prepare subprocess parameters to hide console window on Windows + subprocess_kwargs: Dict[str, Any] = { + "capture_output": True, + "check": True, + "timeout": 10, + "encoding": "utf-8", + "errors": "ignore", + } + + # Hide console window on Windows + if platform.system() == "Windows": + subprocess_kwargs["creationflags"] = ( + 0x08000000 # subprocess.CREATE_NO_WINDOW + ) + + try: + result = subprocess.run( + ["libreoffice", "--version"], **subprocess_kwargs + ) + libreoffice_available = True + working_libreoffice_cmd = "libreoffice" + logging.info(f"LibreOffice detected: {result.stdout.strip()}") # type: ignore + except ( + subprocess.CalledProcessError, + FileNotFoundError, + subprocess.TimeoutExpired, + ): + pass + + # Try alternative commands for LibreOffice + if not libreoffice_available: + for cmd in ["soffice", "libreoffice"]: + try: + result = subprocess.run([cmd, "--version"], **subprocess_kwargs) + libreoffice_available = True + working_libreoffice_cmd = cmd + logging.info( + f"LibreOffice detected with command '{cmd}': {result.stdout.strip()}" # type: ignore + ) + break + except ( + subprocess.CalledProcessError, + FileNotFoundError, + subprocess.TimeoutExpired, + ): + continue + + if not libreoffice_available: + raise RuntimeError( + "LibreOffice is required for Office document conversion but was not found.\n" + "Please install LibreOffice:\n" + "- Windows: Download from https://www.libreoffice.org/download/download/\n" + "- macOS: brew install --cask libreoffice\n" + "- Ubuntu/Debian: sudo apt-get install libreoffice\n" + "- CentOS/RHEL: sudo yum install libreoffice\n" + "Alternatively, convert the document to PDF manually." + ) + + # Create temporary directory for PDF conversion + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Convert to PDF using LibreOffice + logging.info(f"Converting {doc_path.name} to PDF using LibreOffice...") + + # Use the working LibreOffice command first, then try alternatives if it fails + commands_to_try = [working_libreoffice_cmd] + if working_libreoffice_cmd == "libreoffice": + commands_to_try.append("soffice") + else: + commands_to_try.append("libreoffice") + + conversion_successful = False + for cmd in commands_to_try: + if cmd is None: + continue + try: + convert_cmd = [ + cmd, + "--headless", + "--convert-to", + "pdf", + "--outdir", + str(temp_path), + str(doc_path), + ] + + # Prepare conversion subprocess parameters + convert_subprocess_kwargs: Dict[str, Any] = { + "capture_output": True, + "text": True, + "timeout": 60, # 60 second timeout + "encoding": "utf-8", + "errors": "ignore", + } + + # Hide console window on Windows + if platform.system() == "Windows": + convert_subprocess_kwargs["creationflags"] = ( + 0x08000000 # subprocess.CREATE_NO_WINDOW + ) + + result = subprocess.run( + convert_cmd, **convert_subprocess_kwargs + ) + + if result.returncode == 0: # type: ignore + conversion_successful = True + logging.info( + f"Successfully converted {doc_path.name} to PDF" + ) + break + else: + logging.warning( + f"LibreOffice command '{cmd}' failed: {result.stderr}" # type: ignore + ) + except subprocess.TimeoutExpired: + logging.warning(f"LibreOffice command '{cmd}' timed out") + except Exception as e: + logging.error( + f"LibreOffice command '{cmd}' failed with exception: {e}" + ) + + if not conversion_successful: + raise RuntimeError( + f"LibreOffice conversion failed for {doc_path.name}. " + f"Please check if the file is corrupted or try converting manually." + ) + + # Find the generated PDF + pdf_files = list(temp_path.glob("*.pdf")) + if not pdf_files: + raise RuntimeError( + f"PDF conversion failed for {doc_path.name} - no PDF file generated. " + f"Please check LibreOffice installation or try manual conversion." + ) + + pdf_path = pdf_files[0] + logging.info( + f"Generated PDF: {pdf_path.name} ({pdf_path.stat().st_size} bytes)" + ) + + # Validate the generated PDF + if pdf_path.stat().st_size < 100: # Very small file, likely empty + raise RuntimeError( + "Generated PDF appears to be empty or corrupted. " + "Original file may have issues or LibreOffice conversion failed." + ) + + # Copy PDF to final output directory + final_pdf_path = base_output_dir / f"{name_without_suff}.pdf" + shutil.copy2(pdf_path, final_pdf_path) + + return final_pdf_path + + except Exception as e: + logging.error(f"Error in convert_office_to_pdf: {str(e)}") + raise + + @staticmethod + def convert_text_to_pdf( + text_path: Union[str, Path], output_dir: Optional[str] = None + ) -> Path: + """ + Convert text file (.txt, .md) to PDF using ReportLab with full markdown support. + + Args: + text_path: Path to the text file + output_dir: Output directory for the PDF file + + Returns: + Path to the generated PDF file + """ + try: + text_path = Path(text_path) + if not text_path.exists(): + raise FileNotFoundError(f"Text file does not exist: {text_path}") + + # Supported text formats + supported_text_formats = {".txt", ".md"} + if text_path.suffix.lower() not in supported_text_formats: + raise ValueError(f"Unsupported text format: {text_path.suffix}") + + # Read the text content + try: + with open(text_path, "r", encoding="utf-8") as f: + text_content = f.read() + except UnicodeDecodeError: + # Try with different encodings + for encoding in ["gbk", "latin-1", "cp1252"]: + try: + with open(text_path, "r", encoding=encoding) as f: + text_content = f.read() + logging.info(f"Successfully read file with {encoding} encoding") + break + except UnicodeDecodeError: + continue + else: + raise RuntimeError( + f"Could not decode text file {text_path.name} with any supported encoding" + ) + + # Prepare output directory + if output_dir: + base_output_dir = Path(output_dir) + else: + base_output_dir = text_path.parent / "pdf_output" + + base_output_dir.mkdir(parents=True, exist_ok=True) + pdf_path = base_output_dir / f"{text_path.stem}.pdf" + + # Convert text to PDF + logging.info(f"Converting {text_path.name} to PDF...") + + try: + from reportlab.lib.pagesizes import A4 + from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer + from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle + from reportlab.lib.units import inch + from reportlab.pdfbase import pdfmetrics + + # Create PDF document + doc = SimpleDocTemplate( + str(pdf_path), + pagesize=A4, + leftMargin=inch, + rightMargin=inch, + topMargin=inch, + bottomMargin=inch, + ) + + # Get styles + styles = getSampleStyleSheet() + normal_style = styles["Normal"] + heading_style = styles["Heading1"] + + # Try to register a font that supports Chinese characters + try: + # Try to use system fonts that support Chinese + system = platform.system() + if system == "Windows": + # Try common Windows fonts + for font_name in ["SimSun", "SimHei", "Microsoft YaHei"]: + try: + from reportlab.pdfbase.cidfonts import ( + UnicodeCIDFont, + ) + + pdfmetrics.registerFont(UnicodeCIDFont(font_name)) # type: ignore + normal_style.fontName = font_name + heading_style.fontName = font_name + break + except Exception: + continue + elif system == "Darwin": # macOS + for font_name in ["STSong-Light", "STHeiti"]: + try: + from reportlab.pdfbase.cidfonts import ( + UnicodeCIDFont, + ) + + pdfmetrics.registerFont(UnicodeCIDFont(font_name)) # type: ignore + normal_style.fontName = font_name + heading_style.fontName = font_name + break + except Exception: + continue + except Exception: + pass # Use default fonts if Chinese font setup fails + + # Build content + story = [] + + # Handle markdown or plain text + if text_path.suffix.lower() == ".md": + # Handle markdown content - simplified implementation + lines = text_content.split("\n") + for line in lines: + line = line.strip() + if not line: + story.append(Spacer(1, 12)) + continue + + # Headers + if line.startswith("#"): + level = len(line) - len(line.lstrip("#")) + header_text = line.lstrip("#").strip() + if header_text: + header_style = ParagraphStyle( + name=f"Heading{level}", + parent=heading_style, + fontSize=max(16 - level, 10), + spaceAfter=8, + spaceBefore=16 if level <= 2 else 12, + ) + story.append(Paragraph(header_text, header_style)) + else: + # Regular text + processed_line = PDFConverter._process_inline_markdown(line) + story.append(Paragraph(processed_line, normal_style)) + story.append(Spacer(1, 6)) + else: + # Handle plain text files (.txt) + logging.info( + f"Processing plain text file with {len(text_content)} characters..." + ) + + # Split text into lines and process each line + lines = text_content.split("\n") + line_count = 0 + + for line in lines: + line = line.rstrip() + line_count += 1 + + # Empty lines + if not line.strip(): + story.append(Spacer(1, 6)) + continue + + # Regular text lines + # Escape special characters for ReportLab + safe_line = ( + line.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + ) + + # Create paragraph + story.append(Paragraph(safe_line, normal_style)) + story.append(Spacer(1, 3)) + + logging.info(f"Added {line_count} lines to PDF") + + # If no content was added, add a placeholder + if not story: + story.append(Paragraph("(Empty text file)", normal_style)) + + # Build PDF + doc.build(story) + logging.info( + f"Successfully converted {text_path.name} to PDF ({pdf_path.stat().st_size / 1024:.1f} KB)" + ) + + except ImportError: + raise RuntimeError( + "reportlab is required for text-to-PDF conversion. " + "Please install it using: pip install reportlab" + ) + except Exception as e: + raise RuntimeError( + f"Failed to convert text file {text_path.name} to PDF: {str(e)}" + ) + + # Validate the generated PDF + if not pdf_path.exists() or pdf_path.stat().st_size < 100: + raise RuntimeError( + f"PDF conversion failed for {text_path.name} - generated PDF is empty or corrupted." + ) + + return pdf_path + + except Exception as e: + logging.error(f"Error in convert_text_to_pdf: {str(e)}") + raise + + @staticmethod + def _process_inline_markdown(text: str) -> str: + """ + Process inline markdown formatting (bold, italic, code, links) + + Args: + text: Raw text with markdown formatting + + Returns: + Text with ReportLab markup + """ + import re + + # Escape special characters for ReportLab + text = text.replace("&", "&").replace("<", "<").replace(">", ">") + + # Bold text: **text** or __text__ + text = re.sub(r"\*\*(.*?)\*\*", r"\1", text) + text = re.sub(r"__(.*?)__", r"\1", text) + + # Italic text: *text* or _text_ (but not in the middle of words) + text = re.sub(r"(?\1", text) + text = re.sub(r"(?\1", text) + + # Inline code: `code` + text = re.sub( + r"`([^`]+?)`", + r'\1', + text, + ) + + # Links: [text](url) - convert to text with URL annotation + def link_replacer(match): + link_text = match.group(1) + url = match.group(2) + return f'{link_text}' + + text = re.sub(r"\[([^\]]+?)\]\(([^)]+?)\)", link_replacer, text) + + # Strikethrough: ~~text~~ + text = re.sub(r"~~(.*?)~~", r"\1", text) + + return text + + def convert_to_pdf( + self, + file_path: Union[str, Path], + output_dir: Optional[str] = None, + ) -> Path: + """ + Convert document to PDF based on file extension + + Args: + file_path: Path to the file to be converted + output_dir: Output directory path + + Returns: + Path to the generated PDF file + """ + # Convert to Path object + file_path = Path(file_path) + if not file_path.exists(): + raise FileNotFoundError(f"File does not exist: {file_path}") + + # Get file extension + ext = file_path.suffix.lower() + + # Choose appropriate conversion method based on file type + if ext in self.OFFICE_FORMATS: + return self.convert_office_to_pdf(file_path, output_dir) + elif ext in self.TEXT_FORMATS: + return self.convert_text_to_pdf(file_path, output_dir) + else: + raise ValueError( + f"Unsupported file format: {ext}. " + f"Supported formats: {', '.join(self.OFFICE_FORMATS | self.TEXT_FORMATS)}" + ) + + def check_dependencies(self) -> dict: + """ + Check if required dependencies are available + + Returns: + dict: Dictionary with dependency check results + """ + results = { + "libreoffice": False, + "reportlab": False, + } + + # Check LibreOffice + try: + subprocess_kwargs: Dict[str, Any] = { + "capture_output": True, + "text": True, + "check": True, + "encoding": "utf-8", + "errors": "ignore", + } + + if platform.system() == "Windows": + subprocess_kwargs["creationflags"] = ( + 0x08000000 # subprocess.CREATE_NO_WINDOW + ) + + subprocess.run(["libreoffice", "--version"], **subprocess_kwargs) + results["libreoffice"] = True + except (subprocess.CalledProcessError, FileNotFoundError): + try: + subprocess.run(["soffice", "--version"], **subprocess_kwargs) + results["libreoffice"] = True + except (subprocess.CalledProcessError, FileNotFoundError): + pass + + # Check ReportLab + import importlib.util + + if importlib.util.find_spec("reportlab") is not None: + results["reportlab"] = True + + return results + + +def main(): + """ + Main function to run the PDF converter from command line + """ + parser = argparse.ArgumentParser(description="Convert documents to PDF format") + parser.add_argument("file_path", nargs="?", help="Path to the document to convert") + parser.add_argument("--output", "-o", help="Output directory path") + parser.add_argument( + "--check", + action="store_true", + help="Check dependencies installation", + ) + parser.add_argument( + "--verbose", "-v", action="store_true", help="Enable verbose logging" + ) + + args = parser.parse_args() + + # Configure logging + log_level = logging.INFO if args.verbose else logging.WARNING + logging.basicConfig( + level=log_level, + format="%(asctime)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + # Initialize converter + converter = PDFConverter() + + # Check dependencies if requested + if args.check: + print("๐Ÿ” Checking dependencies...") + deps = converter.check_dependencies() + + print( + f"LibreOffice: {'โœ… Available' if deps['libreoffice'] else 'โŒ Not found'}" + ) + print(f"ReportLab: {'โœ… Available' if deps['reportlab'] else 'โŒ Not found'}") + + if not deps["libreoffice"]: + print("\n๐Ÿ“‹ To install LibreOffice:") + print(" - Windows: Download from https://www.libreoffice.org/") + print(" - macOS: brew install --cask libreoffice") + print(" - Ubuntu/Debian: sudo apt-get install libreoffice") + + if not deps["reportlab"]: + print("\n๐Ÿ“‹ To install ReportLab:") + print(" pip install reportlab") + + return 0 + + # If not checking dependencies, file_path is required + if not args.file_path: + parser.error("file_path is required when not using --check") + + try: + # Convert the file + output_pdf = converter.convert_to_pdf( + file_path=args.file_path, + output_dir=args.output, + ) + + print(f"โœ… Successfully converted to PDF: {output_pdf}") + print(f"๐Ÿ“„ File size: {output_pdf.stat().st_size / 1024:.1f} KB") + + except Exception as e: + print(f"โŒ Error: {str(e)}") + return 1 + + return 0 + + +if __name__ == "__main__": + exit(main()) diff --git a/DeepCode/tools/pdf_downloader.py b/DeepCode/tools/pdf_downloader.py new file mode 100644 index 00000000..b9bf1376 --- /dev/null +++ b/DeepCode/tools/pdf_downloader.py @@ -0,0 +1,1420 @@ +#!/usr/bin/env python3 +""" +Smart PDF Downloader MCP Tool + +A standardized MCP tool using FastMCP for intelligent file downloading and document conversion. +Supports natural language instructions for downloading files from URLs, moving local files, +and automatic conversion to Markdown format with image extraction. + +Features: +- Natural language instruction parsing +- URL and local path extraction +- Automatic document conversion (PDF, DOCX, PPTX, HTML, etc.) +- Image extraction and preservation +- Multi-format support with fallback options +""" + +import os +import re +import aiohttp +import aiofiles +import shutil +import sys +import io +from typing import List, Dict, Optional, Any +from urllib.parse import urlparse, unquote +from datetime import datetime + +from mcp.server import FastMCP + +# Docling imports for document conversion +try: + from docling.document_converter import DocumentConverter + from docling.datamodel.base_models import InputFormat + from docling.datamodel.pipeline_options import PdfPipelineOptions + from docling.document_converter import PdfFormatOption + + DOCLING_AVAILABLE = True +except ImportError: + DOCLING_AVAILABLE = False + print( + "Warning: docling package not available. Document conversion will be disabled." + ) + +# Fallback PDF text extraction +try: + import PyPDF2 + + PYPDF2_AVAILABLE = True +except ImportError: + PYPDF2_AVAILABLE = False + print( + "Warning: PyPDF2 package not available. Fallback PDF extraction will be disabled." + ) + +# ่ฎพ็ฝฎๆ ‡ๅ‡†่พ“ๅ‡บ็ผ–็ ไธบUTF-8 +if sys.stdout.encoding != "utf-8": + try: + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8") + sys.stderr.reconfigure(encoding="utf-8") + else: + sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding="utf-8") + sys.stderr = io.TextIOWrapper(sys.stderr.detach(), encoding="utf-8") + except Exception as e: + print(f"Warning: Could not set UTF-8 encoding: {e}") + +# ๅˆ›ๅปบ FastMCP ๅฎžไพ‹ +mcp = FastMCP("smart-pdf-downloader") + + +# ่พ…ๅŠฉๅ‡ฝๆ•ฐ +def format_success_message(action: str, details: Dict[str, Any]) -> str: + """ๆ ผๅผๅŒ–ๆˆๅŠŸๆถˆๆฏ""" + return f"โœ… {action}\n" + "\n".join(f" {k}: {v}" for k, v in details.items()) + + +def format_error_message(action: str, error: str) -> str: + """ๆ ผๅผๅŒ–้”™่ฏฏๆถˆๆฏ""" + return f"โŒ {action}\n Error: {error}" + + +def format_warning_message(action: str, warning: str) -> str: + """ๆ ผๅผๅŒ–่ญฆๅ‘Šๆถˆๆฏ""" + return f"โš ๏ธ {action}\n Warning: {warning}" + + +async def perform_document_conversion( + file_path: str, extract_images: bool = True +) -> Optional[str]: + """ + ๆ‰ง่กŒๆ–‡ๆกฃ่ฝฌๆข็š„ๅ…ฑ็”จ้€ป่พ‘ + + Args: + file_path: ๆ–‡ไปถ่ทฏๅพ„ + extract_images: ๆ˜ฏๅฆๆๅ–ๅ›พ็‰‡ + + Returns: + ่ฝฌๆขไฟกๆฏๅญ—็ฌฆไธฒ๏ผŒๅฆ‚ๆžœๆฒกๆœ‰่ฝฌๆขๅˆ™่ฟ”ๅ›žNone + """ + if not file_path: + return None + + conversion_msg = "" + + # ้ฆ–ๅ…ˆๅฐ่ฏ•ไฝฟ็”จ็ฎ€ๅ•็š„PDF่ฝฌๆขๅ™จ๏ผˆๅฏนไบŽPDFๆ–‡ไปถ๏ผ‰ + # ๆฃ€ๆŸฅๆ–‡ไปถๆ˜ฏๅฆๅฎž้™…ไธบPDF๏ผˆๆ— ่ฎบๆ‰ฉๅฑ•ๅๅฆ‚ไฝ•๏ผ‰ + is_pdf_file = False + if PYPDF2_AVAILABLE: + try: + with open(file_path, "rb") as f: + header = f.read(8) + is_pdf_file = header.startswith(b"%PDF") + except Exception: + is_pdf_file = file_path.lower().endswith(".pdf") + + if is_pdf_file and PYPDF2_AVAILABLE: + try: + simple_converter = SimplePdfConverter() + conversion_result = simple_converter.convert_pdf_to_markdown(file_path) + if conversion_result["success"]: + conversion_msg = "\n [INFO] PDF converted to Markdown (PyPDF2)" + conversion_msg += ( + f"\n Markdown file: {conversion_result['output_file']}" + ) + conversion_msg += ( + f"\n Conversion time: {conversion_result['duration']:.2f} seconds" + ) + conversion_msg += ( + f"\n Pages extracted: {conversion_result['pages_extracted']}" + ) + + else: + conversion_msg = f"\n [WARNING] PDF conversion failed: {conversion_result['error']}" + except Exception as conv_error: + conversion_msg = f"\n [WARNING] PDF conversion error: {str(conv_error)}" + + # ๅฆ‚ๆžœ็ฎ€ๅ•่ฝฌๆขๅคฑ่ดฅ๏ผŒๅฐ่ฏ•ไฝฟ็”จdocling๏ผˆๆ”ฏๆŒๅ›พ็‰‡ๆๅ–๏ผ‰ + # if not conversion_success and DOCLING_AVAILABLE: + # try: + # converter = DoclingConverter() + # if converter.is_supported_format(file_path): + # conversion_result = converter.convert_to_markdown( + # file_path, extract_images=extract_images + # ) + # if conversion_result["success"]: + # conversion_msg = ( + # "\n [INFO] Document converted to Markdown (docling)" + # ) + # conversion_msg += ( + # f"\n Markdown file: {conversion_result['output_file']}" + # ) + # conversion_msg += f"\n Conversion time: {conversion_result['duration']:.2f} seconds" + # if conversion_result.get("images_extracted", 0) > 0: + # conversion_msg += f"\n Images extracted: {conversion_result['images_extracted']}" + # images_dir = os.path.join( + # os.path.dirname(conversion_result["output_file"]), "images" + # ) + # conversion_msg += f"\n Images saved to: {images_dir}" + # else: + # conversion_msg = f"\n [WARNING] Docling conversion failed: {conversion_result['error']}" + # except Exception as conv_error: + # conversion_msg = ( + # f"\n [WARNING] Docling conversion error: {str(conv_error)}" + # ) + + return conversion_msg if conversion_msg else None + + +def format_file_operation_result( + operation: str, + source: str, + destination: str, + result: Dict[str, Any], + conversion_msg: Optional[str] = None, +) -> str: + """ + ๆ ผๅผๅŒ–ๆ–‡ไปถๆ“ไฝœ็ป“ๆžœ็š„ๅ…ฑ็”จ้€ป่พ‘ + + Args: + operation: ๆ“ไฝœ็ฑปๅž‹ ("download", "copy", ๆˆ– "move") + source: ๆบๆ–‡ไปถ/URL + destination: ็›ฎๆ ‡่ทฏๅพ„ + result: ๆ“ไฝœ็ป“ๆžœๅญ—ๅ…ธ + conversion_msg: ่ฝฌๆขๆถˆๆฏ + + Returns: + ๆ ผๅผๅŒ–็š„็ป“ๆžœๆถˆๆฏ + """ + if result["success"]: + size_mb = result["size"] / (1024 * 1024) + + # ๅค„็†ไธๅŒๆ“ไฝœ็ฑปๅž‹็š„ๅŠจ่ฏๅฝขๅผ + if operation == "copy": + operation_verb = "copied" + elif operation == "download": + operation_verb = "downloaded" + else: # move + operation_verb = "moved" + + msg = f"[SUCCESS] Successfully {operation_verb}: {source}\n" + + if operation == "download": + msg += f" File: {destination}\n" + msg += f" Size: {size_mb:.2f} MB\n" + msg += f" Time: {result['duration']:.2f} seconds\n" + speed_mb = result.get("speed", 0) / (1024 * 1024) + msg += f" Speed: {speed_mb:.2f} MB/s" + else: # copy or move + msg += f" To: {destination}\n" + msg += f" Size: {size_mb:.2f} MB\n" + msg += f" Time: {result['duration']:.2f} seconds" + if operation == "copy": + msg += "\n Note: Original file preserved" + + if conversion_msg: + msg += conversion_msg + + return msg + else: + return f"[ERROR] Failed to {operation}: {source}\n Error: {result.get('error', 'Unknown error')}" + + +class LocalPathExtractor: + """ๆœฌๅœฐ่ทฏๅพ„ๆๅ–ๅ™จ""" + + @staticmethod + def is_local_path(path: str) -> bool: + """ๅˆคๆ–ญๆ˜ฏๅฆไธบๆœฌๅœฐ่ทฏๅพ„""" + path = path.strip("\"'") + + # ๆฃ€ๆŸฅๆ˜ฏๅฆไธบURL + if re.match(r"^https?://", path, re.IGNORECASE) or re.match( + r"^ftp://", path, re.IGNORECASE + ): + return False + + # ่ทฏๅพ„ๆŒ‡็คบ็ฌฆ + path_indicators = [os.path.sep, "/", "\\", "~", ".", ".."] + has_extension = bool(os.path.splitext(path)[1]) + + if any(indicator in path for indicator in path_indicators) or has_extension: + expanded_path = os.path.expanduser(path) + return os.path.exists(expanded_path) or any( + indicator in path for indicator in path_indicators + ) + + return False + + @staticmethod + def extract_local_paths(text: str) -> List[str]: + """ไปŽๆ–‡ๆœฌไธญๆๅ–ๆœฌๅœฐๆ–‡ไปถ่ทฏๅพ„""" + patterns = [ + r'"([^"]+)"', + r"'([^']+)'", + r"(?:^|\s)((?:[~./\\]|[A-Za-z]:)?(?:[^/\\\s]+[/\\])*[^/\\\s]+\.[A-Za-z0-9]+)(?:\s|$)", + r"(?:^|\s)((?:~|\.{1,2})?/[^\s]+)(?:\s|$)", + r"(?:^|\s)([A-Za-z]:[/\\][^\s]+)(?:\s|$)", + r"(?:^|\s)(\.{1,2}[/\\][^\s]+)(?:\s|$)", + ] + + local_paths = [] + potential_paths = [] + + for pattern in patterns: + matches = re.findall(pattern, text, re.MULTILINE) + potential_paths.extend(matches) + + for path in potential_paths: + path = path.strip() + if path and LocalPathExtractor.is_local_path(path): + expanded_path = os.path.expanduser(path) + if expanded_path not in local_paths: + local_paths.append(expanded_path) + + return local_paths + + +class URLExtractor: + """URLๆๅ–ๅ™จ""" + + URL_PATTERNS = [ + r"https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+(?:/(?:[-\w._~!$&\'()*+,;=:@]|%[\da-fA-F]{2})*)*(?:\?(?:[-\w._~!$&\'()*+,;=:@/?]|%[\da-fA-F]{2})*)?(?:#(?:[-\w._~!$&\'()*+,;=:@/?]|%[\da-fA-F]{2})*)?", + r"ftp://(?:[-\w.]|(?:%[\da-fA-F]{2}))+(?:/(?:[-\w._~!$&\'()*+,;=:@]|%[\da-fA-F]{2})*)*", + r"(? str: + """ๅฐ†arXiv็ฝ‘้กต้“พๆŽฅ่ฝฌๆขไธบPDFไธ‹่ฝฝ้“พๆŽฅ""" + # ๅŒน้…arXiv่ฎบๆ–‡ID็š„ๆญฃๅˆ™่กจ่พพๅผ + arxiv_pattern = r"arxiv\.org/abs/(\d+\.\d+)(?:v\d+)?" + match = re.search(arxiv_pattern, url, re.IGNORECASE) + if match: + paper_id = match.group(1) + return f"https://arxiv.org/pdf/{paper_id}.pdf" + return url + + @classmethod + def extract_urls(cls, text: str) -> List[str]: + """ไปŽๆ–‡ๆœฌไธญๆๅ–URL""" + urls = [] + + # ้ฆ–ๅ…ˆๅค„็†็‰นๆฎŠๆƒ…ๅ†ต๏ผš@ๅผ€ๅคด็š„URL + at_url_pattern = r"@(https?://[^\s]+)" + at_matches = re.findall(at_url_pattern, text, re.IGNORECASE) + for match in at_matches: + # ๅค„็†arXiv้“พๆŽฅ + url = cls.convert_arxiv_url(match.rstrip("/")) + urls.append(url) + + # ็„ถๅŽไฝฟ็”จๅŽŸๆœ‰็š„ๆญฃๅˆ™ๆจกๅผ + for pattern in cls.URL_PATTERNS: + matches = re.findall(pattern, text, re.IGNORECASE) + for match in matches: + # ๅค„็†ๅฏ่ƒฝ็ผบๅฐ‘ๅ่ฎฎ็š„URL + if not match.startswith(("http://", "https://", "ftp://")): + # ๆฃ€ๆŸฅๆ˜ฏๅฆๆ˜ฏ www ๅผ€ๅคด + if match.startswith("www."): + match = "https://" + match + else: + # ๅ…ถไป–ๆƒ…ๅ†ตไนŸๆทปๅŠ  https + match = "https://" + match + + # ๅค„็†arXiv้“พๆŽฅ + url = cls.convert_arxiv_url(match.rstrip("/")) + urls.append(url) + + # ๅŽป้‡ๅนถไฟๆŒ้กบๅบ + seen = set() + unique_urls = [] + for url in urls: + if url not in seen: + seen.add(url) + unique_urls.append(url) + + return unique_urls + + @staticmethod + def infer_filename_from_url(url: str) -> str: + """ไปŽURLๆŽจๆ–ญๆ–‡ไปถๅ""" + parsed = urlparse(url) + path = unquote(parsed.path) + + # ไปŽ่ทฏๅพ„ไธญๆๅ–ๆ–‡ไปถๅ + filename = os.path.basename(path) + + # ็‰นๆฎŠๅค„็†๏ผšarxiv PDF้“พๆŽฅ + if "arxiv.org" in parsed.netloc and "/pdf/" in path: + if filename: + # ๆฃ€ๆŸฅๆ˜ฏๅฆๅทฒ็ปๆœ‰ๅˆ้€‚็š„ๆ–‡ไปถๆ‰ฉๅฑ•ๅ + if not filename.lower().endswith((".pdf", ".doc", ".docx", ".txt")): + filename = f"{filename}.pdf" + else: + path_parts = [p for p in path.split("/") if p] + if path_parts and path_parts[-1]: + filename = f"{path_parts[-1]}.pdf" + else: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"arxiv_paper_{timestamp}.pdf" + + # ๅฆ‚ๆžœๆฒกๆœ‰ๆ–‡ไปถๅๆˆ–ๆฒกๆœ‰ๆ‰ฉๅฑ•ๅ๏ผŒ็”Ÿๆˆไธ€ไธช + elif not filename or "." not in filename: + # ๅฐ่ฏ•ไปŽURL็”Ÿๆˆๆœ‰ๆ„ไน‰็š„ๆ–‡ไปถๅ + domain = parsed.netloc.replace("www.", "").replace(".", "_") + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + + # ๅฐ่ฏ•ๆ นๆฎ่ทฏๅพ„ๆŽจๆ–ญๆ–‡ไปถ็ฑปๅž‹ + if not path or path == "/": + filename = f"{domain}_{timestamp}.html" + else: + # ไฝฟ็”จ่ทฏๅพ„็š„ๆœ€ๅŽไธ€้ƒจๅˆ† + path_parts = [p for p in path.split("/") if p] + if path_parts: + filename = f"{path_parts[-1]}_{timestamp}" + else: + filename = f"{domain}_{timestamp}" + + # ๅฆ‚ๆžœ่ฟ˜ๆ˜ฏๆฒกๆœ‰ๆ‰ฉๅฑ•ๅ๏ผŒๆ นๆฎ่ทฏๅพ„ๆŽจๆ–ญ + if "." not in filename: + # ๆ นๆฎ่ทฏๅพ„ไธญ็š„ๅ…ณ้”ฎ่ฏๆŽจๆ–ญๆ–‡ไปถ็ฑปๅž‹ + if "/pdf/" in path.lower() or path.lower().endswith("pdf"): + filename += ".pdf" + elif any( + ext in path.lower() for ext in ["/doc/", "/word/", ".docx"] + ): + filename += ".docx" + elif any( + ext in path.lower() + for ext in ["/ppt/", "/powerpoint/", ".pptx"] + ): + filename += ".pptx" + elif any(ext in path.lower() for ext in ["/csv/", ".csv"]): + filename += ".csv" + elif any(ext in path.lower() for ext in ["/zip/", ".zip"]): + filename += ".zip" + else: + filename += ".html" + + return filename + + +class PathExtractor: + """่ทฏๅพ„ๆๅ–ๅ™จ""" + + @staticmethod + def extract_target_path(text: str) -> Optional[str]: + """ไปŽๆ–‡ๆœฌไธญๆๅ–็›ฎๆ ‡่ทฏๅพ„""" + patterns = [ + r'(?:save|download|store|put|place|write|copy|move)\s+(?:to|into|in|at)\s+["\']?([^\s"\']+)["\']?', + r'(?:to|into|in|at)\s+(?:folder|directory|dir|path|location)\s*["\']?([^\s"\']+)["\']?', + r'(?:destination|target|output)\s*(?:is|:)?\s*["\']?([^\s"\']+)["\']?', + r'(?:ไฟๅญ˜|ไธ‹่ฝฝ|ๅญ˜ๅ‚จ|ๆ”พๅˆฐ|ๅ†™ๅ…ฅ|ๅคๅˆถ|็งปๅŠจ)(?:ๅˆฐ|่‡ณ|ๅŽป)\s*["\']?([^\s"\']+)["\']?', + r'(?:ๅˆฐ|ๅœจ|่‡ณ)\s*["\']?([^\s"\']+)["\']?\s*(?:ๆ–‡ไปถๅคน|็›ฎๅฝ•|่ทฏๅพ„|ไฝ็ฝฎ)', + ] + + filter_words = { + "here", + "there", + "current", + "local", + "this", + "that", + "่ฟ™้‡Œ", + "้‚ฃ้‡Œ", + "ๅฝ“ๅ‰", + "ๆœฌๅœฐ", + "่ฟ™ไธช", + "้‚ฃไธช", + } + + for pattern in patterns: + match = re.search(pattern, text, re.IGNORECASE) + if match: + path = match.group(1).strip("ใ€‚๏ผŒ,.ใ€") + if path and path.lower() not in filter_words: + return path + + return None + + +class SimplePdfConverter: + """็ฎ€ๅ•็š„PDF่ฝฌๆขๅ™จ๏ผŒไฝฟ็”จPyPDF2ๆๅ–ๆ–‡ๆœฌ""" + + def convert_pdf_to_markdown( + self, input_file: str, output_file: Optional[str] = None + ) -> Dict[str, Any]: + """ + ไฝฟ็”จPyPDF2ๅฐ†PDF่ฝฌๆขไธบMarkdownๆ ผๅผ + + Args: + input_file: ่พ“ๅ…ฅPDFๆ–‡ไปถ่ทฏๅพ„ + output_file: ่พ“ๅ‡บMarkdownๆ–‡ไปถ่ทฏๅพ„๏ผˆๅฏ้€‰๏ผ‰ + + Returns: + ่ฝฌๆข็ป“ๆžœๅญ—ๅ…ธ + """ + if not PYPDF2_AVAILABLE: + return {"success": False, "error": "PyPDF2 package is not available"} + + try: + # ๆฃ€ๆŸฅ่พ“ๅ…ฅๆ–‡ไปถๆ˜ฏๅฆๅญ˜ๅœจ + if not os.path.exists(input_file): + return { + "success": False, + "error": f"Input file not found: {input_file}", + } + + # ๅฆ‚ๆžœๆฒกๆœ‰ๆŒ‡ๅฎš่พ“ๅ‡บๆ–‡ไปถ๏ผŒ่‡ชๅŠจ็”Ÿๆˆ + if not output_file: + base_name = os.path.splitext(input_file)[0] + output_file = f"{base_name}.md" + + # ็กฎไฟ่พ“ๅ‡บ็›ฎๅฝ•ๅญ˜ๅœจ + output_dir = os.path.dirname(output_file) + if output_dir: + os.makedirs(output_dir, exist_ok=True) + + # ๆ‰ง่กŒ่ฝฌๆข + start_time = datetime.now() + + # ่ฏปๅ–PDFๆ–‡ไปถ + with open(input_file, "rb") as file: + pdf_reader = PyPDF2.PdfReader(file) + text_content = [] + + # ๆๅ–ๆฏ้กตๆ–‡ๆœฌ + for page_num, page in enumerate(pdf_reader.pages, 1): + text = page.extract_text() + if text.strip(): + text_content.append(f"## Page {page_num}\n\n{text.strip()}\n\n") + + # ็”ŸๆˆMarkdownๅ†…ๅฎน + markdown_content = f"# Extracted from {os.path.basename(input_file)}\n\n" + markdown_content += f"*Total pages: {len(pdf_reader.pages)}*\n\n" + markdown_content += "---\n\n" + markdown_content += "".join(text_content) + + # ไฟๅญ˜ๅˆฐๆ–‡ไปถ + with open(output_file, "w", encoding="utf-8") as f: + f.write(markdown_content) + + # ่ฎก็ฎ—่ฝฌๆขๆ—ถ้—ด + duration = (datetime.now() - start_time).total_seconds() + + # ่Žทๅ–ๆ–‡ไปถๅคงๅฐ + input_size = os.path.getsize(input_file) + output_size = os.path.getsize(output_file) + + return { + "success": True, + "input_file": input_file, + "output_file": output_file, + "input_size": input_size, + "output_size": output_size, + "duration": duration, + "markdown_content": markdown_content, + "pages_extracted": len(pdf_reader.pages), + } + + except Exception as e: + return { + "success": False, + "input_file": input_file, + "error": f"Conversion failed: {str(e)}", + } + + +class DoclingConverter: + """ๆ–‡ๆกฃ่ฝฌๆขๅ™จ๏ผŒไฝฟ็”จdoclingๅฐ†ๆ–‡ๆกฃ่ฝฌๆขไธบMarkdownๆ ผๅผ๏ผŒๆ”ฏๆŒๅ›พ็‰‡ๆๅ–""" + + def __init__(self): + if not DOCLING_AVAILABLE: + raise ImportError( + "docling package is not available. Please install it first." + ) + + # ้…็ฝฎPDFๅค„็†้€‰้กน + pdf_pipeline_options = PdfPipelineOptions() + pdf_pipeline_options.do_ocr = False # ๆš‚ๆ—ถ็ฆ็”จOCRไปฅ้ฟๅ…่ฎค่ฏ้—ฎ้ข˜ + pdf_pipeline_options.do_table_structure = False # ๆš‚ๆ—ถ็ฆ็”จ่กจๆ ผ็ป“ๆž„่ฏ†ๅˆซ + + # ๅˆ›ๅปบๆ–‡ๆกฃ่ฝฌๆขๅ™จ๏ผˆไฝฟ็”จๅŸบ็ก€ๆจกๅผ๏ผ‰ + try: + self.converter = DocumentConverter( + format_options={ + InputFormat.PDF: PdfFormatOption( + pipeline_options=pdf_pipeline_options + ) + } + ) + except Exception: + # ๅฆ‚ๆžœๅคฑ่ดฅ๏ผŒๅฐ่ฏ•ๆ›ด็ฎ€ๅ•็š„้…็ฝฎ + self.converter = DocumentConverter() + + def is_supported_format(self, file_path: str) -> bool: + """ๆฃ€ๆŸฅๆ–‡ไปถๆ ผๅผๆ˜ฏๅฆๆ”ฏๆŒ่ฝฌๆข""" + if not DOCLING_AVAILABLE: + return False + + supported_extensions = {".pdf", ".docx", ".pptx", ".html", ".md", ".txt"} + file_extension = os.path.splitext(file_path)[1].lower() + return file_extension in supported_extensions + + def is_url(self, path: str) -> bool: + """ๆฃ€ๆŸฅ่ทฏๅพ„ๆ˜ฏๅฆไธบURL""" + try: + result = urlparse(path) + return result.scheme in ("http", "https") + except Exception: + return False + + def extract_images(self, doc, output_dir: str) -> Dict[str, str]: + """ + ๆๅ–ๆ–‡ๆกฃไธญ็š„ๅ›พ็‰‡ๅนถไฟๅญ˜ๅˆฐๆœฌๅœฐ + + Args: + doc: doclingๆ–‡ๆกฃๅฏน่ฑก + output_dir: ่พ“ๅ‡บ็›ฎๅฝ• + + Returns: + ๅ›พ็‰‡IDๅˆฐๆœฌๅœฐๆ–‡ไปถ่ทฏๅพ„็š„ๆ˜ ๅฐ„ + """ + images_dir = os.path.join(output_dir, "images") + os.makedirs(images_dir, exist_ok=True) + image_map = {} # doclingๅ›พ็‰‡id -> ๆœฌๅœฐๆ–‡ไปถๅ + + try: + # ่Žทๅ–ๆ–‡ๆกฃไธญ็š„ๅ›พ็‰‡ + images = getattr(doc, "images", []) + + for idx, img in enumerate(images): + try: + # ่Žทๅ–ๅ›พ็‰‡ๆ ผๅผ๏ผŒ้ป˜่ฎคไธบpng + ext = getattr(img, "format", None) or "png" + if ext.lower() not in ["png", "jpg", "jpeg", "gif", "bmp", "webp"]: + ext = "png" + + # ็”Ÿๆˆๆ–‡ไปถๅ + filename = f"image_{idx+1}.{ext}" + filepath = os.path.join(images_dir, filename) + + # ไฟๅญ˜ๅ›พ็‰‡ๆ•ฐๆฎ + img_data = getattr(img, "data", None) + if img_data: + with open(filepath, "wb") as f: + f.write(img_data) + + # ่ฎก็ฎ—็›ธๅฏน่ทฏๅพ„ + rel_path = os.path.relpath(filepath, output_dir) + img_id = getattr(img, "id", str(idx + 1)) + image_map[img_id] = rel_path + + except Exception as img_error: + print(f"Warning: Failed to extract image {idx+1}: {img_error}") + continue + + except Exception as e: + print(f"Warning: Failed to extract images: {e}") + + return image_map + + def process_markdown_with_images( + self, markdown_content: str, image_map: Dict[str, str] + ) -> str: + """ + ๅค„็†Markdownๅ†…ๅฎน๏ผŒๆ›ฟๆขๅ›พ็‰‡ๅ ไฝ็ฌฆไธบๅฎž้™…็š„ๅ›พ็‰‡่ทฏๅพ„ + + Args: + markdown_content: ๅŽŸๅง‹Markdownๅ†…ๅฎน + image_map: ๅ›พ็‰‡IDๅˆฐๆœฌๅœฐ่ทฏๅพ„็š„ๆ˜ ๅฐ„ + + Returns: + ๅค„็†ๅŽ็š„Markdownๅ†…ๅฎน + """ + + def replace_img(match): + img_id = match.group(1) + if img_id in image_map: + return f"![Image]({image_map[img_id]})" + else: + return match.group(0) + + # ๆ›ฟๆขdocling็š„ๅ›พ็‰‡ๅ ไฝ็ฌฆ + processed_content = re.sub( + r"!\[Image\]\(docling://image/([^)]+)\)", replace_img, markdown_content + ) + + return processed_content + + def convert_to_markdown( + self, + input_file: str, + output_file: Optional[str] = None, + extract_images: bool = True, + ) -> Dict[str, Any]: + """ + ๅฐ†ๆ–‡ๆกฃ่ฝฌๆขไธบMarkdownๆ ผๅผ๏ผŒๆ”ฏๆŒๅ›พ็‰‡ๆๅ– + + Args: + input_file: ่พ“ๅ…ฅๆ–‡ไปถ่ทฏๅพ„ๆˆ–URL + output_file: ่พ“ๅ‡บMarkdownๆ–‡ไปถ่ทฏๅพ„๏ผˆๅฏ้€‰๏ผ‰ + extract_images: ๆ˜ฏๅฆๆๅ–ๅ›พ็‰‡๏ผˆ้ป˜่ฎคTrue๏ผ‰ + + Returns: + ่ฝฌๆข็ป“ๆžœๅญ—ๅ…ธ + """ + if not DOCLING_AVAILABLE: + return {"success": False, "error": "docling package is not available"} + + try: + # ๆฃ€ๆŸฅ่พ“ๅ…ฅๆ–‡ไปถ๏ผˆๅฆ‚ๆžœไธๆ˜ฏURL๏ผ‰ + if not self.is_url(input_file): + if not os.path.exists(input_file): + return { + "success": False, + "error": f"Input file not found: {input_file}", + } + + # ๆฃ€ๆŸฅๆ–‡ไปถๆ ผๅผๆ˜ฏๅฆๆ”ฏๆŒ + if not self.is_supported_format(input_file): + return { + "success": False, + "error": f"Unsupported file format: {os.path.splitext(input_file)[1]}", + } + else: + # ๅฏนไบŽURL๏ผŒๆฃ€ๆŸฅๆ˜ฏๅฆไธบๆ”ฏๆŒ็š„ๆ ผๅผ + if not input_file.lower().endswith( + (".pdf", ".docx", ".pptx", ".html", ".md", ".txt") + ): + return { + "success": False, + "error": f"Unsupported URL format: {input_file}", + } + + # ๅฆ‚ๆžœๆฒกๆœ‰ๆŒ‡ๅฎš่พ“ๅ‡บๆ–‡ไปถ๏ผŒ่‡ชๅŠจ็”Ÿๆˆ + if not output_file: + if self.is_url(input_file): + # ไปŽURL็”Ÿๆˆๆ–‡ไปถๅ + filename = URLExtractor.infer_filename_from_url(input_file) + base_name = os.path.splitext(filename)[0] + else: + base_name = os.path.splitext(input_file)[0] + output_file = f"{base_name}.md" + + # ็กฎไฟ่พ“ๅ‡บ็›ฎๅฝ•ๅญ˜ๅœจ + output_dir = os.path.dirname(output_file) or "." + os.makedirs(output_dir, exist_ok=True) + + # ๆ‰ง่กŒ่ฝฌๆข + start_time = datetime.now() + result = self.converter.convert(input_file) + doc = result.document + + # ๆๅ–ๅ›พ็‰‡๏ผˆๅฆ‚ๆžœๅฏ็”จ๏ผ‰ + image_map = {} + images_extracted = 0 + if extract_images: + image_map = self.extract_images(doc, output_dir) + images_extracted = len(image_map) + + # ่Žทๅ–Markdownๅ†…ๅฎน + markdown_content = doc.export_to_markdown() + + # ๅค„็†ๅ›พ็‰‡ๅ ไฝ็ฌฆ + if extract_images and image_map: + markdown_content = self.process_markdown_with_images( + markdown_content, image_map + ) + + # ไฟๅญ˜ๅˆฐๆ–‡ไปถ + with open(output_file, "w", encoding="utf-8") as f: + f.write(markdown_content) + + # ่ฎก็ฎ—่ฝฌๆขๆ—ถ้—ด + duration = (datetime.now() - start_time).total_seconds() + + # ่Žทๅ–ๆ–‡ไปถๅคงๅฐ + if self.is_url(input_file): + input_size = 0 # URLๆ— ๆณ•็›ดๆŽฅ่Žทๅ–ๅคงๅฐ + else: + input_size = os.path.getsize(input_file) + output_size = os.path.getsize(output_file) + + return { + "success": True, + "input_file": input_file, + "output_file": output_file, + "input_size": input_size, + "output_size": output_size, + "duration": duration, + "markdown_content": markdown_content, + "images_extracted": images_extracted, + "image_map": image_map, + } + + except Exception as e: + return { + "success": False, + "input_file": input_file, + "error": f"Conversion failed: {str(e)}", + } + + +async def check_url_accessible(url: str) -> Dict[str, Any]: + """ๆฃ€ๆŸฅURLๆ˜ฏๅฆๅฏ่ฎฟ้—ฎ""" + try: + timeout = aiohttp.ClientTimeout(total=10) + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.head(url, allow_redirects=True) as response: + return { + "accessible": response.status < 400, + "status": response.status, + "content_type": response.headers.get("Content-Type", ""), + "content_length": response.headers.get("Content-Length", 0), + } + except Exception: + return { + "accessible": False, + "status": 0, + "content_type": "", + "content_length": 0, + } + + +async def download_file(url: str, destination: str) -> Dict[str, Any]: + """ไธ‹่ฝฝๅ•ไธชๆ–‡ไปถ""" + start_time = datetime.now() + chunk_size = 8192 + + try: + timeout = aiohttp.ClientTimeout(total=300) # 5ๅˆ†้’Ÿ่ถ…ๆ—ถ + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.get(url) as response: + # ๆฃ€ๆŸฅๅ“ๅบ”็Šถๆ€ + response.raise_for_status() + + # ่Žทๅ–ๆ–‡ไปถไฟกๆฏ + content_type = response.headers.get( + "Content-Type", "application/octet-stream" + ) + + # ็กฎไฟ็›ฎๆ ‡็›ฎๅฝ•ๅญ˜ๅœจ + parent_dir = os.path.dirname(destination) + if parent_dir: + os.makedirs(parent_dir, exist_ok=True) + + # ไธ‹่ฝฝๆ–‡ไปถ + downloaded = 0 + async with aiofiles.open(destination, "wb") as file: + async for chunk in response.content.iter_chunked(chunk_size): + await file.write(chunk) + downloaded += len(chunk) + + # ่ฎก็ฎ—ไธ‹่ฝฝๆ—ถ้—ด + duration = (datetime.now() - start_time).total_seconds() + + return { + "success": True, + "url": url, + "destination": destination, + "size": downloaded, + "content_type": content_type, + "duration": duration, + "speed": downloaded / duration if duration > 0 else 0, + } + + except aiohttp.ClientError as e: + return { + "success": False, + "url": url, + "destination": destination, + "error": f"Network error: {str(e)}", + } + except Exception as e: + return { + "success": False, + "url": url, + "destination": destination, + "error": f"Download error: {str(e)}", + } + + +async def move_local_file(source_path: str, destination: str) -> Dict[str, Any]: + """ๅคๅˆถๆœฌๅœฐๆ–‡ไปถๅˆฐ็›ฎๆ ‡ไฝ็ฝฎ๏ผˆไฟ็•™ๅŽŸๆ–‡ไปถ๏ผ‰""" + start_time = datetime.now() + + try: + # ๆฃ€ๆŸฅๆบๆ–‡ไปถๆ˜ฏๅฆๅญ˜ๅœจ + if not os.path.exists(source_path): + return { + "success": False, + "source": source_path, + "destination": destination, + "error": f"Source file not found: {source_path}", + } + + # ่Žทๅ–ๆบๆ–‡ไปถไฟกๆฏ + source_size = os.path.getsize(source_path) + + # ็กฎไฟ็›ฎๆ ‡็›ฎๅฝ•ๅญ˜ๅœจ + parent_dir = os.path.dirname(destination) + if parent_dir: + os.makedirs(parent_dir, exist_ok=True) + + # ๆ‰ง่กŒๅคๅˆถๆ“ไฝœ๏ผˆไฟ็•™ๅŽŸๆ–‡ไปถ๏ผŒ้˜ฒๆญขๆ•ฐๆฎไธขๅคฑ๏ผ‰ + shutil.copy2(source_path, destination) + + # ่ฎก็ฎ—ๆ“ไฝœๆ—ถ้—ด + duration = (datetime.now() - start_time).total_seconds() + + return { + "success": True, + "source": source_path, + "destination": destination, + "size": source_size, + "duration": duration, + "operation": "copy", # ๆ”นไธบcopy + } + + except Exception as e: + return { + "success": False, + "source": source_path, + "destination": destination, + "error": f"Copy error: {str(e)}", + } + + +@mcp.tool() +async def download_files(instruction: str) -> str: + """ + Download files from URLs or move local files mentioned in natural language instructions. + + Args: + instruction: Natural language instruction containing URLs/local paths and optional destination paths + + Returns: + Status message about the download/move operations + + Examples: + - "Download https://example.com/file.pdf to documents folder" + - "Move /home/user/file.pdf to documents folder" + - "Please get https://raw.githubusercontent.com/user/repo/main/data.csv and save it to ~/downloads" + - "็งปๅŠจ ~/Desktop/report.docx ๅˆฐ /tmp/documents/" + - "Download www.example.com/report.xlsx" + """ + urls = URLExtractor.extract_urls(instruction) + local_paths = LocalPathExtractor.extract_local_paths(instruction) + + if not urls and not local_paths: + return format_error_message( + "Failed to parse instruction", + "No downloadable URLs or movable local files found", + ) + + target_path = PathExtractor.extract_target_path(instruction) + + # ๅค„็†ๆ–‡ไปถ + results = [] + + # ๅค„็†URLไธ‹่ฝฝ + for url in urls: + try: + # ๆŽจๆ–ญๆ–‡ไปถๅ + filename = URLExtractor.infer_filename_from_url(url) + + # ๆž„ๅปบๅฎŒๆ•ด็š„็›ฎๆ ‡่ทฏๅพ„ + if target_path: + # ๅค„็†่ทฏๅพ„ + if target_path.startswith("~"): + target_path = os.path.expanduser(target_path) + + # ็กฎไฟไฝฟ็”จ็›ธๅฏน่ทฏๅพ„๏ผˆๅฆ‚ๆžœไธๆ˜ฏ็ปๅฏน่ทฏๅพ„๏ผ‰ + if not os.path.isabs(target_path): + target_path = os.path.normpath(target_path) + + # ๅˆคๆ–ญๆ˜ฏๆ–‡ไปถ่ทฏๅพ„่ฟ˜ๆ˜ฏ็›ฎๅฝ•่ทฏๅพ„ + if os.path.splitext(target_path)[1]: # ๆœ‰ๆ‰ฉๅฑ•ๅ๏ผŒๆ˜ฏๆ–‡ไปถ + destination = target_path + else: # ๆ˜ฏ็›ฎๅฝ• + destination = os.path.join(target_path, filename) + else: + # ้ป˜่ฎคไธ‹่ฝฝๅˆฐๅฝ“ๅ‰็›ฎๅฝ• + destination = filename + + # ๆฃ€ๆŸฅๆ–‡ไปถๆ˜ฏๅฆๅทฒๅญ˜ๅœจ + if os.path.exists(destination): + results.append( + f"[WARNING] Skipped {url}: File already exists at {destination}" + ) + continue + + # ๅ…ˆๆฃ€ๆŸฅURLๆ˜ฏๅฆๅฏ่ฎฟ้—ฎ + check_result = await check_url_accessible(url) + if not check_result["accessible"]: + results.append( + f"[ERROR] Failed to access {url}: HTTP {check_result['status'] or 'Connection failed'}" + ) + continue + + # ๆ‰ง่กŒไธ‹่ฝฝ + result = await download_file(url, destination) + + # ๆ‰ง่กŒ่ฝฌๆข๏ผˆๅฆ‚ๆžœๆˆๅŠŸไธ‹่ฝฝ๏ผ‰ + conversion_msg = None + if result["success"]: + conversion_msg = await perform_document_conversion( + destination, extract_images=True + ) + + # ๆ ผๅผๅŒ–็ป“ๆžœ + msg = format_file_operation_result( + "download", url, destination, result, conversion_msg + ) + + except Exception as e: + msg = f"[ERROR] Failed to download: {url}\n" + msg += f" Error: {str(e)}" + + results.append(msg) + + # ๅค„็†ๆœฌๅœฐๆ–‡ไปถ็งปๅŠจ + for local_path in local_paths: + try: + # ่Žทๅ–ๆ–‡ไปถๅ + filename = os.path.basename(local_path) + + # ๆž„ๅปบๅฎŒๆ•ด็š„็›ฎๆ ‡่ทฏๅพ„ + if target_path: + # ๅค„็†่ทฏๅพ„ + if target_path.startswith("~"): + target_path = os.path.expanduser(target_path) + + # ็กฎไฟไฝฟ็”จ็›ธๅฏน่ทฏๅพ„๏ผˆๅฆ‚ๆžœไธๆ˜ฏ็ปๅฏน่ทฏๅพ„๏ผ‰ + if not os.path.isabs(target_path): + target_path = os.path.normpath(target_path) + + # ๅˆคๆ–ญๆ˜ฏๆ–‡ไปถ่ทฏๅพ„่ฟ˜ๆ˜ฏ็›ฎๅฝ•่ทฏๅพ„ + if os.path.splitext(target_path)[1]: # ๆœ‰ๆ‰ฉๅฑ•ๅ๏ผŒๆ˜ฏๆ–‡ไปถ + destination = target_path + else: # ๆ˜ฏ็›ฎๅฝ• + destination = os.path.join(target_path, filename) + else: + # ้ป˜่ฎค็งปๅŠจๅˆฐๅฝ“ๅ‰็›ฎๅฝ• + destination = filename + + # ๆฃ€ๆŸฅ็›ฎๆ ‡ๆ–‡ไปถๆ˜ฏๅฆๅทฒๅญ˜ๅœจ + if os.path.exists(destination): + results.append( + f"[WARNING] Skipped {local_path}: File already exists at {destination}" + ) + continue + + # ๆ‰ง่กŒๅคๅˆถ๏ผˆไฟ็•™ๅŽŸๆ–‡ไปถ๏ผ‰ + result = await move_local_file(local_path, destination) + + # ๆ‰ง่กŒ่ฝฌๆข๏ผˆๅฆ‚ๆžœๆˆๅŠŸๅคๅˆถ๏ผ‰ + conversion_msg = None + if result["success"]: + conversion_msg = await perform_document_conversion( + destination, extract_images=True + ) + + # ๆ ผๅผๅŒ–็ป“ๆžœ + msg = format_file_operation_result( + "copy", local_path, destination, result, conversion_msg + ) + + except Exception as e: + msg = f"[ERROR] Failed to copy: {local_path}\n" + msg += f" Error: {str(e)}" + + results.append(msg) + + return "\n\n".join(results) + + +@mcp.tool() +async def parse_download_urls(text: str) -> str: + """ + Extract URLs, local paths and target paths from text without downloading or moving. + + Args: + text: Text containing URLs, local paths and optional destination paths + + Returns: + Parsed URLs, local paths and target path information + """ + urls = URLExtractor.extract_urls(text) + local_paths = LocalPathExtractor.extract_local_paths(text) + target_path = PathExtractor.extract_target_path(text) + + content = "๐Ÿ“‹ Parsed file operation information:\n\n" + + if urls: + content += f"๐Ÿ”— URLs found ({len(urls)}):\n" + for i, url in enumerate(urls, 1): + filename = URLExtractor.infer_filename_from_url(url) + content += f" {i}. {url}\n ๐Ÿ“„ Filename: {filename}\n" + else: + content += "๐Ÿ”— No URLs found\n" + + if local_paths: + content += f"\n๐Ÿ“ Local files found ({len(local_paths)}):\n" + for i, path in enumerate(local_paths, 1): + exists = os.path.exists(path) + content += f" {i}. {path}\n" + content += f" โœ… Exists: {'Yes' if exists else 'No'}\n" + if exists: + size_mb = os.path.getsize(path) / (1024 * 1024) + content += f" ๐Ÿ“Š Size: {size_mb:.2f} MB\n" + else: + content += "\n๐Ÿ“ No local files found\n" + + if target_path: + content += f"\n๐ŸŽฏ Target path: {target_path}" + if target_path.startswith("~"): + content += f"\n (Expanded: {os.path.expanduser(target_path)})" + else: + content += "\n๐ŸŽฏ Target path: Not specified (will use current directory)" + + return content + + +@mcp.tool() +async def download_file_to( + url: str, destination: Optional[str] = None, filename: Optional[str] = None +) -> str: + """ + Download a specific file with detailed options. + + Args: + url: URL to download from + destination: Target directory or full file path (optional) + filename: Specific filename to use (optional, ignored if destination is a full file path) + + Returns: + Status message about the download operation + """ + # ็กฎๅฎšๆ–‡ไปถๅ + + url = URLExtractor.extract_urls(url)[0] + + if not filename: + filename = URLExtractor.infer_filename_from_url(url) + + if not filename: + filename = URLExtractor.infer_filename_from_url(url) + else: + name_source, extension_source = os.path.splitext( + os.path.basename(URLExtractor.infer_filename_from_url(url)) + ) + name_destination, extension_destination = os.path.splitext( + os.path.basename(filename) + ) + if extension_source: + filename = name_destination + extension_source + else: + filename = name_destination + extension_destination + + # ็กฎๅฎšๅฎŒๆ•ด่ทฏๅพ„ + if destination: + # ๅฑ•ๅผ€็”จๆˆท็›ฎๅฝ• + if destination.startswith("~"): + destination = os.path.expanduser(destination) + + # ๆฃ€ๆŸฅๆ˜ฏๅฆๆ˜ฏๅฎŒๆ•ดๆ–‡ไปถ่ทฏๅพ„ + if os.path.splitext(destination)[1]: # ๆœ‰ๆ‰ฉๅฑ•ๅ + target_path = destination + else: # ๆ˜ฏ็›ฎๅฝ• + target_path = os.path.join(destination, filename) + else: + target_path = filename + + # ็กฎไฟไฝฟ็”จ็›ธๅฏน่ทฏๅพ„๏ผˆๅฆ‚ๆžœไธๆ˜ฏ็ปๅฏน่ทฏๅพ„๏ผ‰ + if not os.path.isabs(target_path): + target_path = os.path.normpath(target_path) + + # ๆฃ€ๆŸฅๆ–‡ไปถๆ˜ฏๅฆๅทฒๅญ˜ๅœจ + if os.path.exists(target_path): + return format_error_message( + "Download aborted", f"File already exists at {target_path}" + ) + + # ๅ…ˆๆฃ€ๆŸฅURL + check_result = await check_url_accessible(url) + if not check_result["accessible"]: + return format_error_message( + "Cannot access URL", + f"{url} (HTTP {check_result['status'] or 'Connection failed'})", + ) + + # ๆ˜พ็คบไธ‹่ฝฝไฟกๆฏ + size_mb = ( + int(check_result["content_length"]) / (1024 * 1024) + if check_result["content_length"] + else 0 + ) + msg = "[INFO] Downloading file:\n" + msg += f" URL: {url}\n" + msg += f" Target: {target_path}\n" + if size_mb > 0: + msg += f" Expected size: {size_mb:.2f} MB\n" + msg += "\n" + + # ๆ‰ง่กŒไธ‹่ฝฝ + result = await download_file(url, target_path) + + # ๆ‰ง่กŒ่ฝฌๆข๏ผˆๅฆ‚ๆžœๆˆๅŠŸไธ‹่ฝฝ๏ผ‰ + conversion_msg = None + if result["success"]: + conversion_msg = await perform_document_conversion( + target_path, extract_images=True + ) + + # ๆทปๅŠ ไธ‹่ฝฝไฟกๆฏๅ‰็ผ€ + actual_size_mb = result["size"] / (1024 * 1024) + speed_mb = result["speed"] / (1024 * 1024) + info_msg = "[SUCCESS] Download completed!\n" + info_msg += f" Saved to: {target_path}\n" + info_msg += f" Size: {actual_size_mb:.2f} MB\n" + info_msg += f" Duration: {result['duration']:.2f} seconds\n" + info_msg += f" Speed: {speed_mb:.2f} MB/s\n" + info_msg += f" Type: {result['content_type']}" + + if conversion_msg: + info_msg += conversion_msg + + return msg + info_msg + else: + return msg + f"[ERROR] Download failed!\n Error: {result['error']}" + + +@mcp.tool() +async def move_file_to( + source: str, destination: Optional[str] = None, filename: Optional[str] = None +) -> str: + """ + Copy a local file to a new location (preserves original file). + + Note: Despite the name "move_file_to", this tool COPIES the file to preserve the original. + This prevents data loss during file processing workflows. + + Args: + source: Source file path to copy + destination: Target directory or full file path (optional) + filename: Specific filename to use (optional, ignored if destination is a full file path) + + Returns: + Status message about the copy operation + """ + # ๅฑ•ๅผ€ๆบ่ทฏๅพ„ + if source.startswith("~"): + source = os.path.expanduser(source) + + # ๆฃ€ๆŸฅๆบๆ–‡ไปถๆ˜ฏๅฆๅญ˜ๅœจ + if not os.path.exists(source): + return format_error_message("Copy aborted", f"Source file not found: {source}") + + # ็กฎๅฎšๆ–‡ไปถๅ + if not filename: + filename = os.path.basename(source) + else: + name_source, extension_source = os.path.splitext(os.path.basename(source)) + name_destination, extension_destination = os.path.splitext( + os.path.basename(filename) + ) + if extension_source: + filename = name_destination + extension_source + else: + filename = name_destination + extension_destination + + # ็กฎๅฎšๅฎŒๆ•ด่ทฏๅพ„ + if destination: + # ๅฑ•ๅผ€็”จๆˆท็›ฎๅฝ• + if destination.startswith("~"): + destination = os.path.expanduser(destination) + + # ๆฃ€ๆŸฅๆ˜ฏๅฆๆ˜ฏๅฎŒๆ•ดๆ–‡ไปถ่ทฏๅพ„ + if os.path.splitext(destination)[1]: # ๆœ‰ๆ‰ฉๅฑ•ๅ + target_path = destination + else: # ๆ˜ฏ็›ฎๅฝ• + target_path = os.path.join(destination, filename) + + else: + target_path = filename + + # ็กฎไฟไฝฟ็”จ็›ธๅฏน่ทฏๅพ„๏ผˆๅฆ‚ๆžœไธๆ˜ฏ็ปๅฏน่ทฏๅพ„๏ผ‰ + if not os.path.isabs(target_path): + target_path = os.path.normpath(target_path) + + # ๆฃ€ๆŸฅ็›ฎๆ ‡ๆ–‡ไปถๆ˜ฏๅฆๅทฒๅญ˜ๅœจ + if os.path.exists(target_path): + return f"[ERROR] Target file already exists: {target_path}" + + # ๆ˜พ็คบๅคๅˆถไฟกๆฏ + source_size_mb = os.path.getsize(source) / (1024 * 1024) + msg = "[INFO] Copying file (original preserved):\n" + msg += f" Source: {source}\n" + msg += f" Target: {target_path}\n" + msg += f" Size: {source_size_mb:.2f} MB\n" + msg += "\n" + + # ๆ‰ง่กŒๅคๅˆถ๏ผˆไฟ็•™ๅŽŸๆ–‡ไปถ๏ผ‰ + result = await move_local_file(source, target_path) + + # ๆ‰ง่กŒ่ฝฌๆข๏ผˆๅฆ‚ๆžœๆˆๅŠŸๅคๅˆถ๏ผ‰ + conversion_msg = None + if result["success"]: + conversion_msg = await perform_document_conversion( + target_path, extract_images=True + ) + + # ๆทปๅŠ ๅคๅˆถไฟกๆฏๅ‰็ผ€ + info_msg = "[SUCCESS] File copied successfully (original preserved)!\n" + info_msg += f" From: {source}\n" + info_msg += f" To: {target_path}\n" + info_msg += f" Duration: {result['duration']:.2f} seconds" + + if conversion_msg: + info_msg += conversion_msg + + return msg + info_msg + else: + return msg + f"[ERROR] Copy failed!\n Error: {result['error']}" + + +# @mcp.tool() +# async def convert_document_to_markdown( +# file_path: str, output_path: Optional[str] = None, extract_images: bool = True +# ) -> str: +# """ +# Convert a document to Markdown format with image extraction support. + +# Supports both local files and URLs. Uses docling for advanced conversion with image extraction, +# or falls back to PyPDF2 for simple PDF text extraction. + +# Args: +# file_path: Path to the input document file or URL (supports PDF, DOCX, PPTX, HTML, TXT, MD) +# output_path: Path for the output Markdown file (optional, auto-generated if not provided) +# extract_images: Whether to extract images from the document (default: True) + +# Returns: +# Status message about the conversion operation with preview of converted content + +# Examples: +# - "convert_document_to_markdown('paper.pdf')" +# - "convert_document_to_markdown('https://example.com/doc.pdf', 'output.md')" +# - "convert_document_to_markdown('presentation.pptx', extract_images=False)" +# """ +# # ๆฃ€ๆŸฅๆ˜ฏๅฆไธบURL +# is_url_input = False +# try: +# parsed = urlparse(file_path) +# is_url_input = parsed.scheme in ("http", "https") +# except Exception: +# is_url_input = False + +# # ๆฃ€ๆŸฅๆ–‡ไปถๆ˜ฏๅฆๅญ˜ๅœจ๏ผˆๅฆ‚ๆžœไธๆ˜ฏURL๏ผ‰ +# if not is_url_input and not os.path.exists(file_path): +# return f"[ERROR] Input file not found: {file_path}" + +# # ๆฃ€ๆŸฅๆ˜ฏๅฆๆ˜ฏPDFๆ–‡ไปถ๏ผŒไผ˜ๅ…ˆไฝฟ็”จ็ฎ€ๅ•่ฝฌๆขๅ™จ๏ผˆไป…ๅฏนๆœฌๅœฐๆ–‡ไปถ๏ผ‰ +# if ( +# not is_url_input +# and file_path.lower().endswith(".pdf") +# and PYPDF2_AVAILABLE +# and not extract_images +# ): +# try: +# simple_converter = SimplePdfConverter() +# result = simple_converter.convert_pdf_to_markdown(file_path, output_path) +# except Exception as e: +# return f"[ERROR] PDF conversion error: {str(e)}" +# elif DOCLING_AVAILABLE: +# try: +# converter = DoclingConverter() + +# # ๆฃ€ๆŸฅๆ–‡ไปถๆ ผๅผๆ˜ฏๅฆๆ”ฏๆŒ +# if not is_url_input and not converter.is_supported_format(file_path): +# supported_formats = [".pdf", ".docx", ".pptx", ".html", ".md", ".txt"] +# return f"[ERROR] Unsupported file format. Supported formats: {', '.join(supported_formats)}" +# elif is_url_input and not file_path.lower().endswith( +# (".pdf", ".docx", ".pptx", ".html", ".md", ".txt") +# ): +# return f"[ERROR] Unsupported URL format: {file_path}" + +# # ๆ‰ง่กŒ่ฝฌๆข๏ผˆๆ”ฏๆŒๅ›พ็‰‡ๆๅ–๏ผ‰ +# result = converter.convert_to_markdown( +# file_path, output_path, extract_images +# ) +# except Exception as e: +# return f"[ERROR] Docling conversion error: {str(e)}" +# else: +# return ( +# "[ERROR] No conversion tools available. Please install docling or PyPDF2." +# ) + +# if result["success"]: +# msg = "[SUCCESS] Document converted successfully!\n" +# msg += f" Input: {result['input_file']}\n" +# msg += f" Output file: {result['output_file']}\n" +# msg += f" Conversion time: {result['duration']:.2f} seconds\n" + +# if result["input_size"] > 0: +# msg += f" Original size: {result['input_size'] / 1024:.1f} KB\n" +# msg += f" Markdown size: {result['output_size'] / 1024:.1f} KB\n" + +# # ๆ˜พ็คบๅ›พ็‰‡ๆๅ–ไฟกๆฏ +# if extract_images and "images_extracted" in result: +# images_count = result["images_extracted"] +# if images_count > 0: +# msg += f" Images extracted: {images_count}\n" +# msg += f" Images saved to: {os.path.join(os.path.dirname(result['output_file']), 'images')}\n" +# else: +# msg += " No images found in document\n" + +# # ๆ˜พ็คบMarkdownๅ†…ๅฎน็š„ๅ‰ๅ‡ ่กŒไฝœไธบ้ข„่งˆ +# content_lines = result["markdown_content"].split("\n") +# preview_lines = content_lines[:5] +# if len(content_lines) > 5: +# preview_lines.append("...") + +# msg += "\n[PREVIEW] First few lines of converted Markdown:\n" +# for line in preview_lines: +# msg += f" {line}\n" +# else: +# msg = "[ERROR] Conversion failed!\n" +# msg += f" Error: {result['error']}" + +# return msg + + +if __name__ == "__main__": + print("๐Ÿ“„ Smart PDF Downloader MCP Tool") + print("๐Ÿ“ Starting server with FastMCP...") + + if DOCLING_AVAILABLE: + print("โœ… Document conversion to Markdown is ENABLED (docling available)") + else: + print("โŒ Document conversion to Markdown is DISABLED (docling not available)") + print(" Install docling to enable: pip install docling") + + print("\nAvailable tools:") + print( + " โ€ข download_files - Download files or move local files from natural language" + ) + print(" โ€ข parse_download_urls - Extract URLs, local paths and destination paths") + print(" โ€ข download_file_to - Download a specific file with options") + print(" โ€ข move_file_to - Move a specific local file with options") + print(" โ€ข convert_document_to_markdown - Convert documents to Markdown format") + + if DOCLING_AVAILABLE: + print("\nSupported formats: PDF, DOCX, PPTX, HTML, TXT, MD") + print("Features: Image extraction, Layout preservation, Automatic conversion") + + print("") + + # ่ฟ่กŒๆœๅŠกๅ™จ + mcp.run() diff --git a/DeepCode/tools/pdf_utils.py b/DeepCode/tools/pdf_utils.py new file mode 100644 index 00000000..92cc29de --- /dev/null +++ b/DeepCode/tools/pdf_utils.py @@ -0,0 +1,52 @@ +""" +PDF utility functions for the DeepCode agent system. +""" + +from pathlib import Path +import PyPDF2 + + +def read_pdf_metadata(file_path: Path) -> dict: + """Read PDF metadata with proper encoding handling.""" + try: + print(f"\nAttempting to read PDF metadata from: {file_path}") + with open(file_path, "rb") as file: + pdf_reader = PyPDF2.PdfReader(file) + info = pdf_reader.metadata + first_page = pdf_reader.pages[0] + text = first_page.extract_text() + lines = text.split("\n")[:10] + + title = None + authors = [] + + if info: + title = info.get("/Title", "").strip().replace("\x00", "") + author = info.get("/Author", "").strip().replace("\x00", "") + if author: + authors = [author] + + if not title and lines: + title = lines[0].strip() + + if not authors and len(lines) > 1: + for line in lines[1:3]: + if "author" in line.lower() or "by" in line.lower(): + authors = [line.strip()] + break + + return { + "title": title if title else "Unknown Title", + "authors": authors if authors else ["Unknown Author"], + "year": info.get("/CreationDate", "")[:4] if info else "Unknown Year", + "first_lines": lines, + } + + except Exception as e: + print(f"\nError reading PDF: {str(e)}") + return { + "title": "Error reading PDF", + "authors": ["Unknown"], + "year": "Unknown", + "first_lines": [], + } diff --git a/DeepCode/ui/__init__.py b/DeepCode/ui/__init__.py new file mode 100644 index 00000000..c59580a3 --- /dev/null +++ b/DeepCode/ui/__init__.py @@ -0,0 +1,43 @@ +""" +UI Module + +Streamlit application user interface components module + +Contains the following submodules: +- styles: CSS styles +- components: UI components +- layout: Page layout +- handlers: Event handlers +- streamlit_app: Main application +- app: Application entry +""" + +__version__ = "1.0.0" +__author__ = "DeepCode Team" + +# Import main components +from .layout import main_layout +from .components import display_header, display_features, display_status +from .handlers import initialize_session_state +from .styles import get_main_styles + +# Import application main function +try: + from .streamlit_app import main as streamlit_main +except ImportError: + # Fallback to absolute import if relative import fails + import sys + import os + + sys.path.insert(0, os.path.dirname(__file__)) + from streamlit_app import main as streamlit_main + +__all__ = [ + "main_layout", + "display_header", + "display_features", + "display_status", + "initialize_session_state", + "get_main_styles", + "streamlit_main", +] diff --git a/DeepCode/ui/app.py b/DeepCode/ui/app.py new file mode 100644 index 00000000..cce86915 --- /dev/null +++ b/DeepCode/ui/app.py @@ -0,0 +1,13 @@ +""" +DeepCode UI Application Entry Point + +This file serves as the unified entry point for the UI module +""" + +from .streamlit_app import main + +# Directly export main function for external calls +__all__ = ["main"] + +if __name__ == "__main__": + main() diff --git a/DeepCode/ui/components.py b/DeepCode/ui/components.py new file mode 100644 index 00000000..39d7cde7 --- /dev/null +++ b/DeepCode/ui/components.py @@ -0,0 +1,970 @@ +# -*- coding: utf-8 -*- +""" +Streamlit UI Components - Cyber Edition +Contains all reusable UI components with new styling plus +the operational widgets required by the handlers. +""" + +from __future__ import annotations + +import html +import base64 +import sys +from datetime import datetime +from functools import lru_cache +from pathlib import Path +from typing import Dict, Any, Optional, List, Tuple + +import streamlit as st + +from utils.cross_platform_file_handler import get_file_handler + +BASE_DIR = Path(__file__).resolve().parents[1] +ICON_DIR = BASE_DIR / "assets" / "icons" + + +@lru_cache(maxsize=64) +def _icon_data_uri(name: str) -> str: + path = ICON_DIR / f"{name}.png" + if not path.exists(): + return "" + + try: + data = path.read_bytes() + except OSError: + return "" + + encoded = base64.b64encode(data).decode("utf-8") + return f"data:image/png;base64,{encoded}" + + +def icon_img(name: str, size: int = 32, extra_style: str = "") -> str: + """ + Render an inline tag for icons stored in assets/icons via data URI. + """ + data_uri = _icon_data_uri(name) + if not data_uri: + return "" + return f'{name}' + + +def clear_guided_answer_inputs(): + """Remove temporary answer widgets from session state.""" + keys_to_delete = [ + key for key in st.session_state.keys() if key.startswith("guided_answer_") + ] + for key in keys_to_delete: + del st.session_state[key] + + +def display_header(): + """Display the Cyber-styled header""" + st.markdown( + """ +
+
+
DEEPCODE
+
Autonomous Research & Engineering Matrix
+
+
+
+ SYSTEM ONLINE +
+
+ """, + unsafe_allow_html=True, + ) + + +def display_features(): + """Display feature cards grid""" + feature_cards = [ + { + "icon": "feature_synthesis", + "fallback": "๐Ÿงฌ", + "title": "Neural Synthesis", + "desc": "Transform research papers directly into executable repositories via multi-agent LLM pipelines.", + }, + { + "icon": "feature_hyper", + "fallback": "โšก", + "title": "Hyper-Speed Mode", + "desc": "Acceleration layer that parallelizes retrieval, planning, and implementation for fastest delivery.", + }, + { + "icon": "feature_cognition", + "fallback": "๐Ÿง ", + "title": "Cognitive Context", + "desc": "Semantic memory graphs retain methodology, datasets, and evaluation strategy during reasoning.", + }, + { + "icon": "feature_secure", + "fallback": "๐Ÿ›ก๏ธ", + "title": "Secure Sandbox(Coming Soon)", + "desc": "Isolated execution & validation environment keeps experiments safe and reproducible.", + }, + ] + + cards_html = "" + for card in feature_cards: + icon_markup = icon_img( + card["icon"], + 48, + "filter:drop-shadow(0 0 10px rgba(0,242,255,0.4));", + ) + if not icon_markup: + icon_markup = f'{card["fallback"]}' + + cards_html += f""" +
+
+ {icon_markup} +
+
{card['title']}
+
{card['desc']}
+
+ """ + + st.markdown( + f""" +
+ {cards_html} +
+ """, + unsafe_allow_html=True, + ) + + +def display_status(message: str, status_type: str = "info"): + """Display status message with cyber styling""" + colors = { + "success": "var(--success)", + "error": "var(--error)", + "warning": "var(--warning)", + "info": "var(--primary)", + } + color = colors.get(status_type, "var(--primary)") + + st.markdown( + f""" +
+ [{status_type.upper()}] + {message} +
+ """, + unsafe_allow_html=True, + ) + + +def _render_step_card(title: str, subtitle: str, state: str) -> str: + """Return HTML for a workflow step badge.""" + colors = { + "completed": "var(--success)", + "active": "var(--primary)", + "pending": "rgba(255,255,255,0.3)", + "error": "var(--error)", + } + icon = { + "completed": "โœ”", + "active": "โžค", + "pending": "โ€ข", + "error": "!", + }.get(state, "โ€ข") + color = colors.get(state, "rgba(255,255,255,0.3)") + return f""" +
+
{icon}
+
{title}
+
{subtitle}
+
+ """ + + +def enhanced_progress_display_component( + enable_indexing: bool, chat_mode: bool +) -> Tuple[Any, Any, List[Any], List[Dict[str, str]]]: + """ + Render the progress panel required by handlers.handle_processing_workflow. + """ + + if chat_mode: + workflow_steps = [ + {"title": "INIT", "subtitle": "Boot agents"}, + {"title": "PLAN", "subtitle": "Analyze intent"}, + {"title": "SETUP", "subtitle": "Workspace"}, + {"title": "DRAFT", "subtitle": "Generate plan"}, + {"title": "CODE", "subtitle": "Implement"}, + ] + elif not enable_indexing: + workflow_steps = [ + {"title": "INIT", "subtitle": "Load systems"}, + {"title": "ANALYZE", "subtitle": "Parse paper"}, + {"title": "DOWNLOAD", "subtitle": "Collect refs"}, + {"title": "PLAN", "subtitle": "Blueprint"}, + {"title": "CODE", "subtitle": "Implement"}, + ] + else: + workflow_steps = [ + {"title": "INIT", "subtitle": "Load systems"}, + {"title": "ANALYZE", "subtitle": "Paper scan"}, + {"title": "DOWNLOAD", "subtitle": "Docs & data"}, + {"title": "PLAN", "subtitle": "Architect"}, + {"title": "REF", "subtitle": "Key refs"}, + {"title": "REPO", "subtitle": "GitHub sync"}, + {"title": "INDEX", "subtitle": "Vectorize"}, + {"title": "CODE", "subtitle": "Implementation"}, + ] + + st.markdown("### ๐Ÿ›ฐ๏ธ Workflow Monitor") + progress_bar = st.progress(0) + status_text = st.empty() + + cols = st.columns(len(workflow_steps)) + step_indicators: List[Any] = [] + for col, step in zip(cols, workflow_steps): + with col: + placeholder = st.empty() + placeholder.markdown( + _render_step_card(step["title"], step["subtitle"], "pending"), + unsafe_allow_html=True, + ) + step_indicators.append(placeholder) + + return progress_bar, status_text, step_indicators, workflow_steps + + +def update_step_indicator( + step_indicators: List[Any], + workflow_steps: List[Dict[str, str]], + current_step: int, + status: str, +): + """ + Update the workflow step indicators in-place. + """ + total_steps = len(workflow_steps) + + for idx, placeholder in enumerate(step_indicators): + if status == "error" and idx == current_step: + state = "error" + elif current_step >= total_steps: + state = "completed" + elif idx < current_step: + state = "completed" + elif idx == current_step: + state = "active" + else: + state = "pending" + + step = workflow_steps[idx] + placeholder.markdown( + _render_step_card(step["title"], step["subtitle"], state), + unsafe_allow_html=True, + ) + + +def chat_input_component(task_counter: int = 0) -> Optional[str]: + """Render modern chat input for guided mode""" + st.markdown("### ๐Ÿ’ฌ Neural Link Interface") + + user_input = st.chat_input( + placeholder="Input research directive or query...", + key=f"chat_input_{task_counter}", + ) + return user_input + + +def _save_uploaded_pdf(uploaded_file) -> Optional[str]: + """Persist uploaded PDF to a temp file and return its path.""" + try: + file_bytes = uploaded_file.read() + suffix = Path(uploaded_file.name).suffix or ".pdf" + handler = get_file_handler() + temp_path = handler.create_safe_temp_file( + suffix=suffix, prefix="deepcode_upload_", content=file_bytes + ) + return str(temp_path) + except Exception as exc: + st.error(f"Failed to save uploaded file: {exc}") + return None + + +def input_method_selector(task_counter: int) -> Tuple[Optional[str], Optional[str]]: + """Render the input method selection tabs with modern styling""" + + tab1, tab2, tab3 = st.tabs(["๐Ÿ“„ PDF UPLOAD", "๐Ÿ”— URL LINK", "โšก QUICK COMMAND"]) + + input_source: Optional[str] = None + input_type: Optional[str] = None + + with tab1: + st.markdown('
', unsafe_allow_html=True) + uploaded_file = st.file_uploader( + "Upload Research Paper (PDF)", + type="pdf", + key=f"file_uploader_{task_counter}", + ) + if uploaded_file: + saved_path = _save_uploaded_pdf(uploaded_file) + if saved_path: + st.session_state["uploaded_filename"] = uploaded_file.name + input_source = saved_path + input_type = "file" + + with tab2: + st.markdown('
', unsafe_allow_html=True) + url = st.text_input( + "ArXiv / GitHub Resource URL", + placeholder="https://arxiv.org/abs/...", + key=f"url_input_{task_counter}", + ) + if url: + input_source = url.strip() + input_type = "url" + + with tab3: + st.markdown('
', unsafe_allow_html=True) + query = st.text_area( + "Code Specifications / Abstract", + placeholder="Describe the algorithm or system requirements...", + height=150, + key=f"text_input_{task_counter}", + ) + if query: + input_source = query.strip() + input_type = "chat" + + return input_source, input_type + + +def results_display_component(result: Any, task_counter: int): + """Display results in a tech-styled container""" + + status = result.get("status", "unknown") + is_success = status == "success" + status_label = "Mission Complete" if is_success else "Execution Failed" + status_color = "var(--success)" if is_success else "var(--error)" + status_icon = icon_img("status_success" if is_success else "status_error", 56) + if not status_icon: + status_icon = "โœ…" if is_success else "โš ๏ธ" + status_message = ( + "Computation sequence completed successfully." + if is_success + else result.get("error", "Unknown error occurred during processing.") + ) + + st.markdown('
', unsafe_allow_html=True) + st.markdown("### ๐Ÿš€ Operation Result") + + with st.container(): + if is_success: + st.success("Workflow completed across all stages โœ…") + else: + st.error("Workflow interrupted. Check the logs below โš ๏ธ") + + col1, col2 = st.columns([2, 1]) + with col1: + with st.expander("๐Ÿ“œ Execution Logs & Metadata", expanded=True): + st.json(result) + + with col2: + st.markdown( + f""" +
+
{status_icon}
+
{status_label}
+
{status_message}
+
+ """, + unsafe_allow_html=True, + ) + st.download_button( + label="๐Ÿ“ฅ DOWNLOAD ARTIFACTS" if is_success else "๐Ÿ“ฅ DOWNLOAD LOGS", + data=str(result), + file_name=f"deepcode_result_{task_counter}.json", + mime="application/json", + use_container_width=True, + ) + + +def system_status_component(): + """System status check component""" + st.markdown("### ๐Ÿ”ง System Diagnostics") + + col1, col2 = st.columns(2) + + with col1: + st.markdown("#### ๐Ÿ“Š Core Metrics") + st.info(f"**Python:** {sys.version.split()[0]}") + st.info(f"**Platform:** {sys.platform}") + + with col2: + st.markdown("#### โš™๏ธ Runtime Status") + try: + import asyncio + + loop = asyncio.get_event_loop() + if loop.is_running(): + st.success("Event Loop: ACTIVE") + else: + st.warning("Event Loop: STANDBY") + except Exception: + st.info("Event Loop: MANAGED") + + +def error_troubleshooting_component(): + """Error troubleshooting component""" + with st.expander("๐Ÿ› ๏ธ Diagnostics & Troubleshooting", expanded=False): + st.warning( + "If you encounter issues, please check your API keys in the sidebar." + ) + + +def footer_component(): + """Minimal futuristic footer""" + st.markdown( + """ +
+ DEEPCODE_SYSTEMS // OPERATIONAL // VERSION 3.0.1 +
+ """, + unsafe_allow_html=True, + ) + + +def render_sidebar_feed(max_items: int = 12): + """Render live mission feed inside sidebar.""" + st.markdown("#### ๐Ÿ“ก Mission Feed") + events = list(st.session_state.get("sidebar_events", [])) + + col1, col2 = st.columns([1, 1]) + with col1: + st.caption("Real-time agent telemetry") + with col2: + if st.button("Clear Feed", key="sidebar_clear_feed"): + st.session_state.sidebar_events = [] + events = [] + st.session_state.sidebar_feed_last_cleared = datetime.utcnow().strftime( + "%H:%M:%S" + ) + + if not events: + st.caption("Awaiting activity...") + return + + recent_events = list(reversed(events[-max_items:])) + for event in recent_events: + stage = event.get("stage", "STAGE") + message = html.escape(str(event.get("message", ""))) + timestamp = event.get("timestamp", "--:--:--") + level = event.get("level", "info") + extra = event.get("extra") + + st.markdown( + f""" +
+
+ {stage} + {timestamp} +
+
{message}
+
+ """, + unsafe_allow_html=True, + ) + + if isinstance(extra, dict) and extra: + with st.expander("Details", expanded=False): + st.json(extra) + + +def render_system_monitor(): + """Display current backend + command telemetry.""" + st.markdown("#### ๐Ÿงฌ System Monitor") + processing = st.session_state.get("processing", False) + mode = st.session_state.get("requirement_analysis_mode", "direct").upper() + indexing_enabled = st.session_state.get("enable_indexing", True) + task_counter = st.session_state.get("task_counter", 0) + last_error = st.session_state.get("last_error") + events = st.session_state.get("sidebar_events", []) + latest_event = events[-1] if events else None + last_stage = latest_event.get("stage") if latest_event else "--" + last_message = ( + html.escape(str(latest_event.get("message", ""))) if latest_event else "" + ) + last_progress = ( + latest_event.get("extra", {}).get("progress") if latest_event else None + ) + state_label = "ACTIVE" if processing else "IDLE" + + st.markdown( + f""" +
+
+
STATE{state_label}
+
MODE{mode}
+
INDEXING{"ON" if indexing_enabled else "OFF"}
+
TASKS{task_counter}
+
+
+ {last_stage if last_stage else "--"} + {"ยท " + str(last_progress) + "%" if last_progress is not None else ""} +
{last_message or "Awaiting telemetry..."} +
+
+ """, + unsafe_allow_html=True, + ) + + if last_error: + st.warning(f"Last error: {last_error}") + + +def render_log_viewer(max_lines: int = 50): + """Display live log stream for current mission in a scrollable container.""" + st.markdown("#### ๐Ÿ“ Live Log Stream") + logs_dir = BASE_DIR / "logs" + if not logs_dir.exists(): + st.info("Logs directory not found.") + return + + log_files = sorted( + [p for p in logs_dir.glob("*.jsonl") if p.is_file()], + key=lambda p: p.stat().st_mtime, + reverse=True, + ) + if not log_files: + st.info("No log files available yet.") + return + + start_ts = st.session_state.get("workflow_start_time") + selected_path = None + + waiting_for_new_log = False + + if start_ts: + # Use a tolerance window: accept logs created within 10 seconds before workflow_start_time + tolerance = 10.0 + for candidate in log_files: + file_mtime = candidate.stat().st_mtime + if file_mtime >= (start_ts - tolerance): + selected_path = candidate + break + if selected_path is None: + waiting_for_new_log = True + else: + prev = st.session_state.get("active_log_file") + if prev: + prev_path = Path(prev) + if prev_path.exists(): + selected_path = prev_path + if selected_path is None: + selected_path = log_files[0] + + if waiting_for_new_log: + st.caption("Waiting for current task log to be created...") + return + + st.session_state.active_log_file = str(selected_path) + + try: + content = selected_path.read_text(encoding="utf-8", errors="ignore") + except Exception as exc: + st.error(f"Failed to read {selected_path.name}: {exc}") + return + + lines = content.splitlines() + tail_lines = lines[-max_lines:] + + # Show file info + processing = st.session_state.get("processing", False) + status_icon = "๐Ÿ”„" if processing else "โœ…" + st.caption(f"{status_icon} {selected_path.name} | Last {len(tail_lines)} lines") + + if not tail_lines: + st.info("Log file is empty.") + return + + # Build log HTML with scrollable container + import json + + log_html_parts = [] + + for line in tail_lines: + line = line.strip() + if not line: + continue + + try: + event = json.loads(line) + timestamp = event.get("timestamp", "") + level = event.get("level", "INFO") + message = event.get("message", "") + namespace = event.get("namespace", "") + + # Color code by level + if level == "ERROR": + level_color = "#ff4444" + elif level == "WARNING": + level_color = "#ffaa00" + elif "SUCCESS" in level.upper(): + level_color = "#00ff88" + else: + level_color = "#00d4ff" + + # Format display + time_str = ( + timestamp.split("T")[-1][:12] if "T" in timestamp else timestamp[-12:] + ) + namespace_short = namespace.split(".")[-1] if namespace else "" + + log_html_parts.append( + f'
' + f'{time_str} ' + f'[{level}] ' + f'{namespace_short}
' + f'{message[:200]}' + f"
" + ) + except json.JSONDecodeError: + # Raw text fallback + log_html_parts.append( + f'
{line[:200]}
' + ) + + # Render in scrollable container + full_log_html = f""" +
+ {''.join(log_html_parts)} +
+ """ + + st.markdown(full_log_html, unsafe_allow_html=True) + + +def reset_guided_workflow_state(preserve_initial: bool = False): + """ + Reset guided requirement workflow state machine. + """ + if preserve_initial: + initial_text = st.session_state.get( + "guided_initial_requirement", + st.session_state.get("initial_requirement", ""), + ) + else: + initial_text = "" + st.session_state.initial_requirement = "" + + st.session_state.guided_initial_requirement = initial_text + st.session_state.guided_edit_feedback = "" + st.session_state.requirement_analysis_step = "input" + st.session_state.generated_questions = [] + st.session_state.user_answers = {} + st.session_state.detailed_requirements = "" + st.session_state.questions_generating = False + st.session_state.requirements_generating = False + st.session_state.requirements_confirmed = False + st.session_state.requirements_editing = False + st.session_state.edit_feedback = "" + st.session_state.confirmed_requirement_text = None + clear_guided_answer_inputs() + + +def requirement_mode_selector() -> str: + """ + Render the requirement workflow mode selector. + """ + mode_labels = {"direct": "๐Ÿš€ Direct Mode", "guided": "๐Ÿงญ Guided Mode"} + current_mode = st.session_state.get("requirement_analysis_mode", "direct") + + selection = st.radio( + "Requirement Intake Mode", + options=list(mode_labels.keys()), + index=0 if current_mode != "guided" else 1, + horizontal=True, + format_func=lambda key: mode_labels[key], + key="requirement_mode_selector_radio", + ) + + if selection != current_mode: + st.session_state.requirement_analysis_mode = selection + if selection == "direct": + reset_guided_workflow_state(preserve_initial=False) + else: + st.session_state.requirement_analysis_step = "input" + + return selection + + +def guided_requirement_workflow() -> Tuple[Optional[str], bool]: + """ + Render the guided requirement analysis workflow. + """ + + st.markdown("### ๐Ÿงญ Guided Requirement Workflow") + + step = st.session_state.get("requirement_analysis_step", "input") + st.session_state.setdefault( + "guided_initial_requirement", st.session_state.get("initial_requirement", "") + ) + st.session_state.setdefault( + "guided_edit_feedback", st.session_state.get("edit_feedback", "") + ) + + step_titles = { + "input": "Step 1 ยท Describe Requirements", + "questions": "Step 2 ยท Answer Guiding Questions", + "summary": "Step 3 ยท Review Requirement Document", + "editing": "Step 4 ยท Request Changes", + } + st.caption( + f"Current Stage: {step_titles.get(step, 'Step 1 ยท Describe Requirements')}" + ) + + confirmed_doc = st.session_state.get("confirmed_requirement_text") + + if step == "input": + st.markdown("#### 1 ยท Describe your project") + st.text_area( + "Describe the product scope, tech stack, performance targets, and constraints:", + key="guided_initial_requirement", + height=180, + ) + initial_text = st.session_state.get("guided_initial_requirement", "") + + col1, col2 = st.columns(2) + with col1: + if st.button("Generate guiding questions", type="primary"): + if not initial_text.strip(): + st.warning("Please enter your project requirements first.") + else: + st.session_state.initial_requirement = initial_text.strip() + st.session_state.questions_generating = True + st.session_state.requirement_analysis_step = "questions" + st.session_state.generated_questions = [] + st.session_state.user_answers = {} + st.session_state.detailed_requirements = "" + st.session_state.confirmed_requirement_text = None + st.session_state.requirements_generating = False + st.session_state.requirements_confirmed = False + st.session_state.requirements_editing = False + st.session_state.edit_feedback = "" + clear_guided_answer_inputs() + st.rerun() + + with col2: + if st.button("Skip Q&A and use current spec", type="secondary"): + if not initial_text.strip(): + st.warning("Please enter your project requirements first.") + else: + final_doc = initial_text.strip() + st.session_state.initial_requirement = final_doc + st.session_state.confirmed_requirement_text = final_doc + st.session_state.requirements_confirmed = True + st.success( + "Current description locked as the requirement document. Implementation will proceed next." + ) + + elif step == "questions": + st.markdown("#### 2 ยท Answer guiding questions") + if st.session_state.get("questions_generating"): + st.info("LLM is crafting guiding questions. Please wait...") + + questions = st.session_state.get("generated_questions", []) + question_ids: List[str] = [] + + if not questions: + st.caption("Guiding questions will appear once generation is complete.") + else: + for idx, question in enumerate(questions): + if isinstance(question, dict): + q_id = str( + question.get("id") + or question.get("question_id") + or question.get("qid") + or idx + ) + q_text = question.get("question") or question.get("content") or "" + category = question.get("category") + importance = question.get("importance") + hint = question.get("hint") + else: + q_id = str(idx) + q_text = str(question) + category = importance = hint = None + + question_ids.append(q_id) + + st.markdown( + f"**Q{idx + 1}. {q_text or 'Please answer this question'}**" + ) + meta_parts = [part for part in [category, importance] if part] + if meta_parts: + st.caption(" / ".join(meta_parts)) + if hint: + st.caption(f"Hint: {hint}") + + answer_key = f"guided_answer_{idx}" + if answer_key not in st.session_state: + default_answer = st.session_state.user_answers.get(q_id, "") + st.session_state[answer_key] = default_answer + + st.text_area("Your answer", key=answer_key, height=100) + + col1, col2, col3 = st.columns(3) + with col1: + if st.button( + "Generate requirement document", type="primary", disabled=not questions + ): + answers_payload = {} + for idx, q_id in enumerate(question_ids): + answer_value = st.session_state.get( + f"guided_answer_{idx}", "" + ).strip() + if answer_value: + answers_payload[q_id] = answer_value + + st.session_state.user_answers = answers_payload + st.session_state.requirements_generating = True + st.session_state.requirement_analysis_step = "summary" + st.session_state.detailed_requirements = "" + st.session_state.confirmed_requirement_text = None + st.session_state.requirements_confirmed = False + st.rerun() + + with col2: + if st.button( + "Generate without answers", type="secondary", disabled=not questions + ): + st.session_state.user_answers = {} + st.session_state.requirements_generating = True + st.session_state.requirement_analysis_step = "summary" + st.session_state.detailed_requirements = "" + st.session_state.confirmed_requirement_text = None + st.session_state.requirements_confirmed = False + st.rerun() + + with col3: + if st.button("Back to Step 1"): + reset_guided_workflow_state(preserve_initial=True) + st.rerun() + + elif step == "summary": + st.markdown("#### 3 ยท AI-generated requirement document") + if st.session_state.get("requirements_generating"): + st.info("Generating requirement document. Please wait...") + + summary = (st.session_state.get("detailed_requirements") or "").strip() + + if summary: + st.markdown(summary) + st.download_button( + "Download requirement document", + summary, + file_name="deepcode_requirements.md", + mime="text/markdown", + use_container_width=True, + ) + else: + st.caption("Waiting for requirement document to be generated...") + + col1, col2, col3 = st.columns(3) + with col1: + if st.button( + "Confirm and start implementation โœ…", + type="primary", + disabled=not summary, + ): + final_doc = summary or st.session_state.get("initial_requirement", "") + if final_doc.strip(): + st.session_state.confirmed_requirement_text = final_doc.strip() + st.session_state.requirements_confirmed = True + st.success( + "Requirement document confirmed. Implementation pipeline will start next." + ) + else: + st.warning("No requirement document available yet.") + + with col2: + if st.button("Request edits", type="secondary", disabled=not summary): + st.session_state.requirement_analysis_step = "editing" + st.session_state.guided_edit_feedback = "" + + with col3: + if st.button("Restart Q&A", type="secondary"): + reset_guided_workflow_state(preserve_initial=True) + st.rerun() + + elif step == "editing": + st.markdown("#### 4 ยท Modify requirement document") + st.text_area( + "Describe the changes or clarifications you need:", + key="guided_edit_feedback", + height=160, + ) + feedback_value = st.session_state.get("guided_edit_feedback", "") + + col1, col2 = st.columns(2) + with col1: + if st.button("Submit change request", type="primary"): + if not feedback_value.strip(): + st.warning("Please describe the requested changes.") + else: + st.session_state.edit_feedback = feedback_value.strip() + st.session_state.requirements_editing = True + st.info("Updating requirement document based on your feedback...") + + with col2: + if st.button("Back to requirement document"): + st.session_state.requirement_analysis_step = "summary" + st.session_state.guided_edit_feedback = "" + + if st.session_state.get("requirements_editing"): + st.info("Requirement document is updating...") + + if confirmed_doc: + st.success("Requirement document locked. You can start implementation anytime.") + + return (confirmed_doc if confirmed_doc else None, bool(confirmed_doc)) + + +def sidebar_control_panel(): + """Sidebar configuration""" + with st.sidebar: + st.markdown( + """ +
+

CONTROL DECK

+
// MISSION CONTROL
+
+ """, + unsafe_allow_html=True, + ) + + workflow_start = st.session_state.get("workflow_start_time") + + if workflow_start: + render_log_viewer() + else: + st.info("Awaiting next mission run to stream logs.") + st.markdown( + """ +
+ ยฉ 2024 DeepCode Research +
+ """, + unsafe_allow_html=True, + ) + + return {} diff --git a/DeepCode/ui/handlers.py b/DeepCode/ui/handlers.py new file mode 100644 index 00000000..c680c978 --- /dev/null +++ b/DeepCode/ui/handlers.py @@ -0,0 +1,1189 @@ +""" +Streamlit Event Handlers Module + +Contains all event handling and business logic +""" + +import asyncio +import time +import os +import traceback +import atexit +import signal +from datetime import datetime +from typing import Dict, Any + +import streamlit as st +import nest_asyncio +import concurrent.futures + +# Import necessary modules +from mcp_agent.app import MCPApp +from workflows.agent_orchestration_engine import ( + execute_multi_agent_research_pipeline, + execute_chat_based_planning_pipeline, +) +from .sidebar_feed import log_sidebar_event, ensure_sidebar_logging + + +def _emergency_cleanup(): + """ + Emergency resource cleanup function + Called when program exits abnormally + """ + try: + cleanup_resources() + except Exception: + pass # Silent handling to avoid new exceptions during exit + + +def _signal_handler(signum, frame): + """ + Signal handler for program termination signals + """ + try: + cleanup_resources() + except Exception: + pass + finally: + # Restore default signal handling and resend signal + signal.signal(signum, signal.SIG_DFL) + os.kill(os.getpid(), signum) + + +# Register exit cleanup function +atexit.register(_emergency_cleanup) + + +def _safe_register_signal_handlers(): + """Safely register signal handlers""" + try: + # Check if running in main thread + import threading + + if threading.current_thread() is not threading.main_thread(): + return # Signal handlers can only be registered in main thread + + # Try to register signal handlers + signal.signal(signal.SIGTERM, _signal_handler) + signal.signal(signal.SIGINT, _signal_handler) + if hasattr(signal, "SIGBREAK"): # Windows + signal.signal(signal.SIGBREAK, _signal_handler) + except (AttributeError, OSError, ValueError): + # Some signals are not available on certain platforms or disabled in some environments + # This is common in web frameworks like Streamlit + pass + + +# Delayed signal handler registration to avoid import-time errors +try: + _safe_register_signal_handlers() +except Exception: + # If registration fails, silently ignore and don't affect app startup + pass + + +async def process_input_async( + input_source: str, + input_type: str, + enable_indexing: bool = True, + progress_callback=None, +) -> Dict[str, Any]: + """ + Process input asynchronously + + Args: + input_source: Input source + input_type: Input type + enable_indexing: Whether to enable indexing functionality + progress_callback: Progress callback function + + Returns: + Processing result + """ + try: + # Create and use MCP app in the same async context + app = MCPApp(name="paper_to_code") + + async with app.run() as agent_app: + logger = agent_app.logger + context = agent_app.context + context.config.mcp.servers["filesystem"].args.extend([os.getcwd()]) + + # Initialize progress + if progress_callback: + if input_type == "chat": + progress_callback( + 5, "๐Ÿš€ Initializing chat-based planning pipeline..." + ) + else: + progress_callback(5, "๐Ÿš€ Initializing AI research engine...") + + # Choose pipeline based on input type + if input_type == "chat": + # Use chat-based planning pipeline for user requirements + repo_result = await execute_chat_based_planning_pipeline( + input_source, # User's coding requirements + logger, + progress_callback, + enable_indexing=enable_indexing, # Pass indexing control parameter + ) + else: + # Use traditional multi-agent research pipeline for files/URLs + repo_result = await execute_multi_agent_research_pipeline( + input_source, + logger, + progress_callback, + enable_indexing=enable_indexing, # Pass indexing control parameter + ) + + return { + "analysis_result": "Integrated into complete workflow", + "download_result": "Integrated into complete workflow", + "repo_result": repo_result, + "status": "success", + } + + except Exception as e: + error_msg = str(e) + traceback_msg = traceback.format_exc() + + return {"error": error_msg, "traceback": traceback_msg, "status": "error"} + + +def run_async_task(coro): + """ + Helper function to run async tasks + + Args: + coro: Coroutine object + + Returns: + Task result + """ + # Apply nest_asyncio to support nested event loops + nest_asyncio.apply() + + # Save current Streamlit context + try: + from streamlit.runtime.scriptrunner import get_script_run_ctx + from streamlit.runtime.scriptrunner.script_run_context import ( + SCRIPT_RUN_CONTEXT_ATTR_NAME, + ) + + current_ctx = get_script_run_ctx() + context_available = True + except ImportError: + # If Streamlit context modules can't be imported, use fallback method + current_ctx = None + context_available = False + + def run_in_new_loop(): + """Run coroutine in new event loop""" + # Set Streamlit context in new thread (if available) + if context_available and current_ctx: + try: + import threading + + setattr( + threading.current_thread(), + SCRIPT_RUN_CONTEXT_ATTR_NAME, + current_ctx, + ) + except Exception: + pass # Ignore context setting errors + + loop = None + try: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + result = loop.run_until_complete(coro) + return result + except Exception as e: + raise e + finally: + # Clean up resources + if loop: + try: + loop.close() + except Exception: + pass + asyncio.set_event_loop(None) + + # Clean up thread context (if available) + if context_available: + try: + import threading + + if hasattr( + threading.current_thread(), SCRIPT_RUN_CONTEXT_ATTR_NAME + ): + delattr( + threading.current_thread(), SCRIPT_RUN_CONTEXT_ATTR_NAME + ) + except Exception: + pass # Ignore cleanup errors + + # Force garbage collection + import gc + + gc.collect() + + # Use thread pool to run async task, avoiding event loop conflicts + executor = None + try: + executor = concurrent.futures.ThreadPoolExecutor( + max_workers=1, thread_name_prefix="deepcode_ctx_async" + ) + future = executor.submit(run_in_new_loop) + result = future.result(timeout=300) # 5 minute timeout + return result + except concurrent.futures.TimeoutError: + st.error("Processing timeout after 5 minutes. Please try again.") + raise TimeoutError("Processing timeout") + except Exception as e: + # If thread pool execution fails, try direct execution + st.warning(f"Threaded async execution failed: {e}, trying direct execution...") + try: + # Fallback method: run directly in current thread + loop = None + try: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + result = loop.run_until_complete(coro) + return result + finally: + if loop: + try: + loop.close() + except Exception: + pass + asyncio.set_event_loop(None) + import gc + + gc.collect() + except Exception as backup_error: + st.error(f"All execution methods failed: {backup_error}") + raise backup_error + finally: + # Ensure thread pool is properly closed + if executor: + try: + executor.shutdown(wait=True, cancel_futures=True) + except Exception: + pass + # Force garbage collection + import gc + + gc.collect() + + +def run_async_task_simple(coro): + """ + Simple async task runner, avoiding threading issues + + Args: + coro: Coroutine object + + Returns: + Task result + """ + # Apply nest_asyncio to support nested event loops + nest_asyncio.apply() + + try: + # Try to run in current event loop + loop = asyncio.get_event_loop() + if loop.is_running(): + # If current loop is running, use improved thread pool method + import concurrent.futures + import gc + + def run_in_thread(): + # Create new event loop and set as current thread's loop + new_loop = asyncio.new_event_loop() + asyncio.set_event_loop(new_loop) + try: + result = new_loop.run_until_complete(coro) + return result + except Exception as e: + # Ensure exception information is properly passed + raise e + finally: + # Ensure loop is properly closed + try: + new_loop.close() + except Exception: + pass + # Clear current thread's event loop reference + asyncio.set_event_loop(None) + # Force garbage collection + gc.collect() + + # Use context manager to ensure thread pool is properly closed + executor = None + try: + executor = concurrent.futures.ThreadPoolExecutor( + max_workers=1, thread_name_prefix="deepcode_async" + ) + future = executor.submit(run_in_thread) + result = future.result(timeout=300) # 5 minute timeout + return result + except concurrent.futures.TimeoutError: + st.error( + "Processing timeout after 5 minutes. Please try again with a smaller file." + ) + raise TimeoutError("Processing timeout") + except Exception as e: + st.error(f"Async processing error: {e}") + raise e + finally: + # Ensure thread pool is properly closed + if executor: + try: + executor.shutdown(wait=True, cancel_futures=True) + except Exception: + pass + # Force garbage collection + gc.collect() + else: + # Run directly in current loop + return loop.run_until_complete(coro) + except Exception: + # Final fallback method: create new event loop + loop = None + try: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + result = loop.run_until_complete(coro) + return result + except Exception as backup_error: + st.error(f"All async methods failed: {backup_error}") + raise backup_error + finally: + if loop: + try: + loop.close() + except Exception: + pass + asyncio.set_event_loop(None) + # Force garbage collection + import gc + + gc.collect() + + +def handle_processing_workflow( + input_source: str, input_type: str, enable_indexing: bool = True +) -> Dict[str, Any]: + """ + Main processing function for workflow + + Args: + input_source: Input source + input_type: Input type + enable_indexing: Whether to enable indexing functionality + + Returns: + Processing result + """ + from .components import ( + enhanced_progress_display_component, + update_step_indicator, + display_status, + ) + + # Display enhanced progress components + chat_mode = input_type == "chat" + progress_bar, status_text, step_indicators, workflow_steps = ( + enhanced_progress_display_component(enable_indexing, chat_mode) + ) + log_sidebar_event( + "SYSTEM", + f"Workflow started ({'guided/chat' if chat_mode else 'research'} mode)", + extra={"input_type": input_type, "indexing": enable_indexing}, + ) + + # Step mapping: map progress percentages to step indices - adjust based on mode and indexing toggle + if chat_mode: + # Chat mode step mapping: Initialize -> Planning -> Setup -> Save Plan -> Implement + step_mapping = { + 5: 0, # Initialize + 30: 1, # Planning (analyzing requirements) + 50: 2, # Setup (creating workspace) + 70: 3, # Save Plan (saving implementation plan) + 85: 4, # Implement (generating code) + 100: 4, # Complete + } + elif not enable_indexing: + # Skip indexing-related steps progress mapping - fast mode order: Initialize -> Analyze -> Download -> Plan -> Implement + step_mapping = { + 5: 0, # Initialize + 10: 1, # Analyze + 25: 2, # Download + 40: 3, # Plan (now prioritized over References, 40%) + 85: 4, # Implement (skip References, Repos and Index) + 100: 4, # Complete + } + else: + # Full workflow step mapping - new order: Initialize -> Analyze -> Download -> Plan -> References -> Repos -> Index -> Implement + step_mapping = { + 5: 0, # Initialize + 10: 1, # Analyze + 25: 2, # Download + 40: 3, # Plan (now 4th position, 40%) + 50: 4, # References (now 5th position, conditional, 50%) + 60: 5, # Repos (GitHub download) + 70: 6, # Index (code indexing) + 85: 7, # Implement (code implementation) + 100: 7, # Complete + } + + current_step = 0 + + # Define enhanced progress callback function + def update_progress(progress: int, message: str): + nonlocal current_step + + # Update progress bar + progress_bar.progress(progress) + status_text.markdown(f"**{message}**") + + # Determine current step + new_step = step_mapping.get(progress, current_step) + if new_step != current_step: + current_step = new_step + update_step_indicator( + step_indicators, workflow_steps, current_step, "active" + ) + + stage_index = ( + min(current_step, len(workflow_steps) - 1) if workflow_steps else 0 + ) + stage_label = ( + workflow_steps[stage_index]["title"] if workflow_steps else "STAGE" + ) + log_sidebar_event( + stage_label, + message, + extra={"progress": progress}, + ) + time.sleep(0.3) # Brief pause for users to see progress changes + + # Step 1: Initialization + if chat_mode: + update_progress(5, "๐Ÿš€ Initializing chat-based planning engine...") + elif enable_indexing: + update_progress(5, "๐Ÿš€ Initializing AI research engine and loading models...") + else: + update_progress( + 5, "๐Ÿš€ Initializing AI research engine (Fast mode - indexing disabled)..." + ) + update_step_indicator(step_indicators, workflow_steps, 0, "active") + + # Start async processing with progress callback + with st.spinner("๐Ÿ”„ Processing workflow stages..."): + try: + # First try using simple async processing method + result = run_async_task_simple( + process_input_async( + input_source, input_type, enable_indexing, update_progress + ) + ) + except Exception as e: + st.warning(f"Primary async method failed: {e}") + # Fallback method: use original thread pool method + try: + result = run_async_task( + process_input_async( + input_source, input_type, enable_indexing, update_progress + ) + ) + except Exception as backup_error: + st.error(f"Both async methods failed. Error: {backup_error}") + return { + "status": "error", + "error": str(backup_error), + "traceback": traceback.format_exc(), + } + + # Update final status based on results + if result["status"] == "success": + # Complete all steps + update_progress(100, "โœ… All processing stages completed successfully!") + update_step_indicator( + step_indicators, workflow_steps, len(workflow_steps), "completed" + ) + + # Display success information + st.balloons() # Add celebration animation + if chat_mode: + display_status( + "๐ŸŽ‰ Chat workflow completed! Your requirements have been analyzed and code has been generated.", + "success", + ) + elif enable_indexing: + display_status( + "๐ŸŽ‰ Workflow completed! Your research paper has been successfully processed and code has been generated.", + "success", + ) + else: + display_status( + "๐ŸŽ‰ Fast workflow completed! Your research paper has been processed (indexing skipped for faster processing).", + "success", + ) + log_sidebar_event( + "COMPLETE", + "All stages completed successfully.", + level="success", + extra={ + "input_type": input_type, + "indexing": enable_indexing, + "timestamp": datetime.utcnow().isoformat(), + }, + ) + + else: + # Processing failed + update_progress(0, "โŒ Processing failed - see error details below") + update_step_indicator(step_indicators, workflow_steps, current_step, "error") + display_status( + f"โŒ Processing encountered an error: {result.get('error', 'Unknown error')}", + "error", + ) + failure_stage = ( + workflow_steps[current_step]["title"] + if workflow_steps and current_step < len(workflow_steps) + else "ERROR" + ) + log_sidebar_event( + failure_stage, + f"Processing failed: {result.get('error', 'Unknown error')}", + level="error", + ) + + # Wait a moment for users to see completion status + time.sleep(2.5) + + return result + + +def update_session_state_with_result(result: Dict[str, Any], input_type: str): + """ + Update session state with result + + Args: + result: Processing result + input_type: Input type + """ + if result["status"] == "success": + # Save result to session state + st.session_state.last_result = result + st.session_state.show_results = True + + # Save to history + st.session_state.results.append( + { + "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "input_type": input_type, + "status": "success", + "result": result, + } + ) + else: + # Save error information to session state for display + st.session_state.last_error = result.get("error", "Unknown error") + + # Save error to history + st.session_state.results.append( + { + "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "input_type": input_type, + "status": "error", + "error": result.get("error", "Unknown error"), + } + ) + + # Limit history to maximum 50 records + if len(st.session_state.results) > 50: + st.session_state.results = st.session_state.results[-50:] + + +def cleanup_temp_file(input_source: str, input_type: str): + """ + Cleanup temporary file using cross-platform safe method. + + Args: + input_source: Input source + input_type: Input type + """ + if input_type == "file" and input_source: + try: + from utils.cross_platform_file_handler import get_file_handler + + file_handler = get_file_handler() + file_handler.safe_remove_file(input_source) + except Exception as e: + # Log but don't fail - cleanup is best effort + import logging + + logging.getLogger(__name__).warning( + f"Failed to cleanup temp file {input_source}: {e}" + ) + + +async def handle_requirement_analysis_workflow( + user_input: str, analysis_mode: str, user_answers: Dict[str, str] = None +) -> Dict[str, Any]: + """ + Handle requirement analysis workflow + + Args: + user_input: User initial requirements + analysis_mode: Analysis mode ("generate_questions" or "summarize_requirements") + user_answers: User answer dictionary + + Returns: + Processing result dictionary + """ + try: + # Import required modules + from workflows.agent_orchestration_engine import ( + execute_requirement_analysis_workflow, + ) + + # Create progress callback function + def update_progress(progress: int, message: str): + # Display progress in Streamlit + st.session_state.current_progress = progress + st.session_state.current_message = message + + # Execute requirement analysis workflow + result = await execute_requirement_analysis_workflow( + user_input=user_input, + analysis_mode=analysis_mode, + user_answers=user_answers, + logger=None, # Can pass in logger + progress_callback=update_progress, + ) + + return result + + except Exception as e: + return { + "status": "error", + "error": str(e), + "message": f"Requirement analysis workflow execution failed: {str(e)}", + } + + +async def handle_requirement_modification_workflow( + current_requirements: str, modification_feedback: str +) -> Dict[str, Any]: + """ + Handle requirement modification workflow + + Args: + current_requirements: Current requirement document content + modification_feedback: User's modification requests and feedback + + Returns: + Processing result dictionary + """ + try: + # Import required modules + from workflows.agents.requirement_analysis_agent import RequirementAnalysisAgent + + # Create progress callback function + def update_progress(progress: int, message: str): + # Display progress in Streamlit + st.session_state.current_progress = progress + st.session_state.current_message = message + + update_progress(10, "๐Ÿ”ง Initializing requirement modification agent...") + + # Initialize RequirementAnalysisAgent + agent = RequirementAnalysisAgent() + + # Initialize agent (LLM is initialized internally) + await agent.initialize() + + update_progress(50, "โœ๏ธ Modifying requirements based on your feedback...") + + # Modify requirements + result = await agent.modify_requirements( + current_requirements=current_requirements, + modification_feedback=modification_feedback, + ) + + # Cleanup + await agent.cleanup() + + update_progress(100, "โœ… Requirements modification completed!") + + return { + "status": "success", + "result": result, + "message": "Requirements modification completed successfully", + } + + except Exception as e: + return { + "status": "error", + "error": str(e), + "message": f"Requirements modification workflow execution failed: {str(e)}", + } + + +def handle_guided_mode_processing(): + """Handle asynchronous processing for guided mode""" + # Check if questions need to be generated + if st.session_state.get("questions_generating", False): + st.session_state.questions_generating = False + + # Asynchronously generate questions + initial_req = st.session_state.get("initial_requirement", "") + if initial_req: + try: + # Use asynchronous processing to generate questions + result = run_async_task_simple( + handle_requirement_analysis_workflow( + user_input=initial_req, analysis_mode="generate_questions" + ) + ) + + if result["status"] == "success": + # Parse JSON result + import json + + questions = json.loads(result["result"]) + st.session_state.generated_questions = questions + else: + st.error( + f"Question generation failed: {result.get('error', 'Unknown error')}" + ) + + except Exception as e: + st.error(f"Question generation exception: {str(e)}") + + # Check if detailed requirements need to be generated + if st.session_state.get("requirements_generating", False): + st.session_state.requirements_generating = False + + # Asynchronously generate detailed requirements + initial_req = st.session_state.get("initial_requirement", "") + user_answers = st.session_state.get("user_answers", {}) + + if initial_req: + try: + # Use asynchronous processing to generate requirement summary + result = run_async_task_simple( + handle_requirement_analysis_workflow( + user_input=initial_req, + analysis_mode="summarize_requirements", + user_answers=user_answers, + ) + ) + + if result["status"] == "success": + st.session_state.detailed_requirements = result["result"] + else: + st.error( + f"Requirement summary generation failed: {result.get('error', 'Unknown error')}" + ) + + except Exception as e: + st.error(f"Requirement summary generation exception: {str(e)}") + + # Check if requirements need to be edited + if st.session_state.get("requirements_editing", False): + st.session_state.requirements_editing = False + st.info("๐Ÿ”ง Starting requirement modification process...") + + # Asynchronously modify requirements based on user feedback + current_requirements = st.session_state.get("detailed_requirements", "") + edit_feedback = st.session_state.get("edit_feedback", "") + + if current_requirements and edit_feedback: + try: + # Use asynchronous processing to modify requirements + result = run_async_task_simple( + handle_requirement_modification_workflow( + current_requirements=current_requirements, + modification_feedback=edit_feedback, + ) + ) + + if result["status"] == "success": + st.session_state.detailed_requirements = result["result"] + st.session_state.requirement_analysis_step = "summary" + st.session_state.edit_feedback = "" + st.success("โœ… Requirements updated successfully!") + st.rerun() + else: + st.error( + f"Requirements modification failed: {result.get('error', 'Unknown error')}" + ) + + except Exception as e: + st.error(f"Requirements modification exception: {str(e)}") + + +def _background_workflow_runner( + input_source: str, input_type: str, enable_indexing: bool, session_id: str +): + """ + Background thread function to run the workflow WITHOUT any Streamlit UI calls + This runs in a separate thread to avoid blocking Streamlit's main thread + """ + import logging + + # Store results in a thread-safe way using a simple dict + if not hasattr(_background_workflow_runner, "results"): + _background_workflow_runner.results = {} + + # Create a simple progress callback that only logs (no Streamlit UI calls) + def background_progress_callback(progress: int, message: str): + # Just log to Python logger, which will be captured by our logging handler + logging.info(f"Progress: {progress}% - {message}") + + try: + # Call the core async workflow directly without UI components + import asyncio + import nest_asyncio + + nest_asyncio.apply() + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + result = loop.run_until_complete( + process_input_async( + input_source, + input_type, + enable_indexing, + background_progress_callback, + ) + ) + _background_workflow_runner.results[session_id] = { + "status": "completed", + "result": result, + } + finally: + loop.close() + asyncio.set_event_loop(None) + + except Exception as e: + logging.error(f"Background workflow error: {e}", exc_info=True) + _background_workflow_runner.results[session_id] = { + "status": "error", + "error": str(e), + "traceback": traceback.format_exc(), + } + + +def handle_start_processing_button(input_source: str, input_type: str): + """ + Handle start processing button click - synchronous execution + + Args: + input_source: Input source + input_type: Input type + """ + from .components import display_status + + st.session_state.processing = True + st.session_state.workflow_start_time = time.time() + st.session_state.active_log_file = None + + # Get indexing toggle status + enable_indexing = st.session_state.get("enable_indexing", True) + log_sidebar_event( + "SYSTEM", + "Engaging DeepCode pipeline...", + extra={ + "input_type": input_type, + "indexing": enable_indexing, + }, + ) + + try: + # Process workflow synchronously + result = handle_processing_workflow(input_source, input_type, enable_indexing) + + # Display result status + if result["status"] == "success": + display_status("All operations completed successfully! ๐ŸŽ‰", "success") + else: + display_status("Error during processing", "error") + + # Update session state + update_session_state_with_result(result, input_type) + + except Exception as e: + # Handle exceptional cases + st.error(f"Unexpected error during processing: {e}") + result = {"status": "error", "error": str(e)} + update_session_state_with_result(result, input_type) + + finally: + # Reset state and clean up resources after processing + st.session_state.processing = False + + # Clean up temporary files + cleanup_temp_file(input_source, input_type) + + # Clean up system resources + cleanup_resources() + + # Rerun to display results or errors + st.rerun() + + +def check_background_workflow_status(): + """ + Check if background workflow has completed and handle results + This should be called on every Streamlit rerun + """ + from .components import display_status + + if not st.session_state.get("processing"): + return + + session_id = st.session_state.get("workflow_session_id") + if not session_id: + return + + # Check if background thread has finished + if ( + hasattr(_background_workflow_runner, "results") + and session_id in _background_workflow_runner.results + ): + workflow_result = _background_workflow_runner.results[session_id] + + # Clean up the result from the cache + del _background_workflow_runner.results[session_id] + + # Process the result + if workflow_result["status"] == "completed": + result = workflow_result["result"] + + # Display result status + if result["status"] == "success": + display_status("All operations completed successfully! ๐ŸŽ‰", "success") + else: + display_status("Error during processing", "error") + + # Update session state + update_session_state_with_result( + result, st.session_state.get("workflow_input_type", "") + ) + + elif workflow_result["status"] == "error": + st.error(f"Unexpected error during processing: {workflow_result['error']}") + result = {"status": "error", "error": workflow_result["error"]} + update_session_state_with_result( + result, st.session_state.get("workflow_input_type", "") + ) + + # Clean up + st.session_state.processing = False + cleanup_temp_file( + st.session_state.get("workflow_input_source"), + st.session_state.get("workflow_input_type"), + ) + cleanup_resources() + + # Clear workflow tracking variables + st.session_state.workflow_session_id = None + st.session_state.workflow_thread = None + st.session_state.workflow_input_source = None + st.session_state.workflow_input_type = None + + # Rerun to show results + st.rerun() + + +def handle_error_display(): + """Handle error display""" + if hasattr(st.session_state, "last_error") and st.session_state.last_error: + st.error(f"โŒ Error: {st.session_state.last_error}") + if st.button("๐Ÿ”„ Try Again", type="secondary", use_container_width=True): + st.session_state.last_error = None + st.session_state.task_counter += 1 + st.rerun() + + +def initialize_session_state(): + """Initialize session state""" + if "processing" not in st.session_state: + st.session_state.processing = False + if "results" not in st.session_state: + st.session_state.results = [] + if "current_step" not in st.session_state: + st.session_state.current_step = 0 + if "task_counter" not in st.session_state: + st.session_state.task_counter = 0 + if "show_results" not in st.session_state: + st.session_state.show_results = False + if "last_result" not in st.session_state: + st.session_state.last_result = None + if "last_error" not in st.session_state: + st.session_state.last_error = None + if "enable_indexing" not in st.session_state: + st.session_state.enable_indexing = ( + False # Default enable indexing functionality + ) + + # Requirement analysis related states + if "requirement_analysis_mode" not in st.session_state: + st.session_state.requirement_analysis_mode = "direct" # direct/guided + if "requirement_analysis_step" not in st.session_state: + st.session_state.requirement_analysis_step = "input" # input/questions/summary + if "generated_questions" not in st.session_state: + st.session_state.generated_questions = [] + if "user_answers" not in st.session_state: + st.session_state.user_answers = {} + if "detailed_requirements" not in st.session_state: + st.session_state.detailed_requirements = "" + if "initial_requirement" not in st.session_state: + st.session_state.initial_requirement = "" + if "questions_generating" not in st.session_state: + st.session_state.questions_generating = False + if "requirements_generating" not in st.session_state: + st.session_state.requirements_generating = False + if "requirements_confirmed" not in st.session_state: + st.session_state.requirements_confirmed = False + if "edit_feedback" not in st.session_state: + st.session_state.edit_feedback = "" + if "requirements_editing" not in st.session_state: + st.session_state.requirements_editing = False + if "guided_initial_requirement" not in st.session_state: + st.session_state.guided_initial_requirement = "" + if "guided_edit_feedback" not in st.session_state: + st.session_state.guided_edit_feedback = "" + if "confirmed_requirement_text" not in st.session_state: + st.session_state.confirmed_requirement_text = None + if "sidebar_events" not in st.session_state: + st.session_state.sidebar_events = [] + ensure_sidebar_logging() + if "workflow_start_time" not in st.session_state: + st.session_state.workflow_start_time = None + if "active_log_file" not in st.session_state: + st.session_state.active_log_file = None + if "workflow_session_id" not in st.session_state: + st.session_state.workflow_session_id = None + if "workflow_thread" not in st.session_state: + st.session_state.workflow_thread = None + if "workflow_input_source" not in st.session_state: + st.session_state.workflow_input_source = None + if "workflow_input_type" not in st.session_state: + st.session_state.workflow_input_type = None + if "guided_payload" not in st.session_state: + st.session_state.guided_payload = None + + +def cleanup_resources(): + """ + Clean up system resources to prevent memory leaks + """ + try: + import gc + import threading + import multiprocessing + import asyncio + import sys + + # 1. Clean up asyncio-related resources + try: + # Get current event loop (if exists) + try: + loop = asyncio.get_running_loop() + # Cancel all pending tasks + if loop and not loop.is_closed(): + pending_tasks = [ + task for task in asyncio.all_tasks(loop) if not task.done() + ] + if pending_tasks: + for task in pending_tasks: + if not task.cancelled(): + task.cancel() + # Wait for task cancellation to complete + try: + if pending_tasks: + # Use timeout to avoid blocking too long + import time + + time.sleep(0.1) + except Exception: + pass + except RuntimeError: + # No running event loop, continue with other cleanup + pass + except Exception: + pass + + # 2. Force garbage collection + gc.collect() + + # 3. Clean up active threads (except main thread) + active_threads = threading.active_count() + if active_threads > 1: + # Wait some time for threads to naturally finish + import time + + time.sleep(0.5) + + # 4. Clean up multiprocessing resources + try: + # Clean up possible multiprocessing resources + if hasattr(multiprocessing, "active_children"): + for child in multiprocessing.active_children(): + if child.is_alive(): + child.terminate() + child.join(timeout=1.0) + # If join times out, force kill + if child.is_alive(): + try: + child.kill() + child.join(timeout=0.5) + except Exception: + pass + + # Clean up multiprocessing-related resource tracker + try: + import multiprocessing.resource_tracker + + if hasattr(multiprocessing.resource_tracker, "_resource_tracker"): + tracker = multiprocessing.resource_tracker._resource_tracker + if tracker and hasattr(tracker, "_stop"): + tracker._stop() + except Exception: + pass + + except Exception: + pass + + # 5. Force clean up Python internal caches + try: + # Clean up some temporary objects in module cache + import sys + + # Don't delete key modules, only clean up possible temporary resources + if hasattr(sys, "_clear_type_cache"): + sys._clear_type_cache() + except Exception: + pass + + # 6. Final garbage collection + gc.collect() + + except Exception as e: + # Silently handle cleanup errors to avoid affecting main flow + # But can log errors in debug mode + try: + import logging + + logging.getLogger(__name__).debug(f"Resource cleanup warning: {e}") + except Exception: + pass diff --git a/DeepCode/ui/layout.py b/DeepCode/ui/layout.py new file mode 100644 index 00000000..8b3b31c0 --- /dev/null +++ b/DeepCode/ui/layout.py @@ -0,0 +1,142 @@ +""" +DeepCode Layout Manager +Organizes the visual structure using the Cyber components. +""" + +from typing import Optional + +import streamlit as st +from .components import ( + display_features, + display_header, + footer_component, + guided_requirement_workflow, + input_method_selector, + requirement_mode_selector, + results_display_component, + sidebar_control_panel, + system_status_component, +) +from .styles import get_main_styles +from .handlers import ( + initialize_session_state, + handle_start_processing_button, + handle_error_display, + handle_guided_mode_processing, +) + + +def setup_page_config(): + st.set_page_config( + page_title="DeepCode", + page_icon="assets/logo.png", + layout="wide", + initial_sidebar_state="expanded", + menu_items={ + "Get Help": "https://github.com/deepcode", + "About": "DeepCode AI Research Engine v3.0", + }, + ) + + +def main_layout(): + """Main layout execution""" + # Initialize Core + initialize_session_state() + setup_page_config() + + # Inject Cyber Styles + st.markdown(get_main_styles(), unsafe_allow_html=True) + + # Render Sidebar + sidebar_control_panel() + + # Main Content Area + display_header() + + # Determine Content State + show_results = st.session_state.get("show_results", False) + last_result = st.session_state.get("last_result", None) + + if show_results and last_result: + results_display_component(last_result, st.session_state.task_counter) + else: + # Landing State + display_features() + system_status_component() + + st.markdown('
', unsafe_allow_html=True) + + # Input Interface + render_input_area() + + # Global Error Handler (Always active) + handle_error_display() + + # Footer + footer_component() + + return {} + + +def render_input_area(): + """Handles the logic for which input to show""" + + # Handle guided mode async processing (background) + handle_guided_mode_processing() + + mode = requirement_mode_selector() + is_guided = mode == "guided" + processing = st.session_state.get("processing", False) + requirements_confirmed = st.session_state.get("requirements_confirmed", False) + + input_source: Optional[str] = None + input_type: Optional[str] = None + + with st.container(): + if is_guided: + input_source, _ = guided_requirement_workflow() + input_type = "chat" if input_source else None + else: + input_source, input_type = input_method_selector( + st.session_state.task_counter + ) + + st.markdown('
', unsafe_allow_html=True) + + if is_guided and requirements_confirmed and input_source and not processing: + payload = input_source + st.session_state.requirements_confirmed = False + st.session_state.confirmed_requirement_text = None + handle_start_processing_button(payload, input_type or "chat") + + elif input_source and not processing: + col1, col2, col3 = st.columns([1, 2, 1]) + with col2: + if st.button( + "START CODING ๐Ÿš€", type="primary", use_container_width=True + ): + if is_guided: + st.session_state.confirmed_requirement_text = None + handle_start_processing_button(input_source, input_type or "chat") + + elif processing: + st.markdown( + """ +
+
+ NEURAL PROCESSING ACTIVE... +
+ """, + unsafe_allow_html=True, + ) + + elif not input_source and not is_guided: + st.markdown( + """ +
+ AWAITING INPUT SIGNAL... +
+ """, + unsafe_allow_html=True, + ) diff --git a/DeepCode/ui/sidebar_feed.py b/DeepCode/ui/sidebar_feed.py new file mode 100644 index 00000000..9649eeb4 --- /dev/null +++ b/DeepCode/ui/sidebar_feed.py @@ -0,0 +1,91 @@ +""" +Sidebar mission feed utilities. +""" + +from __future__ import annotations + +import logging +from datetime import datetime +from typing import Optional, Dict, Any + +import streamlit as st + + +def _init_event_store(): + if "sidebar_events" not in st.session_state: + st.session_state.sidebar_events = [] + + +def log_sidebar_event( + stage: str, + message: str, + level: str = "info", + extra: Optional[Dict[str, Any]] = None, +): + """ + Record a sidebar feed event for live mission status display. + Thread-safe: if called from background thread, just log to Python logger instead. + """ + try: + # Check if we're in a Streamlit context + from streamlit.runtime.scriptrunner import get_script_run_ctx + + if get_script_run_ctx() is None: + # Running in background thread, just use Python logging + import logging + + logging.info(f"[{stage}] {message}") + return + + _init_event_store() + events = list(st.session_state.sidebar_events) + events.append( + { + "timestamp": datetime.utcnow().strftime("%H:%M:%S"), + "stage": stage.upper(), + "message": message, + "level": level, + "extra": extra or {}, + } + ) + st.session_state.sidebar_events = events[-80:] + except Exception: + # Fallback to Python logging + import logging + + logging.info(f"[{stage}] {message}") + + +class SidebarLogHandler(logging.Handler): + """Forward Python logging records to the sidebar mission feed.""" + + def emit(self, record: logging.LogRecord): + try: + msg = self.format(record) + stage = getattr(record, "stage", record.name.split(".")[-1]).upper() + level = record.levelname.lower() + payload = { + "logger": record.name, + "level": record.levelname, + } + if record.exc_info: + payload["exception"] = self.formatException(record.exc_info) + log_sidebar_event(stage, msg, level=level, extra=payload) + except Exception: + pass + + +def ensure_sidebar_logging(): + """ + Attach sidebar logging handler once per session to bridge backend logs. + """ + if st.session_state.get("_sidebar_logging_attached"): + return + + handler = SidebarLogHandler() + handler.setLevel(logging.INFO) + formatter = logging.Formatter("%(message)s") + handler.setFormatter(formatter) + + logging.getLogger().addHandler(handler) + st.session_state._sidebar_logging_attached = True diff --git a/DeepCode/ui/streamlit_app.py b/DeepCode/ui/streamlit_app.py new file mode 100644 index 00000000..9915e477 --- /dev/null +++ b/DeepCode/ui/streamlit_app.py @@ -0,0 +1,38 @@ +""" +DeepCode - AI Research Engine + +Streamlit Web Interface Main Application File +""" + +import os +import sys + +# Disable .pyc file generation +os.environ["PYTHONDONTWRITEBYTECODE"] = "1" + +# Add parent directory to path for module imports +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +if parent_dir not in sys.path: + sys.path.insert(0, parent_dir) + +# Import UI modules +from ui.layout import main_layout + + +def main(): + """ + Main function - Streamlit application entry + + All UI logic has been modularized into ui/ folder + """ + # Run main layout + sidebar_info = main_layout() + + # Additional global logic can be added here if needed + + return sidebar_info + + +if __name__ == "__main__": + main() diff --git a/DeepCode/ui/styles.py b/DeepCode/ui/styles.py new file mode 100644 index 00000000..eee0d7f3 --- /dev/null +++ b/DeepCode/ui/styles.py @@ -0,0 +1,356 @@ +""" +DeepCode UI Styles - Cyber/AI Tech Theme +Modernized with Glassmorphism, Neon Accents, and Fluid Typography. +""" + + +def get_main_styles() -> str: + return """ + + """ diff --git a/DeepCode/utils/__init__.py b/DeepCode/utils/__init__.py new file mode 100644 index 00000000..fca68f58 --- /dev/null +++ b/DeepCode/utils/__init__.py @@ -0,0 +1,17 @@ +""" +Utils package for paper processing tools. +""" + +from .file_processor import FileProcessor +from .dialogue_logger import ( + DialogueLogger, + create_dialogue_logger, + extract_paper_id_from_path, +) + +__all__ = [ + "FileProcessor", + "DialogueLogger", + "create_dialogue_logger", + "extract_paper_id_from_path", +] diff --git a/DeepCode/utils/cli_interface.py b/DeepCode/utils/cli_interface.py new file mode 100644 index 00000000..29fbdb1f --- /dev/null +++ b/DeepCode/utils/cli_interface.py @@ -0,0 +1,459 @@ +#!/usr/bin/env python3 +""" +Professional CLI Interface Module +ไธ“ไธšCLI็•Œ้ขๆจกๅ— - ๅŒ…ๅซlogoใ€้ขœ่‰ฒๅฎšไน‰ๅ’Œ็•Œ้ข็ป„ไปถ +""" + +import os +import time +import platform +from pathlib import Path +from typing import Optional +import tkinter as tk +from tkinter import filedialog + + +class Colors: + """ANSI color codes for terminal styling""" + + HEADER = "\033[95m" + OKBLUE = "\033[94m" + OKCYAN = "\033[96m" + OKGREEN = "\033[92m" + WARNING = "\033[93m" + FAIL = "\033[91m" + ENDC = "\033[0m" + BOLD = "\033[1m" + UNDERLINE = "\033[4m" + + # Gradient colors + PURPLE = "\033[35m" + MAGENTA = "\033[95m" + BLUE = "\033[34m" + CYAN = "\033[36m" + GREEN = "\033[32m" + YELLOW = "\033[33m" + + +class CLIInterface: + """Professional CLI interface with modern styling""" + + def __init__(self): + self.uploaded_file = None + self.is_running = True + + # Check tkinter availability + self.tkinter_available = True + try: + import tkinter as tk + + # Test if tkinter can create a window (some systems have tkinter but no display) + test_root = tk.Tk() + test_root.withdraw() + test_root.destroy() + except Exception: + self.tkinter_available = False + + def clear_screen(self): + """Clear terminal screen""" + os.system("cls" if os.name == "nt" else "clear") + + def print_logo(self): + """Print a beautiful ASCII logo with gradient colors and tech elements""" + # ็กฎไฟๆฏ่กŒๆ€ปๅ…ฑ79ไธชๅญ—็ฌฆ๏ผˆไธๅŒ…ๆ‹ฌ้ขœ่‰ฒไปฃ็ ๏ผ‰๏ผŒ่พนๆก†ๅฎŒ็พŽๅฏน้ฝ + logo = f""" +{Colors.CYAN}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•— +โ•‘ โ•‘ +โ•‘ {Colors.BOLD}{Colors.MAGENTA}โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—{Colors.CYAN} โ•‘ +โ•‘ {Colors.BOLD}{Colors.PURPLE}โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•”โ•โ•โ•โ•โ•โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•”โ•โ•โ•โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘{Colors.CYAN} โ•‘ +โ•‘ {Colors.BOLD}{Colors.BLUE}โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ• โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘{Colors.CYAN} โ•‘ +โ•‘ {Colors.BOLD}{Colors.OKBLUE}โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•”โ•โ•โ• โ–ˆโ–ˆโ•”โ•โ•โ•โ• โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘{Colors.CYAN} โ•‘ +โ•‘ {Colors.BOLD}{Colors.OKCYAN}โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ• โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘{Colors.CYAN} โ•‘ +โ•‘ {Colors.BOLD}{Colors.GREEN}โ•šโ•โ• โ•šโ•โ• โ•šโ•โ•โ•โ•โ•โ•โ•โ•šโ•โ• โ•šโ•โ• โ•šโ•โ• โ•šโ•โ•โ•โ•โ•โ• โ•šโ•โ• โ•šโ•โ•โ•šโ•โ•{Colors.CYAN} โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.BOLD}{Colors.YELLOW}โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”{Colors.CYAN} โ•‘ +โ•‘ {Colors.BOLD}{Colors.YELLOW}โ”‚ ๐Ÿค– AI-POWERED RESEARCH PAPER REPRODUCTION ENGINE ๐Ÿš€ โ”‚{Colors.CYAN} โ•‘ +โ•‘ {Colors.BOLD}{Colors.YELLOW}โ”‚ โšก INTELLIGENT โ€ข AUTOMATED โ€ข CUTTING-EDGE โšก โ”‚{Colors.CYAN} โ•‘ +โ•‘ {Colors.BOLD}{Colors.YELLOW}โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜{Colors.CYAN} โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.BOLD}{Colors.GREEN}๐Ÿ’Ž CORE CAPABILITIES:{Colors.ENDC} {Colors.CYAN}โ•‘ +โ•‘ {Colors.BOLD}{Colors.OKCYAN}โ–ถ Neural PDF Analysis & Code Extraction {Colors.CYAN}โ•‘ +โ•‘ {Colors.BOLD}{Colors.OKCYAN}โ–ถ Advanced Document Processing Engine {Colors.CYAN}โ•‘ +โ•‘ {Colors.BOLD}{Colors.OKCYAN}โ–ถ Multi-Format Support (PDFโ€ขDOCXโ€ขPPTXโ€ขHTML) {Colors.CYAN}โ•‘ +โ•‘ {Colors.BOLD}{Colors.OKCYAN}โ–ถ Smart File Upload Interface {Colors.CYAN}โ•‘ +โ•‘ {Colors.BOLD}{Colors.OKCYAN}โ–ถ Automated Repository Management {Colors.CYAN}โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.BOLD}{Colors.PURPLE}๐Ÿ”ฌ TECH STACK: Pythonโ€ขAIโ€ขMCPโ€ขDoclingโ€ขLLM {Colors.CYAN}โ•‘ +โ•‘ โ•‘ +โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{Colors.ENDC} +""" + print(logo) + + def print_welcome_banner(self): + """Print welcome banner with version info""" + banner = f""" +{Colors.BOLD}{Colors.CYAN}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•— +โ•‘ WELCOME TO ReproAI โ•‘ +โ• โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ฃ +โ•‘ โ•‘ +โ•‘ {Colors.YELLOW}Version: 2.0.0 | Build: Professional Edition {Colors.CYAN}โ•‘ +โ•‘ {Colors.GREEN}Status: Ready | Engine: Initialized {Colors.CYAN}โ•‘ +โ•‘ {Colors.PURPLE}Author: AI Research Team | License: MIT {Colors.CYAN}โ•‘ +โ•‘ โ•‘ +โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{Colors.ENDC} +""" + print(banner) + + def print_separator(self, char="โ•", length=79, color=Colors.CYAN): + """Print a styled separator line""" + print(f"{color}{char * length}{Colors.ENDC}") + + def print_status(self, message: str, status_type: str = "info"): + """Print status message with appropriate styling""" + status_styles = { + "success": f"{Colors.OKGREEN}โœ…", + "error": f"{Colors.FAIL}โŒ", + "warning": f"{Colors.WARNING}โš ๏ธ ", + "info": f"{Colors.OKBLUE}โ„น๏ธ ", + "processing": f"{Colors.YELLOW}โณ", + "upload": f"{Colors.PURPLE}๐Ÿ“", + "download": f"{Colors.CYAN}๐Ÿ“ฅ", + "analysis": f"{Colors.MAGENTA}๐Ÿ”", + } + + icon = status_styles.get(status_type, status_styles["info"]) + print(f"{icon} {Colors.BOLD}{message}{Colors.ENDC}") + + def create_menu(self): + """Create an interactive menu""" + menu = f""" +{Colors.BOLD}{Colors.CYAN}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•— +โ•‘ MAIN MENU โ•‘ +โ• โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ฃ +โ•‘ โ•‘ +โ•‘ {Colors.OKGREEN}๐ŸŒ [U] Process URL {Colors.CYAN}โ”‚ {Colors.PURPLE}๐Ÿ“ [F] Upload File {Colors.CYAN}โ”‚ {Colors.FAIL}โŒ [Q] Quit{Colors.CYAN} โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.YELLOW}๐Ÿ“ Enter a research paper URL (arXiv, IEEE, ACM, etc.) {Colors.CYAN}โ•‘ +โ•‘ {Colors.YELLOW} or upload a PDF/DOC file for intelligent analysis {Colors.CYAN}โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.OKCYAN}๐Ÿ’ก Tip: Press 'F' to open file browser or 'U' to enter URL manually {Colors.CYAN}โ•‘ +โ•‘ โ•‘ +โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{Colors.ENDC} +""" + print(menu) + + def get_user_input(self): + """Get user input with styled prompt""" + print(f"\n{Colors.BOLD}{Colors.OKCYAN}โžค Your choice: {Colors.ENDC}", end="") + return input().strip().lower() + + def upload_file_gui(self) -> Optional[str]: + """Modern file upload interface using tkinter with cross-platform compatibility""" + # Check if tkinter is available + if not self.tkinter_available: + self.print_status("GUI file dialog not available on this system", "warning") + self.print_status("Using manual file path input instead", "info") + return self._get_manual_file_path() + + def select_file(): + try: + # Create a hidden root window + root = tk.Tk() + root.withdraw() # Hide the main window + + # Platform-specific configurations + system = platform.system() + + if system == "Darwin": # macOS + # macOS specific settings + try: + root.call("wm", "attributes", ".", "-topmost", True) + except Exception: + pass + + # macOS compatible file types + file_types = [ + ("PDF Files", ".pdf"), + ("Word Documents", ".docx .doc"), + ("PowerPoint Files", ".pptx .ppt"), + ("HTML Files", ".html .htm"), + ("Text Files", ".txt .md"), + ("All Files", ".*"), + ] + else: + # Windows and Linux + root.attributes("-topmost", True) + + # Windows/Linux compatible file types + file_types = [ + ("PDF Files", "*.pdf"), + ("Word Documents", "*.docx;*.doc"), + ("PowerPoint Files", "*.pptx;*.ppt"), + ("HTML Files", "*.html;*.htm"), + ("Text Files", "*.txt;*.md"), + ("All Files", "*.*"), + ] + + # Set window title + root.title("Repro-AI - File Selector") + + try: + # Open file dialog with platform-appropriate settings + file_path = filedialog.askopenfilename( + title="Select Research Paper File", + filetypes=file_types, + initialdir=os.getcwd(), + ) + except Exception as e: + self.print_status(f"File dialog error: {str(e)}", "error") + return None + finally: + # Clean up + try: + root.destroy() + except Exception: + pass + + return file_path + + except Exception as e: + # Fallback: destroy root if it exists + try: + if "root" in locals(): + root.destroy() + except Exception: + pass + + # Print error and suggest alternative + self.print_status(f"GUI file dialog failed: {str(e)}", "error") + self.print_status( + "Please use manual file path input instead", "warning" + ) + return self._get_manual_file_path() + + self.print_status("Opening file browser dialog...", "upload") + file_path = select_file() + + if file_path: + # Validate file + if not os.path.exists(file_path): + self.print_status("File not found!", "error") + return None + + file_size = os.path.getsize(file_path) / (1024 * 1024) # Size in MB + file_ext = Path(file_path).suffix.lower() + + # Display file info with beautiful formatting + file_name = Path(file_path).name + directory = str(Path(file_path).parent) + + # Truncate long paths for display + if len(file_name) > 50: + display_name = file_name[:47] + "..." + else: + display_name = file_name + + if len(directory) > 49: + display_dir = "..." + directory[-46:] + else: + display_dir = directory + + print(f""" +{Colors.OKGREEN}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•— +โ•‘ FILE SELECTED โ•‘ +โ• โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ฃ +โ•‘ โ•‘ +โ•‘ {Colors.BOLD}๐Ÿ“„ File Name:{Colors.ENDC} {Colors.CYAN}{display_name:<50}{Colors.OKGREEN}โ•‘ +โ•‘ {Colors.BOLD}๐Ÿ“ Directory:{Colors.ENDC} {Colors.YELLOW}{display_dir:<49}{Colors.OKGREEN}โ•‘ +โ•‘ {Colors.BOLD}๐Ÿ“Š File Size:{Colors.ENDC} {Colors.PURPLE}{file_size:.2f} MB{Colors.OKGREEN} โ•‘ +โ•‘ {Colors.BOLD}๐Ÿ”– File Type:{Colors.ENDC} {Colors.MAGENTA}{file_ext.upper():<50}{Colors.OKGREEN}โ•‘ +โ•‘ โ•‘ +โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{Colors.ENDC} +""") + + self.print_status(f"File successfully selected: {file_name}", "success") + return file_path + else: + self.print_status("No file selected", "warning") + return None + + def _get_manual_file_path(self) -> Optional[str]: + """Fallback method for manual file path input when GUI fails""" + print( + f"\n{Colors.BOLD}{Colors.CYAN}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—" + ) + print( + "โ•‘ MANUAL FILE INPUT โ•‘" + ) + print( + f"โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{Colors.ENDC}" + ) + + print(f"\n{Colors.YELLOW}๐Ÿ“ Supported file types:{Colors.ENDC}") + print(f" {Colors.CYAN}โ€ข PDF files (.pdf)") + print(f" {Colors.CYAN}โ€ข Word documents (.docx, .doc)") + print(f" {Colors.CYAN}โ€ข PowerPoint files (.pptx, .ppt)") + print(f" {Colors.CYAN}โ€ข HTML files (.html, .htm)") + print(f" {Colors.CYAN}โ€ข Text files (.txt, .md){Colors.ENDC}") + + print( + f"\n{Colors.BOLD}{Colors.OKCYAN}๐Ÿ“ Enter file path (or drag & drop): {Colors.ENDC}", + end="", + ) + file_path = input().strip() + + # Clean up the path (remove quotes if present) + file_path = file_path.strip("\"'") + + if file_path: + # Expand user directory if needed + file_path = os.path.expanduser(file_path) + + # Check if file exists + if os.path.exists(file_path): + self.print_status( + f"File found: {os.path.basename(file_path)}", "success" + ) + return file_path + else: + self.print_status("File not found at the specified path", "error") + return None + else: + self.print_status("No file path provided", "warning") + return None + + def get_url_input(self) -> str: + """Get URL input with validation and examples""" + print( + f"\n{Colors.BOLD}{Colors.CYAN}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—" + ) + print( + "โ•‘ URL INPUT โ•‘" + ) + print( + f"โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{Colors.ENDC}" + ) + + print(f"\n{Colors.YELLOW}๐Ÿ“ Supported URL Examples:{Colors.ENDC}") + print(f" {Colors.CYAN}โ€ข arXiv: https://arxiv.org/pdf/2403.00813") + print(f" {Colors.CYAN}โ€ข arXiv: @https://arxiv.org/pdf/2403.00813") + print(f" {Colors.CYAN}โ€ข IEEE: https://ieeexplore.ieee.org/document/...") + print(f" {Colors.CYAN}โ€ข ACM: https://dl.acm.org/doi/...") + print( + f" {Colors.CYAN}โ€ข Direct PDF: https://example.com/paper.pdf{Colors.ENDC}" + ) + + print( + f"\n{Colors.BOLD}{Colors.OKCYAN}๐ŸŒ Enter paper URL: {Colors.ENDC}", end="" + ) + url = input().strip() + + if url: + # Basic URL validation + if any( + domain in url.lower() + for domain in ["arxiv.org", "ieee", "acm.org", ".pdf", "researchgate"] + ): + self.print_status(f"URL received: {url}", "success") + return url + else: + self.print_status("URL appears valid, proceeding...", "info") + return url + else: + self.print_status("No URL provided", "warning") + return "" + + def show_progress_bar(self, message: str, duration: float = 2.0): + """Show a progress animation with enhanced styling""" + print(f"\n{Colors.YELLOW}{message}{Colors.ENDC}") + + # Progress bar animation with different styles + bar_length = 50 + for i in range(bar_length + 1): + percent = (i / bar_length) * 100 + filled = "โ–ˆ" * i + empty = "โ–‘" * (bar_length - i) + + # Color gradient effect + if percent < 33: + color = Colors.FAIL + elif percent < 66: + color = Colors.WARNING + else: + color = Colors.OKGREEN + + print( + f"\r{color}[{filled}{empty}] {percent:6.1f}%{Colors.ENDC}", + end="", + flush=True, + ) + time.sleep(duration / bar_length) + + print(f"\n{Colors.OKGREEN}โœ… {message} completed!{Colors.ENDC}\n") + + def show_spinner(self, message: str, duration: float = 1.0): + """Show a spinner animation""" + spinner_chars = "โ ‹โ ™โ นโ ธโ ผโ ดโ ฆโ งโ ‡โ " + end_time = time.time() + duration + + while time.time() < end_time: + for char in spinner_chars: + print( + f"\r{Colors.CYAN}{char} {Colors.BOLD}{message}{Colors.ENDC}", + end="", + flush=True, + ) + time.sleep(0.1) + if time.time() >= end_time: + break + + print(f"\r{Colors.OKGREEN}โœ… {Colors.BOLD}{message} - Done!{Colors.ENDC}") + + def print_results_header(self): + """Print results section header""" + header = f""" +{Colors.OKGREEN}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•— +โ•‘ PROCESSING RESULTS โ•‘ +โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{Colors.ENDC} +""" + print(header) + + def print_error_box(self, title: str, error_msg: str): + """Print error message in a styled box""" + print(f""" +{Colors.FAIL}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•— +โ•‘ ERROR โ•‘ +โ• โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ฃ +โ•‘ โ•‘ +โ•‘ {Colors.BOLD}Title: {title:<66}{Colors.FAIL}โ•‘ +โ•‘ {Colors.BOLD}Error: {error_msg:<66}{Colors.FAIL}โ•‘ +โ•‘ โ•‘ +โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{Colors.ENDC} +""") + + def print_goodbye(self): + """Print goodbye message""" + goodbye = f""" +{Colors.BOLD}{Colors.YELLOW}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•— +โ•‘ GOODBYE! โ•‘ +โ• โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ฃ +โ•‘ โ•‘ +โ•‘ {Colors.CYAN}Thank you for using ReproAI! {Colors.YELLOW}โ•‘ +โ•‘ {Colors.GREEN}๐ŸŒŸ Star us on GitHub: https://github.com/your-repo {Colors.YELLOW}โ•‘ +โ•‘ {Colors.PURPLE}๐Ÿ“ง Contact: support@reproai.com {Colors.YELLOW}โ•‘ +โ•‘ {Colors.MAGENTA}๐Ÿ› Report issues: https://github.com/your-repo/issues {Colors.YELLOW}โ•‘ +โ•‘ โ•‘ +โ•‘ {Colors.OKGREEN}โœจ Happy coding! See you next time! โœจ {Colors.YELLOW}โ•‘ +โ•‘ โ•‘ +โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{Colors.ENDC} +""" + print(goodbye) + + def ask_continue(self) -> bool: + """Ask user if they want to continue""" + print( + f"\n{Colors.BOLD}{Colors.CYAN}Press Enter to continue or 'q' to quit: {Colors.ENDC}", + end="", + ) + choice = input().strip().lower() + return choice not in ["q", "quit", "exit"] diff --git a/DeepCode/utils/cross_platform_file_handler.py b/DeepCode/utils/cross_platform_file_handler.py new file mode 100644 index 00000000..ac4703ce --- /dev/null +++ b/DeepCode/utils/cross_platform_file_handler.py @@ -0,0 +1,475 @@ +#!/usr/bin/env python3 +""" +Cross-Platform File Handler +่ทจๅนณๅฐๆ–‡ไปถๅค„็†ๆจกๅ— + +This module provides robust file handling utilities that work consistently +across Windows, Linux, and macOS, with proper error handling and cleanup. + +Key features: +- Safe temporary file creation with proper cleanup +- Cross-platform path handling +- Atomic file operations +- Comprehensive error handling and logging +""" + +import os +import shutil +import tempfile +import logging +import atexit +import platform +from pathlib import Path +from typing import Optional, Union +from contextlib import contextmanager + + +class CrossPlatformFileHandler: + """ + Robust cross-platform file handler with proper error handling. + + Handles common pitfalls in file operations across different operating systems: + - Windows file handle issues + - Path separator inconsistencies + - Permission problems + - Temporary file cleanup + """ + + def __init__(self, logger: Optional[logging.Logger] = None): + """ + Initialize the file handler. + + Args: + logger: Optional logger instance for tracking operations + """ + self.logger = logger or self._create_default_logger() + self.temp_files = [] # Track temporary files for cleanup + self.platform = platform.system() + + # Register cleanup handler + atexit.register(self.cleanup_all_temp_files) + + self.logger.info(f"CrossPlatformFileHandler initialized on {self.platform}") + + def _create_default_logger(self) -> logging.Logger: + """Create a default logger if none provided.""" + logger = logging.getLogger(__name__) + if not logger.handlers: + handler = logging.StreamHandler() + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + handler.setFormatter(formatter) + logger.addHandler(handler) + logger.setLevel(logging.INFO) + return logger + + @staticmethod + def normalize_path(path: Union[str, Path]) -> Path: + """ + Normalize a path to use proper separators for the current OS. + + Args: + path: Input path (string or Path object) + + Returns: + Normalized Path object + + Example: + >>> handler = CrossPlatformFileHandler() + >>> handler.normalize_path("data/files\\test.txt") + PosixPath('data/files/test.txt') # On Linux/Mac + WindowsPath('data\\files\\test.txt') # On Windows + """ + if isinstance(path, str): + # Replace all path separators with the OS-specific one + path = path.replace("\\", os.sep).replace("/", os.sep) + return Path(path).resolve() + return Path(path).resolve() + + def create_safe_temp_file( + self, + suffix: str = "", + prefix: str = "deepcode_", + content: Optional[bytes] = None, + ) -> Path: + """ + Create a temporary file with proper cross-platform handling. + + This method addresses Windows file handle issues by: + 1. Properly closing the file before returning + 2. Setting delete=False to prevent premature deletion + 3. Tracking the file for later cleanup + + Args: + suffix: File suffix (e.g., ".pdf", ".txt") + prefix: File prefix for identification + content: Optional content to write to the file + + Returns: + Path to the created temporary file + + Raises: + IOError: If file creation or writing fails + """ + try: + # Create temporary file with proper flags + fd, temp_path = tempfile.mkstemp( + suffix=suffix, + prefix=prefix, + dir=None, # Use system default temp directory + text=False, # Always use binary mode for consistency + ) + + # Convert to Path object + temp_path_obj = Path(temp_path) + + # Write content if provided + if content is not None: + try: + # Write using the file descriptor (more reliable on Windows) + os.write(fd, content) + finally: + # Always close the file descriptor + os.close(fd) + + self.logger.info( + f"Created temp file with content: {temp_path_obj.name} " + f"({len(content)} bytes)" + ) + else: + # Close immediately if no content + os.close(fd) + self.logger.info(f"Created empty temp file: {temp_path_obj.name}") + + # Track for cleanup + self.temp_files.append(temp_path_obj) + + return temp_path_obj + + except Exception as e: + self.logger.error(f"Failed to create temporary file: {e}") + raise IOError(f"Temporary file creation failed: {e}") + + @contextmanager + def temp_directory(self, prefix: str = "deepcode_"): + """ + Context manager for temporary directory with automatic cleanup. + + Args: + prefix: Directory prefix for identification + + Yields: + Path to temporary directory + + Example: + >>> with handler.temp_directory() as temp_dir: + ... # Use temp_dir + ... print(temp_dir) + # Directory automatically cleaned up after context + """ + temp_dir = None + try: + temp_dir = Path(tempfile.mkdtemp(prefix=prefix)) + self.logger.info(f"Created temporary directory: {temp_dir}") + yield temp_dir + finally: + if temp_dir and temp_dir.exists(): + try: + shutil.rmtree(temp_dir, ignore_errors=True) + self.logger.info(f"Cleaned up temporary directory: {temp_dir}") + except Exception as e: + self.logger.warning( + f"Failed to clean up temporary directory {temp_dir}: {e}" + ) + + def safe_copy_file( + self, + source: Union[str, Path], + destination: Union[str, Path], + preserve_metadata: bool = True, + overwrite: bool = False, + ) -> Path: + """ + Safely copy a file with proper error handling. + + This method uses copy instead of move to preserve the original file, + addressing the issue mentioned by the user. + + Args: + source: Source file path + destination: Destination file path + preserve_metadata: Whether to preserve file metadata (timestamps, etc.) + overwrite: Whether to overwrite if destination exists + + Returns: + Path to the destination file + + Raises: + FileNotFoundError: If source file doesn't exist + FileExistsError: If destination exists and overwrite=False + IOError: If copy operation fails + """ + source_path = self.normalize_path(source) + dest_path = self.normalize_path(destination) + + # Validate source + if not source_path.exists(): + raise FileNotFoundError(f"Source file not found: {source_path}") + + # Check destination + if dest_path.exists() and not overwrite: + raise FileExistsError( + f"Destination already exists: {dest_path}. " + f"Use overwrite=True to replace." + ) + + try: + # Ensure destination directory exists + dest_path.parent.mkdir(parents=True, exist_ok=True) + + # Copy file (preserves original!) + if preserve_metadata: + shutil.copy2(source_path, dest_path) + else: + shutil.copy(source_path, dest_path) + + self.logger.info( + f"Copied file: {source_path.name} -> {dest_path} " + f"({source_path.stat().st_size} bytes)" + ) + + return dest_path + + except Exception as e: + self.logger.error( + f"Failed to copy file from {source_path} to {dest_path}: {e}" + ) + raise IOError(f"File copy failed: {e}") + + def safe_move_file( + self, + source: Union[str, Path], + destination: Union[str, Path], + overwrite: bool = False, + ) -> Path: + """ + Safely move a file (only if explicitly needed). + + Note: Prefer safe_copy_file to preserve originals. + + Args: + source: Source file path + destination: Destination file path + overwrite: Whether to overwrite if destination exists + + Returns: + Path to the destination file + + Raises: + FileNotFoundError: If source file doesn't exist + FileExistsError: If destination exists and overwrite=False + IOError: If move operation fails + """ + source_path = self.normalize_path(source) + dest_path = self.normalize_path(destination) + + # Validate source + if not source_path.exists(): + raise FileNotFoundError(f"Source file not found: {source_path}") + + # Check destination + if dest_path.exists() and not overwrite: + raise FileExistsError( + f"Destination already exists: {dest_path}. " + f"Use overwrite=True to replace." + ) + + try: + # Ensure destination directory exists + dest_path.parent.mkdir(parents=True, exist_ok=True) + + # Move file + shutil.move(str(source_path), str(dest_path)) + + self.logger.info(f"Moved file: {source_path.name} -> {dest_path}") + + return dest_path + + except Exception as e: + self.logger.error( + f"Failed to move file from {source_path} to {dest_path}: {e}" + ) + raise IOError(f"File move failed: {e}") + + def safe_remove_file(self, file_path: Union[str, Path]) -> bool: + """ + Safely remove a file with proper error handling. + + Args: + file_path: Path to file to remove + + Returns: + True if file was removed, False if it didn't exist or removal failed + """ + path = self.normalize_path(file_path) + + if not path.exists(): + self.logger.debug(f"File already removed or doesn't exist: {path}") + return False + + try: + # On Windows, ensure file is not read-only + if self.platform == "Windows": + os.chmod(path, 0o777) + + path.unlink() + self.logger.info(f"Removed file: {path.name}") + + # Remove from tracking list if present + if path in self.temp_files: + self.temp_files.remove(path) + + return True + + except PermissionError as e: + self.logger.warning(f"Permission denied when removing {path}: {e}") + return False + except Exception as e: + self.logger.error(f"Failed to remove file {path}: {e}") + return False + + def cleanup_all_temp_files(self): + """ + Clean up all tracked temporary files. + + This is automatically called on program exit via atexit, + but can also be called manually. + """ + if not self.temp_files: + return + + self.logger.info(f"Cleaning up {len(self.temp_files)} temporary files...") + + cleaned = 0 + failed = 0 + + for temp_file in self.temp_files[ + : + ]: # Copy list to avoid modification during iteration + if self.safe_remove_file(temp_file): + cleaned += 1 + else: + failed += 1 + + self.logger.info(f"Cleanup complete: {cleaned} files removed, {failed} failed") + + self.temp_files.clear() + + def get_system_temp_dir(self) -> Path: + """ + Get the system temporary directory with proper cross-platform handling. + + Returns: + Path to system temporary directory + """ + return Path(tempfile.gettempdir()) + + def create_workspace_directory( + self, base_dir: Union[str, Path], workspace_name: str, clean: bool = False + ) -> Path: + """ + Create a workspace directory with proper structure. + + Args: + base_dir: Base directory for workspace + workspace_name: Name of the workspace + clean: Whether to clean the directory if it exists + + Returns: + Path to the created workspace directory + """ + base_path = self.normalize_path(base_dir) + workspace_path = base_path / workspace_name + + if clean and workspace_path.exists(): + self.logger.info(f"Cleaning existing workspace: {workspace_path}") + shutil.rmtree(workspace_path, ignore_errors=True) + + workspace_path.mkdir(parents=True, exist_ok=True) + self.logger.info(f"Created workspace directory: {workspace_path}") + + return workspace_path + + +# Singleton instance for convenience +_file_handler_instance: Optional[CrossPlatformFileHandler] = None + + +def get_file_handler( + logger: Optional[logging.Logger] = None, +) -> CrossPlatformFileHandler: + """ + Get or create a singleton file handler instance. + + Args: + logger: Optional logger instance + + Returns: + CrossPlatformFileHandler instance + """ + global _file_handler_instance + if _file_handler_instance is None: + _file_handler_instance = CrossPlatformFileHandler(logger) + return _file_handler_instance + + +# Example usage +if __name__ == "__main__": + # Configure logging + logging.basicConfig(level=logging.INFO) + + # Create handler + handler = CrossPlatformFileHandler() + + print(f"\n{'='*70}") + print("Cross-Platform File Handler - Demo") + print(f"{'='*70}\n") + + print(f"Platform: {handler.platform}") + print(f"System temp directory: {handler.get_system_temp_dir()}") + + # Demo: Create temporary file + print("\n1. Creating temporary file...") + temp_file = handler.create_safe_temp_file( + suffix=".txt", content=b"Test content for cross-platform file handling" + ) + print(f" Created: {temp_file}") + + # Demo: Use temporary directory + print("\n2. Using temporary directory...") + with handler.temp_directory() as temp_dir: + print(f" Temp directory: {temp_dir}") + test_file = temp_dir / "test.txt" + test_file.write_text("Hello from temp directory!") + print(f" Created file in temp dir: {test_file}") + print(" Temp directory automatically cleaned up") + + # Demo: Path normalization + print("\n3. Path normalization:") + test_paths = [ + "data/files\\test.txt", + "data\\files/test.txt", + "data\\files\\test.txt", + ] + for path in test_paths: + normalized = handler.normalize_path(path) + print(f" {path} -> {normalized}") + + # Demo: Cleanup + print("\n4. Cleaning up tracked files...") + handler.cleanup_all_temp_files() + + print(f"\n{'='*70}") + print("Demo completed successfully!") + print(f"{'='*70}\n") diff --git a/DeepCode/utils/dialogue_logger.py b/DeepCode/utils/dialogue_logger.py new file mode 100644 index 00000000..3123df8e --- /dev/null +++ b/DeepCode/utils/dialogue_logger.py @@ -0,0 +1,671 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Comprehensive Dialogue Logger for Code Implementation Workflow +Logs complete conversation rounds with detailed formatting and paper-specific organization +""" + +import json +import os +from datetime import datetime +from pathlib import Path +from typing import Dict, Any, List + + +class DialogueLogger: + """ + Comprehensive dialogue logger for code implementation workflow + Captures complete conversation rounds with proper formatting and organization + """ + + def __init__(self, paper_id: str, base_path: str = None): + """ + Initialize dialogue logger for a specific paper + + Args: + paper_id: Paper identifier (e.g., "1", "2", etc.) + base_path: Base path for logs (defaults to agent_folders structure) + """ + self.paper_id = paper_id + self.base_path = ( + base_path + or "/data2/bjdwhzzh/project-hku/Code-Agent2.0/Code-Agent/deepcode-mcp/agent_folders" + ) + self.log_directory = os.path.join( + self.base_path, "papers", str(paper_id), "logs" + ) + + # Create log directory if it doesn't exist + Path(self.log_directory).mkdir(parents=True, exist_ok=True) + + # Session tracking (initialize before log file creation) + self.round_counter = 0 + self.session_start_time = datetime.now() + self.current_round_data = {} + + # Generate log filename with timestamp + timestamp = self.session_start_time.strftime("%Y%m%d_%H%M%S") + self.log_filename = f"dialogue_log_{timestamp}.md" + self.log_filepath = os.path.join(self.log_directory, self.log_filename) + + # Initialize log file with header + self._initialize_log_file() + + print(f"๐Ÿ“ Dialogue Logger initialized for Paper {paper_id}") + print(f"๐Ÿ“ Log file: {self.log_filepath}") + + def _initialize_log_file(self): + """Initialize the log file with header information""" + header = f"""# Code Implementation Dialogue Log + +**Paper ID:** {self.paper_id} +**Session Start:** {self.session_start_time.strftime('%Y-%m-%d %H:%M:%S')} +**Log File:** {self.log_filename} + +--- + +## Session Overview + +This log contains the complete conversation rounds between the user and assistant during the code implementation workflow. Each round includes: + +- System prompts and user messages +- Assistant responses with tool calls +- Tool execution results +- Implementation progress markers + +--- + +""" + try: + with open(self.log_filepath, "w", encoding="utf-8") as f: + f.write(header) + except Exception as e: + print(f"โš ๏ธ Failed to initialize log file: {e}") + + def start_new_round( + self, round_type: str = "implementation", context: Dict[str, Any] = None + ): + """ + Start a new dialogue round + + Args: + round_type: Type of round (implementation, summary, error_handling, etc.) + context: Additional context information (may include 'iteration' to sync with workflow) + """ + # Use iteration from context if provided, otherwise increment round_counter + if context and "iteration" in context: + self.round_counter = context["iteration"] + else: + self.round_counter += 1 + + self.current_round_data = { + "round_number": self.round_counter, + "round_type": round_type, + "start_time": datetime.now(), + "context": context or {}, + "messages": [], + "tool_calls": [], + "results": [], + "metadata": {}, + } + + print(f"๐Ÿ”„ Starting Round {self.round_counter}: {round_type}") + + def log_system_prompt(self, prompt: str, prompt_type: str = "system"): + """ + Log system prompt or instructions + + Args: + prompt: System prompt content + prompt_type: Type of prompt (system, instruction, etc.) + """ + if not self.current_round_data: + self.start_new_round("system_setup") + + self.current_round_data["messages"].append( + { + "role": "system", + "type": prompt_type, + "content": prompt, + "timestamp": datetime.now().isoformat(), + } + ) + + def log_user_message(self, message: str, message_type: str = "user_input"): + """ + Log user message + + Args: + message: User message content + message_type: Type of message (user_input, feedback, guidance, etc.) + """ + if not self.current_round_data: + self.start_new_round("user_interaction") + + self.current_round_data["messages"].append( + { + "role": "user", + "type": message_type, + "content": message, + "timestamp": datetime.now().isoformat(), + } + ) + + def log_assistant_response( + self, response: str, response_type: str = "assistant_response" + ): + """ + Log assistant response + + Args: + response: Assistant response content + response_type: Type of response (assistant_response, analysis, etc.) + """ + if not self.current_round_data: + self.start_new_round("assistant_interaction") + + self.current_round_data["messages"].append( + { + "role": "assistant", + "type": response_type, + "content": response, + "timestamp": datetime.now().isoformat(), + } + ) + + def log_tool_calls(self, tool_calls: List[Dict[str, Any]]): + """ + Log tool calls made by the assistant + + Args: + tool_calls: List of tool calls with id, name, and input + """ + if not self.current_round_data: + self.start_new_round("tool_execution") + + for tool_call in tool_calls: + self.current_round_data["tool_calls"].append( + { + "id": tool_call.get("id", ""), + "name": tool_call.get("name", ""), + "input": tool_call.get("input", {}), + "timestamp": datetime.now().isoformat(), + } + ) + + def log_tool_results(self, tool_results: List[Dict[str, Any]]): + """ + Log tool execution results + + Args: + tool_results: List of tool results with tool_name and result + """ + if not self.current_round_data: + self.start_new_round("tool_results") + + for result in tool_results: + self.current_round_data["results"].append( + { + "tool_name": result.get("tool_name", ""), + "result": result.get("result", ""), + "timestamp": datetime.now().isoformat(), + } + ) + + def log_metadata(self, key: str, value: Any): + """ + Log metadata information + + Args: + key: Metadata key + value: Metadata value + """ + if not self.current_round_data: + self.start_new_round("metadata") + + self.current_round_data["metadata"][key] = value + + def log_memory_optimization( + self, + messages_before: List[Dict], + messages_after: List[Dict], + optimization_stats: Dict[str, Any], + approach: str = "memory_optimization", + ): + """ + Log memory optimization details including before/after message content + + Args: + messages_before: Messages before optimization + messages_after: Messages after optimization + optimization_stats: Statistics about the optimization + approach: Optimization approach used + """ + if not self.current_round_data: + self.start_new_round("memory_optimization") + + # Calculate what was removed/kept + removed_count = len(messages_before) - len(messages_after) + compression_ratio = ( + (removed_count / len(messages_before) * 100) if messages_before else 0 + ) + + # Log the optimization details + optimization_data = { + "approach": approach, + "messages_before_count": len(messages_before), + "messages_after_count": len(messages_after), + "messages_removed_count": removed_count, + "compression_ratio": f"{compression_ratio:.1f}%", + "optimization_stats": optimization_stats, + "timestamp": datetime.now().isoformat(), + } + + # Store the optimization data + if "memory_optimizations" not in self.current_round_data: + self.current_round_data["memory_optimizations"] = [] + + self.current_round_data["memory_optimizations"].append( + { + "optimization_data": optimization_data, + "messages_before": messages_before, + "messages_after": messages_after, + } + ) + + # Log metadata + self.log_metadata("memory_optimization", optimization_data) + + print( + f"๐Ÿงน Memory optimization logged: {len(messages_before)} โ†’ {len(messages_after)} messages ({compression_ratio:.1f}% compression)" + ) + + def complete_round(self, summary: str = "", status: str = "completed"): + """ + Complete the current round and write to log file + + Args: + summary: Round summary + status: Round completion status + """ + if not self.current_round_data: + print("โš ๏ธ No active round to complete") + return + + self.current_round_data["end_time"] = datetime.now() + self.current_round_data["duration"] = ( + self.current_round_data["end_time"] - self.current_round_data["start_time"] + ).total_seconds() + self.current_round_data["summary"] = summary + self.current_round_data["status"] = status + + # Write round to log file + self._write_round_to_log() + + print(f"โœ… Round {self.round_counter} completed: {status}") + + # Clear current round data + self.current_round_data = {} + + def _write_round_to_log(self): + """Write the current round data to the log file in markdown format""" + try: + with open(self.log_filepath, "a", encoding="utf-8") as f: + round_data = self.current_round_data + + # Round header + f.write( + f"\n## Round {round_data['round_number']}: {round_data['round_type'].title()}\n\n" + ) + f.write( + f"**Start Time:** {round_data['start_time'].strftime('%Y-%m-%d %H:%M:%S')}\n" + ) + f.write( + f"**End Time:** {round_data['end_time'].strftime('%Y-%m-%d %H:%M:%S')}\n" + ) + f.write(f"**Duration:** {round_data['duration']:.2f} seconds\n") + f.write(f"**Status:** {round_data['status']}\n\n") + + # Context information + if round_data.get("context"): + f.write("### Context\n\n") + for key, value in round_data["context"].items(): + f.write(f"- **{key}:** {value}\n") + f.write("\n") + + # Messages + if round_data.get("messages"): + f.write("### Messages\n\n") + for i, msg in enumerate(round_data["messages"], 1): + role_emoji = { + "system": "๐Ÿ”ง", + "user": "๐Ÿ‘ค", + "assistant": "๐Ÿค–", + }.get(msg["role"], "๐Ÿ“") + f.write( + f"#### {role_emoji} {msg['role'].title()} Message {i}\n\n" + ) + f.write(f"**Type:** {msg['type']}\n") + f.write(f"**Timestamp:** {msg['timestamp']}\n\n") + f.write("```\n") + f.write(msg["content"]) + f.write("\n```\n\n") + + # Tool calls + if round_data.get("tool_calls"): + f.write("### Tool Calls\n\n") + for i, tool_call in enumerate(round_data["tool_calls"], 1): + f.write(f"#### ๐Ÿ› ๏ธ Tool Call {i}: {tool_call['name']}\n\n") + f.write(f"**ID:** {tool_call['id']}\n") + f.write(f"**Timestamp:** {tool_call['timestamp']}\n\n") + f.write("**Input:**\n") + f.write("```json\n") + f.write( + json.dumps(tool_call["input"], indent=2, ensure_ascii=False) + ) + f.write("\n```\n\n") + + # Tool results + if round_data.get("results"): + f.write("### Tool Results\n\n") + for i, result in enumerate(round_data["results"], 1): + f.write(f"#### ๐Ÿ“Š Result {i}: {result['tool_name']}\n\n") + f.write(f"**Timestamp:** {result['timestamp']}\n\n") + f.write("**Result:**\n") + f.write("```\n") + f.write(str(result["result"])) + f.write("\n```\n\n") + + # Memory Optimizations + if round_data.get("memory_optimizations"): + f.write("### Memory Optimizations\n\n") + for i, opt in enumerate(round_data["memory_optimizations"], 1): + opt_data = opt["optimization_data"] + messages_before = opt["messages_before"] + messages_after = opt["messages_after"] + + f.write(f"#### ๐Ÿงน Memory Optimization {i}\n\n") + f.write(f"**Approach:** {opt_data['approach']}\n") + f.write( + f"**Messages Before:** {opt_data['messages_before_count']}\n" + ) + f.write( + f"**Messages After:** {opt_data['messages_after_count']}\n" + ) + f.write( + f"**Messages Removed:** {opt_data['messages_removed_count']}\n" + ) + f.write( + f"**Compression Ratio:** {opt_data['compression_ratio']}\n" + ) + f.write(f"**Timestamp:** {opt_data['timestamp']}\n\n") + + # Show optimization stats + if opt_data.get("optimization_stats"): + f.write("**Optimization Statistics:**\n") + f.write("```json\n") + f.write( + json.dumps( + opt_data["optimization_stats"], + indent=2, + ensure_ascii=False, + ) + ) + f.write("\n```\n\n") + + # Show messages before optimization (limited to last 5 for readability) + if messages_before: + f.write("**Messages Before Optimization (last 5):**\n\n") + for j, msg in enumerate(messages_before[-5:], 1): + role = msg.get("role", "unknown") + content = msg.get("content", "") + # Truncate very long messages + if len(content) > 3000: + content = content[:3000] + "...[truncated]" + f.write( + f"- **{role} {j}:** {content[:3000]}{'...' if len(content) > 100 else ''}\n" + ) + f.write("\n") + + # Show messages after optimization + if messages_after: + f.write("**Messages After Optimization:**\n\n") + for j, msg in enumerate(messages_after, 1): + role = msg.get("role", "unknown") + content = msg.get("content", "") + # Truncate very long messages + if len(content) > 3000: + content = content[:3000] + "...[truncated]" + f.write( + f"- **{role} {j}:** {content[:3000]}{'...' if len(content) > 100 else ''}\n" + ) + f.write("\n") + + # Show what was removed + if len(messages_before) > len(messages_after): + removed_messages = ( + messages_before[: -len(messages_after)] + if messages_after + else messages_before + ) + f.write( + f"**Messages Removed ({len(removed_messages)}):**\n\n" + ) + for j, msg in enumerate( + removed_messages[-3:], 1 + ): # Show last 3 removed + role = msg.get("role", "unknown") + content = msg.get("content", "") + if len(content) > 3000: + content = content[:3000] + "...[truncated]" + f.write(f"- **{role} {j}:** {content}\n") + f.write("\n") + + f.write("\n") + + # Metadata + if round_data.get("metadata"): + f.write("### Metadata\n\n") + for key, value in round_data["metadata"].items(): + if ( + key != "memory_optimization" + ): # Skip memory optimization metadata as it's shown above + f.write(f"- **{key}:** {value}\n") + f.write("\n") + + # Summary + if round_data.get("summary"): + f.write("### Summary\n\n") + f.write(round_data["summary"]) + f.write("\n\n") + + # Separator + f.write("---\n\n") + + except Exception as e: + print(f"โš ๏ธ Failed to write round to log: {e}") + + def log_complete_exchange( + self, + system_prompt: str = "", + user_message: str = "", + assistant_response: str = "", + tool_calls: List[Dict] = None, + tool_results: List[Dict] = None, + round_type: str = "exchange", + context: Dict = None, + summary: str = "", + ): + """ + Log a complete exchange in a single call + + Args: + system_prompt: System prompt (optional) + user_message: User message + assistant_response: Assistant response + tool_calls: Tool calls made + tool_results: Tool execution results + round_type: Type of round + context: Additional context + summary: Round summary + """ + self.start_new_round(round_type, context) + + if system_prompt: + self.log_system_prompt(system_prompt) + + if user_message: + self.log_user_message(user_message) + + if assistant_response: + self.log_assistant_response(assistant_response) + + if tool_calls: + self.log_tool_calls(tool_calls) + + if tool_results: + self.log_tool_results(tool_results) + + self.complete_round(summary) + + def get_session_stats(self) -> Dict[str, Any]: + """Get session statistics""" + return { + "paper_id": self.paper_id, + "session_start": self.session_start_time.isoformat(), + "total_rounds": self.round_counter, + "log_file": self.log_filepath, + "session_duration": ( + datetime.now() - self.session_start_time + ).total_seconds(), + } + + def finalize_session(self, final_summary: str = ""): + """ + Finalize the logging session + + Args: + final_summary: Final session summary + """ + try: + with open(self.log_filepath, "a", encoding="utf-8") as f: + f.write("\n## Session Summary\n\n") + f.write(f"**Total Rounds:** {self.round_counter}\n") + f.write( + f"**Session Duration:** {(datetime.now() - self.session_start_time).total_seconds():.2f} seconds\n" + ) + f.write( + f"**End Time:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n" + ) + + if final_summary: + f.write("### Final Summary\n\n") + f.write(final_summary) + f.write("\n\n") + + f.write("---\n\n") + f.write("*End of Session*\n") + + except Exception as e: + print(f"โš ๏ธ Failed to finalize session: {e}") + + print(f"๐ŸŽฏ Session finalized: {self.round_counter} rounds logged") + + +# Utility functions for easy integration +def create_dialogue_logger(paper_id: str, base_path: str = None) -> DialogueLogger: + """ + Create a dialogue logger for a specific paper + + Args: + paper_id: Paper identifier + base_path: Base path for logs + + Returns: + DialogueLogger instance + """ + return DialogueLogger(paper_id, base_path) + + +def extract_paper_id_from_path(path: str) -> str: + """ + Extract paper ID from a file path + + Args: + path: File path containing paper information + + Returns: + Paper ID string + """ + # Extract paper ID from path like "/data2/.../papers/1/initial_plan.txt" + parts = path.split("/") + for i, part in enumerate(parts): + if part == "papers" and i + 1 < len(parts): + return parts[i + 1] + return "unknown" + + +# Example usage +if __name__ == "__main__": + # Test the dialogue logger + logger = DialogueLogger("1") + + # Log a complete exchange + logger.log_complete_exchange( + system_prompt="You are a code implementation assistant.", + user_message="Implement the transformer model", + assistant_response="I'll implement the transformer model step by step.", + tool_calls=[ + {"id": "1", "name": "write_file", "input": {"filename": "transformer.py"}} + ], + tool_results=[ + {"tool_name": "write_file", "result": "File created successfully"} + ], + round_type="implementation", + context={"files_implemented": 1}, + summary="Successfully implemented transformer model", + ) + + # Test memory optimization logging + logger.start_new_round( + "memory_optimization", {"trigger_reason": "write_file_detected"} + ) + + # Mock messages before and after optimization + messages_before = [ + {"role": "user", "content": "Original message 1"}, + {"role": "assistant", "content": "Original response 1"}, + {"role": "user", "content": "Original message 2"}, + {"role": "assistant", "content": "Original response 2"}, + {"role": "user", "content": "Original message 3"}, + ] + + messages_after = [ + {"role": "user", "content": "Original message 1"}, + {"role": "assistant", "content": "Original response 1"}, + {"role": "user", "content": "Original message 3"}, + ] + + # Mock optimization stats + optimization_stats = { + "implemented_files_tracked": 2, + "current_round": 5, + "concise_mode_active": True, + } + + # Log memory optimization + logger.log_memory_optimization( + messages_before=messages_before, + messages_after=messages_after, + optimization_stats=optimization_stats, + approach="clear_after_write_file", + ) + + logger.complete_round("Memory optimization test completed") + + # Finalize session + logger.finalize_session( + "Test session with memory optimization logging completed successfully" + ) + + print("โœ… Dialogue logger test completed with memory optimization") diff --git a/DeepCode/utils/file_processor.py b/DeepCode/utils/file_processor.py new file mode 100644 index 00000000..79e26ba5 --- /dev/null +++ b/DeepCode/utils/file_processor.py @@ -0,0 +1,453 @@ +""" +File processing utilities for handling paper files and related operations. +""" + +import json +import os +import re +from typing import Dict, List, Optional, Union + + +class FileProcessor: + """ + A class to handle file processing operations including path extraction and file reading. + """ + + @staticmethod + def extract_file_path(file_info: Union[str, Dict]) -> Optional[str]: + """ + Extract paper directory path from the input information. + + Args: + file_info: Either a JSON string or a dictionary containing file information + + Returns: + Optional[str]: The extracted paper directory path or None if not found + """ + try: + # Handle direct file path input + if isinstance(file_info, str): + # Check if it's a file path (existing or not) + if file_info.endswith( + (".md", ".pdf", ".txt", ".docx", ".doc", ".html", ".htm") + ): + # It's a file path, return the directory + return os.path.dirname(os.path.abspath(file_info)) + elif os.path.exists(file_info): + if os.path.isfile(file_info): + return os.path.dirname(os.path.abspath(file_info)) + elif os.path.isdir(file_info): + return os.path.abspath(file_info) + + # Try to parse as JSON + try: + info_dict = json.loads(file_info) + except json.JSONDecodeError: + # ๅฐ่ฏ•ไปŽๆ–‡ๆœฌไธญๆๅ–JSON + info_dict = FileProcessor.extract_json_from_text(file_info) + if not info_dict: + # If not JSON and doesn't look like a file path, raise error + raise ValueError( + f"Input is neither a valid file path nor JSON: {file_info}" + ) + else: + info_dict = file_info + + # Extract paper path from dictionary + paper_path = info_dict.get("paper_path") + if not paper_path: + raise ValueError("No paper_path found in input dictionary") + + # Get the directory path instead of the file path + paper_dir = os.path.dirname(paper_path) + + # Convert to absolute path if relative + if not os.path.isabs(paper_dir): + paper_dir = os.path.abspath(paper_dir) + + return paper_dir + + except (AttributeError, TypeError) as e: + raise ValueError(f"Invalid input format: {str(e)}") + + @staticmethod + def find_markdown_file(directory: str) -> Optional[str]: + """ + Find the first markdown file in the given directory. + + Args: + directory: Directory path to search + + Returns: + Optional[str]: Path to the markdown file or None if not found + """ + if not os.path.isdir(directory): + return None + + for file in os.listdir(directory): + if file.endswith(".md"): + return os.path.join(directory, file) + return None + + @staticmethod + def parse_markdown_sections(content: str) -> List[Dict[str, Union[str, int, List]]]: + """ + Parse markdown content and organize it by sections based on headers. + + Args: + content: The markdown content to parse + + Returns: + List[Dict]: A list of sections, each containing: + - level: The header level (1-6) + - title: The section title + - content: The section content + - subsections: List of subsections + """ + # Split content into lines + lines = content.split("\n") + sections = [] + current_section = None + current_content = [] + + for line in lines: + # Check if line is a header + header_match = re.match(r"^(#{1,6})\s+(.+)$", line) + + if header_match: + # If we were building a section, save its content + if current_section is not None: + current_section["content"] = "\n".join(current_content).strip() + sections.append(current_section) + + # Start a new section + level = len(header_match.group(1)) + title = header_match.group(2).strip() + current_section = { + "level": level, + "title": title, + "content": "", + "subsections": [], + } + current_content = [] + elif current_section is not None: + current_content.append(line) + + # Don't forget to save the last section + if current_section is not None: + current_section["content"] = "\n".join(current_content).strip() + sections.append(current_section) + + return FileProcessor._organize_sections(sections) + + @staticmethod + def _organize_sections(sections: List[Dict]) -> List[Dict]: + """ + Organize sections into a hierarchical structure based on their levels. + + Args: + sections: List of sections with their levels + + Returns: + List[Dict]: Organized hierarchical structure of sections + """ + result = [] + section_stack = [] + + for section in sections: + while section_stack and section_stack[-1]["level"] >= section["level"]: + section_stack.pop() + + if section_stack: + section_stack[-1]["subsections"].append(section) + else: + result.append(section) + + section_stack.append(section) + + return result + + @staticmethod + async def read_file_content(file_path: str) -> str: + """ + Read the content of a file asynchronously. + + Args: + file_path: Path to the file to read + + Returns: + str: The content of the file + + Raises: + FileNotFoundError: If the file doesn't exist + IOError: If there's an error reading the file + """ + try: + # Ensure the file exists + if not os.path.exists(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + + # Check if file is actually a PDF by reading the first few bytes + with open(file_path, "rb") as f: + header = f.read(8) + if header.startswith(b"%PDF"): + raise IOError( + f"File {file_path} is a PDF file, not a text file. Please convert it to markdown format or use PDF processing tools." + ) + + # Read file content + # Note: Using async with would be better for large files + # but for simplicity and compatibility, using regular file reading + with open(file_path, "r", encoding="utf-8") as f: + content = f.read() + + return content + + except UnicodeDecodeError as e: + raise IOError( + f"Error reading file {file_path}: File encoding is not UTF-8. Original error: {str(e)}" + ) + except Exception as e: + raise IOError(f"Error reading file {file_path}: {str(e)}") + + @staticmethod + def format_section_content(section: Dict) -> str: + """ + Format a section's content with standardized spacing and structure. + + Args: + section: Dictionary containing section information + + Returns: + str: Formatted section content + """ + # Start with section title + formatted = f"\n{'#' * section['level']} {section['title']}\n" + + # Add section content if it exists + if section["content"]: + formatted += f"\n{section['content'].strip()}\n" + + # Process subsections + if section["subsections"]: + # Add a separator before subsections if there's content + if section["content"]: + formatted += "\n---\n" + + # Process each subsection + for subsection in section["subsections"]: + formatted += FileProcessor.format_section_content(subsection) + + # Add section separator + formatted += "\n" + "=" * 80 + "\n" + + return formatted + + @staticmethod + def standardize_output(sections: List[Dict]) -> str: + """ + Convert structured sections into a standardized string format. + + Args: + sections: List of section dictionaries + + Returns: + str: Standardized string output + """ + output = [] + + # Process each top-level section + for section in sections: + output.append(FileProcessor.format_section_content(section)) + + # Join all sections with clear separation + return "\n".join(output) + + @classmethod + async def process_file_input( + cls, file_input: Union[str, Dict], base_dir: str = None + ) -> Dict: + """ + Process file input information and return the structured content. + + Args: + file_input: File input information (JSON string, dict, or direct file path) + base_dir: Optional base directory to use for creating paper directories (for sync support) + + Returns: + Dict: The structured content with sections and standardized text + """ + try: + # ้ฆ–ๅ…ˆๅฐ่ฏ•ไปŽๅญ—็ฌฆไธฒไธญๆๅ–markdownๆ–‡ไปถ่ทฏๅพ„ + if isinstance(file_input, str): + import re + + # Try to extract path from backticks first + file_path_match = re.search(r"`([^`]+\.md)`", file_input) + if file_path_match: + paper_path = file_path_match.group(1) + file_input = {"paper_path": paper_path} + else: + # Try to extract from "Saved Path:" or similar patterns + path_patterns = [ + r"[Ss]aved [Pp]ath[:\s]+([^\s\n]+\.md)", + r"[Pp]aper [Pp]ath[:\s]+([^\s\n]+\.md)", + r"[Ff]ile[:\s]+([^\s\n]+\.md)", + r"[Oo]utput[:\s]+([^\s\n]+\.md)", + ] + for pattern in path_patterns: + match = re.search(pattern, file_input) + if match: + paper_path = match.group(1) + file_input = {"paper_path": paper_path} + break + + # Extract paper directory path + paper_dir = cls.extract_file_path(file_input) + + # If base_dir is provided, adjust paper_dir to be relative to base_dir + if base_dir and paper_dir: + # If paper_dir is using default location, move it to base_dir + if paper_dir.endswith(("deepcode_lab", "agent_folders")): + paper_dir = base_dir + else: + # Extract the relative part and combine with base_dir + paper_name = os.path.basename(paper_dir) + # ไฟๆŒๅŽŸๅง‹็›ฎๅฝ•ๅไธๅ˜๏ผŒไธๅšไปปไฝ•ๆ›ฟๆข + paper_dir = os.path.join(base_dir, "papers", paper_name) + + # Ensure the directory exists + os.makedirs(paper_dir, exist_ok=True) + + if not paper_dir: + raise ValueError("Could not determine paper directory path") + + # Get the actual file path + file_path = None + if isinstance(file_input, str): + # ๅฐ่ฏ•่งฃๆžไธบJSON๏ผˆๅค„็†ไธ‹่ฝฝ็ป“ๆžœ๏ผ‰ + try: + parsed_json = json.loads(file_input) + if isinstance(parsed_json, dict) and "paper_path" in parsed_json: + file_path = parsed_json.get("paper_path") + # ๅฆ‚ๆžœๆ–‡ไปถไธๅญ˜ๅœจ๏ผŒๅฐ่ฏ•ๆŸฅๆ‰พmarkdownๆ–‡ไปถ + if file_path and not os.path.exists(file_path): + paper_dir = os.path.dirname(file_path) + if os.path.isdir(paper_dir): + file_path = cls.find_markdown_file(paper_dir) + if not file_path: + raise ValueError( + f"No markdown file found in directory: {paper_dir}" + ) + else: + raise ValueError("Invalid JSON format: missing paper_path") + except json.JSONDecodeError: + # ๅฐ่ฏ•ไปŽๆ–‡ๆœฌไธญๆๅ–JSON๏ผˆๅค„็†ๅŒ…ๅซ้ขๅค–ๆ–‡ๆœฌ็š„ไธ‹่ฝฝ็ป“ๆžœ๏ผ‰ + extracted_json = cls.extract_json_from_text(file_input) + if extracted_json and "paper_path" in extracted_json: + file_path = extracted_json.get("paper_path") + # ๅฆ‚ๆžœๆ–‡ไปถไธๅญ˜ๅœจ๏ผŒๅฐ่ฏ•ๆŸฅๆ‰พmarkdownๆ–‡ไปถ + if file_path and not os.path.exists(file_path): + paper_dir = os.path.dirname(file_path) + if os.path.isdir(paper_dir): + file_path = cls.find_markdown_file(paper_dir) + if not file_path: + raise ValueError( + f"No markdown file found in directory: {paper_dir}" + ) + else: + # ไธๆ˜ฏJSON๏ผŒๆŒ‰ๆ–‡ไปถ่ทฏๅพ„ๅค„็† + # Check if it's a file path (existing or not) + if file_input.endswith( + (".md", ".pdf", ".txt", ".docx", ".doc", ".html", ".htm") + ): + if os.path.exists(file_input): + file_path = file_input + else: + # File doesn't exist, try to find markdown in the directory + file_path = cls.find_markdown_file(paper_dir) + if not file_path: + raise ValueError( + f"No markdown file found in directory: {paper_dir}" + ) + elif os.path.exists(file_input): + if os.path.isfile(file_input): + file_path = file_input + elif os.path.isdir(file_input): + # If it's a directory, find the markdown file + file_path = cls.find_markdown_file(file_input) + if not file_path: + raise ValueError( + f"No markdown file found in directory: {file_input}" + ) + else: + raise ValueError(f"Invalid input: {file_input}") + else: + # Dictionary input + file_path = file_input.get("paper_path") + # If the file doesn't exist, try to find markdown in the directory + if file_path and not os.path.exists(file_path): + paper_dir = os.path.dirname(file_path) + if os.path.isdir(paper_dir): + file_path = cls.find_markdown_file(paper_dir) + if not file_path: + raise ValueError( + f"No markdown file found in directory: {paper_dir}" + ) + + if not file_path: + raise ValueError("No valid file path found") + + # Read file content + content = await cls.read_file_content(file_path) + + # Parse and structure the content + structured_content = cls.parse_markdown_sections(content) + + # Generate standardized text output + standardized_text = cls.standardize_output(structured_content) + + return { + "paper_dir": paper_dir, + "file_path": file_path, + "sections": structured_content, + "standardized_text": standardized_text, + } + + except Exception as e: + raise ValueError(f"Error processing file input: {str(e)}") + + @staticmethod + def extract_json_from_text(text: str) -> Optional[Dict]: + """ + Extract JSON from text that may contain markdown code blocks or other content. + + Args: + text: Text that may contain JSON + + Returns: + Optional[Dict]: Extracted JSON as dictionary or None if not found + """ + import re + + # Try to find JSON in markdown code blocks + json_pattern = r"```json\s*(\{.*?\})\s*```" + match = re.search(json_pattern, text, re.DOTALL) + if match: + try: + return json.loads(match.group(1)) + except json.JSONDecodeError: + pass + + # Try to find standalone JSON + json_pattern = r"(\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\})" + matches = re.findall(json_pattern, text, re.DOTALL) + for match in matches: + try: + parsed = json.loads(match) + if isinstance(parsed, dict) and "paper_path" in parsed: + return parsed + except json.JSONDecodeError: + continue + + return None diff --git a/DeepCode/utils/llm_utils.py b/DeepCode/utils/llm_utils.py new file mode 100644 index 00000000..f8bbccdd --- /dev/null +++ b/DeepCode/utils/llm_utils.py @@ -0,0 +1,322 @@ +""" +LLM utility functions for DeepCode project. + +This module provides common LLM-related utilities to avoid circular imports +and reduce code duplication across the project. +""" + +import os +import yaml +from typing import Any, Type, Dict, Tuple + +# Import LLM classes +from mcp_agent.workflows.llm.augmented_llm_anthropic import AnthropicAugmentedLLM +from mcp_agent.workflows.llm.augmented_llm_openai import OpenAIAugmentedLLM +from mcp_agent.workflows.llm.augmented_llm_google import GoogleAugmentedLLM + + +def get_preferred_llm_class(config_path: str = "mcp_agent.secrets.yaml") -> Type[Any]: + """ + Select the LLM class based on user preference and API key availability. + + Priority: + 1. Check mcp_agent.config.yaml for llm_provider preference + 2. Verify the preferred provider has API key + 3. Fallback to first available provider + + Args: + config_path: Path to the secrets YAML configuration file + + Returns: + class: The preferred LLM class + """ + try: + # Read API keys from secrets file + if not os.path.exists(config_path): + print(f"๐Ÿค– Config file {config_path} not found, using OpenAIAugmentedLLM") + return OpenAIAugmentedLLM + + with open(config_path, "r", encoding="utf-8") as f: + secrets = yaml.safe_load(f) + + # Get API keys + anthropic_key = secrets.get("anthropic", {}).get("api_key", "").strip() + google_key = secrets.get("google", {}).get("api_key", "").strip() + openai_key = secrets.get("openai", {}).get("api_key", "").strip() + + # Read user preference from main config + main_config_path = "mcp_agent.config.yaml" + preferred_provider = None + if os.path.exists(main_config_path): + with open(main_config_path, "r", encoding="utf-8") as f: + main_config = yaml.safe_load(f) + preferred_provider = main_config.get("llm_provider", "").strip().lower() + + # Map of providers to their classes and keys + provider_map = { + "anthropic": ( + AnthropicAugmentedLLM, + anthropic_key, + "AnthropicAugmentedLLM", + ), + "google": (GoogleAugmentedLLM, google_key, "GoogleAugmentedLLM"), + "openai": (OpenAIAugmentedLLM, openai_key, "OpenAIAugmentedLLM"), + } + + # Try user's preferred provider first + if preferred_provider and preferred_provider in provider_map: + llm_class, api_key, class_name = provider_map[preferred_provider] + if api_key: + print(f"๐Ÿค– Using {class_name} (user preference: {preferred_provider})") + return llm_class + else: + print( + f"โš ๏ธ Preferred provider '{preferred_provider}' has no API key, checking alternatives..." + ) + + # Fallback: try providers in order of availability + for provider, (llm_class, api_key, class_name) in provider_map.items(): + if api_key: + print(f"๐Ÿค– Using {class_name} ({provider} API key found)") + return llm_class + + # No API keys found + print("โš ๏ธ No API keys configured, falling back to OpenAIAugmentedLLM") + return OpenAIAugmentedLLM + + except Exception as e: + print(f"๐Ÿค– Error reading config file {config_path}: {e}") + print("๐Ÿค– Falling back to OpenAIAugmentedLLM") + return OpenAIAugmentedLLM + + +def get_token_limits(config_path: str = "mcp_agent.config.yaml") -> Tuple[int, int]: + """ + Get token limits from configuration. + + Args: + config_path: Path to the main configuration file + + Returns: + tuple: (base_max_tokens, retry_max_tokens) + """ + # Default values that work with qwen/qwen-max (32768 total context) + default_base = 20000 + default_retry = 15000 + + try: + if os.path.exists(config_path): + with open(config_path, "r", encoding="utf-8") as f: + config = yaml.safe_load(f) + + openai_config = config.get("openai", {}) + base_tokens = openai_config.get("base_max_tokens", default_base) + retry_tokens = openai_config.get("retry_max_tokens", default_retry) + + print( + f"โš™๏ธ Token limits from config: base={base_tokens}, retry={retry_tokens}" + ) + return base_tokens, retry_tokens + else: + print( + f"โš ๏ธ Config file {config_path} not found, using defaults: base={default_base}, retry={default_retry}" + ) + return default_base, default_retry + except Exception as e: + print(f"โš ๏ธ Error reading token config from {config_path}: {e}") + print( + f"๐Ÿ”ง Falling back to default token limits: base={default_base}, retry={default_retry}" + ) + return default_base, default_retry + + +def get_default_models(config_path: str = "mcp_agent.config.yaml"): + """ + Get default models from configuration file. + + Args: + config_path: Path to the configuration file + + Returns: + dict: Dictionary with 'anthropic', 'openai', and 'google' default models + """ + try: + if os.path.exists(config_path): + with open(config_path, "r", encoding="utf-8") as f: + config = yaml.safe_load(f) + + # Handle null values in config sections + anthropic_config = config.get("anthropic") or {} + openai_config = config.get("openai") or {} + google_config = config.get("google") or {} + + anthropic_model = anthropic_config.get( + "default_model", "claude-sonnet-4-20250514" + ) + openai_model = openai_config.get("default_model", "o3-mini") + google_model = google_config.get("default_model", "gemini-2.0-flash") + + return { + "anthropic": anthropic_model, + "openai": openai_model, + "google": google_model, + } + else: + print(f"Config file {config_path} not found, using default models") + return { + "anthropic": "claude-sonnet-4-20250514", + "openai": "o3-mini", + "google": "gemini-2.0-flash", + } + + except Exception as e: + print(f"โŒError reading config file {config_path}: {e}") + return { + "anthropic": "claude-sonnet-4-20250514", + "openai": "o3-mini", + "google": "gemini-2.0-flash", + } + + +def get_document_segmentation_config( + config_path: str = "mcp_agent.config.yaml", +) -> Dict[str, Any]: + """ + Get document segmentation configuration from config file. + + Args: + config_path: Path to the main configuration file + + Returns: + Dict containing segmentation configuration with default values + """ + try: + if os.path.exists(config_path): + with open(config_path, "r", encoding="utf-8") as f: + config = yaml.safe_load(f) + + # Get document segmentation config with defaults + seg_config = config.get("document_segmentation", {}) + return { + "enabled": seg_config.get("enabled", True), + "size_threshold_chars": seg_config.get("size_threshold_chars", 50000), + } + else: + print( + f"๐Ÿ“„ Config file {config_path} not found, using default segmentation settings" + ) + return {"enabled": True, "size_threshold_chars": 50000} + + except Exception as e: + print(f"๐Ÿ“„ Error reading segmentation config from {config_path}: {e}") + print("๐Ÿ“„ Using default segmentation settings") + return {"enabled": True, "size_threshold_chars": 50000} + + +def should_use_document_segmentation( + document_content: str, config_path: str = "mcp_agent.config.yaml" +) -> Tuple[bool, str]: + """ + Determine whether to use document segmentation based on configuration and document size. + + Args: + document_content: The content of the document to analyze + config_path: Path to the configuration file + + Returns: + Tuple of (should_segment, reason) where: + - should_segment: Boolean indicating whether to use segmentation + - reason: String explaining the decision + """ + seg_config = get_document_segmentation_config(config_path) + + if not seg_config["enabled"]: + return False, "Document segmentation disabled in configuration" + + doc_size = len(document_content) + threshold = seg_config["size_threshold_chars"] + + if doc_size > threshold: + return ( + True, + f"Document size ({doc_size:,} chars) exceeds threshold ({threshold:,} chars)", + ) + else: + return ( + False, + f"Document size ({doc_size:,} chars) below threshold ({threshold:,} chars)", + ) + + +def get_adaptive_agent_config( + use_segmentation: bool, search_server_names: list = None +) -> Dict[str, list]: + """ + Get adaptive agent configuration based on whether to use document segmentation. + + Args: + use_segmentation: Whether to include document-segmentation server + search_server_names: Base search server names (from get_search_server_names) + + Returns: + Dict containing server configurations for different agents + """ + if search_server_names is None: + search_server_names = [] + + # Base configuration + config = { + "concept_analysis": [], + "algorithm_analysis": search_server_names.copy(), + "code_planner": search_server_names.copy(), + } + + # Add document-segmentation server if needed + if use_segmentation: + config["concept_analysis"] = ["document-segmentation"] + if "document-segmentation" not in config["algorithm_analysis"]: + config["algorithm_analysis"].append("document-segmentation") + if "document-segmentation" not in config["code_planner"]: + config["code_planner"].append("document-segmentation") + else: + config["concept_analysis"] = ["filesystem"] + if "filesystem" not in config["algorithm_analysis"]: + config["algorithm_analysis"].append("filesystem") + if "filesystem" not in config["code_planner"]: + config["code_planner"].append("filesystem") + + return config + + +def get_adaptive_prompts(use_segmentation: bool) -> Dict[str, str]: + """ + Get appropriate prompt versions based on segmentation usage. + + Args: + use_segmentation: Whether to use segmented reading prompts + + Returns: + Dict containing prompt configurations + """ + # Import here to avoid circular imports + from prompts.code_prompts import ( + PAPER_CONCEPT_ANALYSIS_PROMPT, + PAPER_ALGORITHM_ANALYSIS_PROMPT, + CODE_PLANNING_PROMPT, + PAPER_CONCEPT_ANALYSIS_PROMPT_TRADITIONAL, + PAPER_ALGORITHM_ANALYSIS_PROMPT_TRADITIONAL, + CODE_PLANNING_PROMPT_TRADITIONAL, + ) + + if use_segmentation: + return { + "concept_analysis": PAPER_CONCEPT_ANALYSIS_PROMPT, + "algorithm_analysis": PAPER_ALGORITHM_ANALYSIS_PROMPT, + "code_planning": CODE_PLANNING_PROMPT, + } + else: + return { + "concept_analysis": PAPER_CONCEPT_ANALYSIS_PROMPT_TRADITIONAL, + "algorithm_analysis": PAPER_ALGORITHM_ANALYSIS_PROMPT_TRADITIONAL, + "code_planning": CODE_PLANNING_PROMPT_TRADITIONAL, + } diff --git a/DeepCode/utils/simple_llm_logger.py b/DeepCode/utils/simple_llm_logger.py new file mode 100644 index 00000000..eabbc0fb --- /dev/null +++ b/DeepCode/utils/simple_llm_logger.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +่ถ…็ฎ€ๅŒ–LLMๅ“ๅบ”ๆ—ฅๅฟ—่ฎฐๅฝ•ๅ™จ +ไธ“ๆณจไบŽ่ฎฐๅฝ•LLMๅ›žๅค็š„ๆ ธๅฟƒๅ†…ๅฎน๏ผŒ้…็ฝฎ็ฎ€ๅ•ๆ˜“็”จ +""" + +import json +import os +import yaml +from datetime import datetime +from pathlib import Path +from typing import Dict, Any + + +class SimpleLLMLogger: + """่ถ…็ฎ€ๅŒ–็š„LLMๅ“ๅบ”ๆ—ฅๅฟ—่ฎฐๅฝ•ๅ™จ""" + + def __init__(self, config_path: str = "mcp_agent.config.yaml"): + """ + ๅˆๅง‹ๅŒ–ๆ—ฅๅฟ—่ฎฐๅฝ•ๅ™จ + + Args: + config_path: ้…็ฝฎๆ–‡ไปถ่ทฏๅพ„ + """ + self.config = self._load_config(config_path) + self.llm_config = self.config.get("llm_logger", {}) + + # ๅฆ‚ๆžœ็ฆ็”จๅˆ™็›ดๆŽฅ่ฟ”ๅ›ž + if not self.llm_config.get("enabled", True): + self.enabled = False + return + + self.enabled = True + self._setup_logger() + + def _load_config(self, config_path: str) -> Dict[str, Any]: + """ๅŠ ่ฝฝ้…็ฝฎๆ–‡ไปถ""" + try: + with open(config_path, "r", encoding="utf-8") as f: + return yaml.safe_load(f) + except Exception as e: + print(f"โš ๏ธ ้…็ฝฎๆ–‡ไปถๅŠ ่ฝฝๅคฑ่ดฅ: {e}๏ผŒไฝฟ็”จ้ป˜่ฎค้…็ฝฎ") + return self._get_default_config() + + def _get_default_config(self) -> Dict[str, Any]: + """่Žทๅ–้ป˜่ฎค้…็ฝฎ""" + return { + "llm_logger": { + "enabled": True, + "output_format": "json", + "log_level": "basic", + "log_directory": "logs/llm_responses", + "filename_pattern": "llm_responses_{timestamp}.jsonl", + "include_models": ["claude-sonnet-4", "gpt-4", "o3-mini"], + "min_response_length": 50, + } + } + + def _setup_logger(self): + """่ฎพ็ฝฎๆ—ฅๅฟ—่ฎฐๅฝ•ๅ™จ""" + log_dir = self.llm_config.get("log_directory", "logs/llm_responses") + + # ๅˆ›ๅปบๆ—ฅๅฟ—็›ฎๅฝ• + Path(log_dir).mkdir(parents=True, exist_ok=True) + + # ็”Ÿๆˆๆ—ฅๅฟ—ๆ–‡ไปถๅ + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename_pattern = self.llm_config.get( + "filename_pattern", "llm_responses_{timestamp}.jsonl" + ) + self.log_file = os.path.join( + log_dir, filename_pattern.format(timestamp=timestamp) + ) + + print(f"๐Ÿ“ LLMๅ“ๅบ”ๆ—ฅๅฟ—: {self.log_file}") + + def log_response(self, content: str, model: str = "", agent: str = "", **kwargs): + """ + ่ฎฐๅฝ•LLMๅ“ๅบ” - ็ฎ€ๅŒ–็‰ˆๆœฌ + + Args: + content: LLMๅ“ๅบ”ๅ†…ๅฎน + model: ๆจกๅž‹ๅ็งฐ + agent: Agentๅ็งฐ + **kwargs: ๅ…ถไป–ๅฏ้€‰ไฟกๆฏ + """ + if not self.enabled: + return + + # ๆฃ€ๆŸฅๆ˜ฏๅฆๅบ”่ฏฅ่ฎฐๅฝ• + if not self._should_log(content, model): + return + + # ๆž„ๅปบๆ—ฅๅฟ—่ฎฐๅฝ• + log_entry = self._build_entry(content, model, agent, kwargs) + + # ๅ†™ๅ…ฅๆ—ฅๅฟ— + self._write_log(log_entry) + + # ๆŽงๅˆถๅฐๆ˜พ็คบ + self._console_log(content, model, agent) + + def _should_log(self, content: str, model: str) -> bool: + """ๆฃ€ๆŸฅๆ˜ฏๅฆๅบ”่ฏฅ่ฎฐๅฝ•""" + # ๆฃ€ๆŸฅ้•ฟๅบฆ + min_length = self.llm_config.get("min_response_length", 50) + if len(content) < min_length: + return False + + # ๆฃ€ๆŸฅๆจกๅž‹ + include_models = self.llm_config.get("include_models", []) + if include_models and not any(m in model for m in include_models): + return False + + return True + + def _build_entry(self, content: str, model: str, agent: str, extra: Dict) -> Dict: + """ๆž„ๅปบๆ—ฅๅฟ—ๆก็›ฎ""" + log_level = self.llm_config.get("log_level", "basic") + + if log_level == "basic": + # ๅŸบ็ก€็บงๅˆซ๏ผšๅช่ฎฐๅฝ•ๆ ธๅฟƒๅ†…ๅฎน + return { + "timestamp": datetime.now().isoformat(), + "content": content, + "model": model, + } + else: + # ่ฏฆ็ป†็บงๅˆซ๏ผšๅŒ…ๅซๆ›ดๅคšไฟกๆฏ + entry = { + "timestamp": datetime.now().isoformat(), + "content": content, + "model": model, + "agent": agent, + } + # ๆทปๅŠ ้ขๅค–ไฟกๆฏ + if "token_usage" in extra: + entry["tokens"] = extra["token_usage"] + if "session_id" in extra: + entry["session"] = extra["session_id"] + return entry + + def _write_log(self, entry: Dict): + """ๅ†™ๅ…ฅๆ—ฅๅฟ—ๆ–‡ไปถ""" + output_format = self.llm_config.get("output_format", "json") + + try: + with open(self.log_file, "a", encoding="utf-8") as f: + if output_format == "json": + f.write(json.dumps(entry, ensure_ascii=False) + "\n") + elif output_format == "text": + timestamp = entry.get("timestamp", "") + model = entry.get("model", "") + content = entry.get("content", "") + f.write(f"[{timestamp}] {model}: {content}\n\n") + elif output_format == "markdown": + timestamp = entry.get("timestamp", "") + model = entry.get("model", "") + content = entry.get("content", "") + f.write(f"**{timestamp}** | {model}\n\n{content}\n\n---\n\n") + except Exception as e: + print(f"โš ๏ธ ๅ†™ๅ…ฅๆ—ฅๅฟ—ๅคฑ่ดฅ: {e}") + + def _console_log(self, content: str, model: str, agent: str): + """ๆŽงๅˆถๅฐ็ฎ€่ฆๆ˜พ็คบ""" + preview = content[:80] + "..." if len(content) > 80 else content + print(f"๐Ÿค– {model} ({agent}): {preview}") + + +# ๅ…จๅฑ€ๅฎžไพ‹ +_global_logger = None + + +def get_llm_logger() -> SimpleLLMLogger: + """่Žทๅ–ๅ…จๅฑ€LLMๆ—ฅๅฟ—่ฎฐๅฝ•ๅ™จๅฎžไพ‹""" + global _global_logger + if _global_logger is None: + _global_logger = SimpleLLMLogger() + return _global_logger + + +def log_llm_response(content: str, model: str = "", agent: str = "", **kwargs): + """ไพฟๆทๅ‡ฝๆ•ฐ๏ผš่ฎฐๅฝ•LLMๅ“ๅบ”""" + logger = get_llm_logger() + logger.log_response(content, model, agent, **kwargs) + + +# ็คบไพ‹ไฝฟ็”จ +if __name__ == "__main__": + # ๆต‹่ฏ•ๆ—ฅๅฟ—่ฎฐๅฝ• + log_llm_response( + content="่ฟ™ๆ˜ฏไธ€ไธชๆต‹่ฏ•็š„LLMๅ“ๅบ”ๅ†…ๅฎน๏ผŒ็”จไบŽ้ชŒ่ฏ็ฎ€ๅŒ–ๆ—ฅๅฟ—่ฎฐๅฝ•ๅ™จ็š„ๅŠŸ่ƒฝๆ˜ฏๅฆๆญฃๅธธๅทฅไฝœใ€‚", + model="claude-sonnet-4-20250514", + agent="TestAgent", + ) + + print("โœ… ็ฎ€ๅŒ–LLMๆ—ฅๅฟ—ๆต‹่ฏ•ๅฎŒๆˆ") diff --git a/DeepCode/workflows/__init__.py b/DeepCode/workflows/__init__.py new file mode 100644 index 00000000..a833399a --- /dev/null +++ b/DeepCode/workflows/__init__.py @@ -0,0 +1,31 @@ +""" +Intelligent Agent Orchestration Workflows for Research-to-Code Automation. + +This package provides advanced AI-driven workflow orchestration capabilities +for automated research analysis and code implementation synthesis. +""" + +from .agent_orchestration_engine import ( + run_research_analyzer, + run_resource_processor, + run_code_analyzer, + github_repo_download, + paper_reference_analyzer, + execute_multi_agent_research_pipeline, + paper_code_preparation, # Deprecated, for backward compatibility +) + +from .code_implementation_workflow import CodeImplementationWorkflow + +__all__ = [ + # Initial workflows + "run_research_analyzer", + "run_resource_processor", + "run_code_analyzer", + "github_repo_download", + "paper_reference_analyzer", + "execute_multi_agent_research_pipeline", # Main multi-agent pipeline function + "paper_code_preparation", # Deprecated, for backward compatibility + # Code implementation workflows + "CodeImplementationWorkflow", +] diff --git a/DeepCode/workflows/agent_orchestration_engine.py b/DeepCode/workflows/agent_orchestration_engine.py new file mode 100644 index 00000000..68af299b --- /dev/null +++ b/DeepCode/workflows/agent_orchestration_engine.py @@ -0,0 +1,2034 @@ +""" +Intelligent Agent Orchestration Engine for Research-to-Code Automation + +This module serves as the core orchestration engine that coordinates multiple specialized +AI agents to automate the complete research-to-code transformation pipeline: + +1. Research Analysis Agent - Intelligent content processing and extraction +2. Workspace Infrastructure Agent - Automated environment synthesis +3. Code Architecture Agent - AI-driven design and planning +4. Reference Intelligence Agent - Automated knowledge discovery +5. Repository Acquisition Agent - Intelligent code repository management +6. Codebase Intelligence Agent - Advanced relationship analysis +7. Code Implementation Agent - AI-powered code synthesis + +Core Features: +- Multi-agent coordination with intelligent task distribution +- Local environment automation for seamless deployment +- Real-time progress monitoring with comprehensive error handling +- Adaptive workflow optimization based on processing requirements +- Advanced intelligence analysis with configurable performance modes + +Architecture: +- Async/await based high-performance agent coordination +- Modular agent design with specialized role separation +- Intelligent resource management and optimization +- Comprehensive logging and monitoring infrastructure +""" + +import asyncio +import json +import os +import re +import yaml +from typing import Any, Callable, Dict, List, Optional, Tuple + +# MCP Agent imports +from mcp_agent.agents.agent import Agent +from mcp_agent.workflows.llm.augmented_llm import RequestParams +from mcp_agent.workflows.parallel.parallel_llm import ParallelLLM + +# Local imports +from prompts.code_prompts import ( + PAPER_INPUT_ANALYZER_PROMPT, + PAPER_DOWNLOADER_PROMPT, + PAPER_REFERENCE_ANALYZER_PROMPT, + CHAT_AGENT_PLANNING_PROMPT, +) +from utils.file_processor import FileProcessor +from workflows.code_implementation_workflow import CodeImplementationWorkflow +from tools.pdf_downloader import move_file_to, download_file_to +from workflows.code_implementation_workflow_index import ( + CodeImplementationWorkflowWithIndex, +) +from utils.llm_utils import ( + get_preferred_llm_class, + should_use_document_segmentation, + get_adaptive_agent_config, + get_adaptive_prompts, + get_token_limits, +) +from workflows.agents.document_segmentation_agent import prepare_document_segments +from workflows.agents.requirement_analysis_agent import RequirementAnalysisAgent + +# Environment configuration +os.environ["PYTHONDONTWRITEBYTECODE"] = "1" # Prevent .pyc file generation + + +def _assess_output_completeness(text: str) -> float: + """ + ็ฒพๅ‡†่ฏ„ไผฐYAMLๆ ผๅผๅฎž็Žฐ่ฎกๅˆ’็š„ๅฎŒๆ•ดๆ€ง + + ๅŸบไบŽCODE_PLANNING_PROMPT_TRADITIONAL็š„ๅฎž้™…่ฆๆฑ‚๏ผš + 1. ๆฃ€ๆŸฅ5ไธชๅฟ…้œ€็š„YAML sectionsๆ˜ฏๅฆ้ƒฝๅญ˜ๅœจ + 2. ้ชŒ่ฏYAML็ป“ๆž„็š„ๅฎŒๆ•ดๆ€ง๏ผˆๅผ€ๅง‹ๅ’Œ็ป“ๆŸๆ ‡่ฎฐ๏ผ‰ + 3. ๆฃ€ๆŸฅๆœ€ๅŽไธ€่กŒๆ˜ฏๅฆ่ขซๆˆชๆ–ญ + 4. ้ชŒ่ฏๆœ€ๅฐๅˆ็†้•ฟๅบฆ + + Returns: + float: ๅฎŒๆ•ดๆ€งๅˆ†ๆ•ฐ (0.0-1.0)๏ผŒ่ถŠ้ซ˜่กจ็คบ่ถŠๅฎŒๆ•ด + """ + if not text or len(text.strip()) < 500: + return 0.0 + + score = 0.0 + text_lower = text.lower() + + # 1. ๆฃ€ๆŸฅ5ไธชๅฟ…้œ€็š„YAML sections (ๆƒ้‡: 0.5 - ๆœ€้‡่ฆ) + # ่ฟ™ๆ˜ฏpromptๆ˜Ž็กฎ่ฆๆฑ‚็š„5ไธชsections + required_sections = [ + "file_structure:", + "implementation_components:", + "validation_approach:", + "environment_setup:", + "implementation_strategy:", + ] + + sections_found = sum(1 for section in required_sections if section in text_lower) + section_score = sections_found / len(required_sections) + score += section_score * 0.5 + + print(f" ๐Ÿ“‹ Required sections: {sections_found}/{len(required_sections)}") + + # 2. ๆฃ€ๆŸฅYAML็ป“ๆž„ๅฎŒๆ•ดๆ€ง (ๆƒ้‡: 0.2) + has_yaml_start = any( + marker in text + for marker in ["```yaml", "complete_reproduction_plan:", "paper_info:"] + ) + has_yaml_end = any( + marker in text[-500:] + for marker in ["```", "implementation_strategy:", "validation_approach:"] + ) + + if has_yaml_start and has_yaml_end: + score += 0.2 + elif has_yaml_start: + score += 0.1 + + # 3. ๆฃ€ๆŸฅๆœ€ๅŽไธ€่กŒๅฎŒๆ•ดๆ€ง (ๆƒ้‡: 0.15) + lines = text.strip().split("\n") + if lines: + last_line = lines[-1].strip() + # YAML็š„ๆœ€ๅŽไธ€่กŒ้€šๅธธๆ˜ฏ็ผฉ่ฟ›็š„ๅ†…ๅฎน่กŒๆˆ–็ป“ๆŸๆ ‡่ฎฐ + if ( + last_line.endswith(("```", ".", ":", "]", "}")) + or last_line.startswith(("-", "*", " ")) # YAMLๅˆ—่กจ้กนๆˆ–็ผฉ่ฟ›ๅ†…ๅฎน + or ( + len(last_line) < 100 and not last_line.endswith(",") + ) # ็Ÿญ่กŒไธ”ไธๆ˜ฏ่ขซๆˆชๆ–ญ็š„ + ): + score += 0.15 + else: + # ้•ฟ่กŒไธ”ๆฒกๆœ‰ๅˆ้€‚็š„็ป“ๅฐพ๏ผŒๅพˆๅฏ่ƒฝ่ขซๆˆชๆ–ญ + print(f" โš ๏ธ Last line suspicious: '{last_line[-50:]}'") + + # 4. ๆฃ€ๆŸฅๅˆ็†็š„ๆœ€ๅฐ้•ฟๅบฆ (ๆƒ้‡: 0.15) + # ไธ€ไธชๅฎŒๆ•ด็š„5-section่ฎกๅˆ’ๅบ”่ฏฅ่‡ณๅฐ‘8000ๅญ—็ฌฆ + length = len(text) + if length >= 10000: + score += 0.15 + elif length >= 5000: + score += 0.10 + elif length >= 2000: + score += 0.05 + + print(f" ๐Ÿ“ Content length: {length} chars") + + return min(score, 1.0) + + +def _adjust_params_for_retry( + params: RequestParams, retry_count: int, config_path: str = "mcp_agent.config.yaml" +) -> RequestParams: + """ + Tokenๅ‡ๅฐ‘็ญ–็•ฅไปฅ้€‚ๅบ”ๆจกๅž‹context้™ๅˆถ + + ็ญ–็•ฅ่ฏดๆ˜Ž๏ผˆ้’ˆๅฏนqwen/qwen-max็š„32768 token้™ๅˆถ๏ผ‰๏ผš + - ็ฌฌ1ๆฌก้‡่ฏ•๏ผšREDUCEๅˆฐretry_max_tokens๏ผˆไปŽconfig่ฏปๅ–๏ผŒ้ป˜่ฎค15000๏ผ‰ + - ็ฌฌ2ๆฌก้‡่ฏ•๏ผšREDUCEๅˆฐretry_max_tokens็š„80% + - ็ฌฌ3ๆฌก้‡่ฏ•๏ผšREDUCEๅˆฐretry_max_tokens็š„60% + - ้™ไฝŽtemperatureๆ้ซ˜็จณๅฎšๆ€งๅ’Œๅฏ้ข„ๆต‹ๆ€ง + + ไธบไป€ไนˆ่ฆREDUCE่€Œไธๆ˜ฏINCREASE๏ผŸ + - qwen/qwen-maxๆœ€ๅคงcontext = 32768 tokens (input + output ๆ€ปๅ’Œ) + - ๅฝ“้‡ๅˆฐ "maximum context length exceeded" ้”™่ฏฏๆ—ถ๏ผŒ่ฏดๆ˜Ž input + requested_output > 32768 + - INCREASING max_tokensๅชไผš่ฎฉ้—ฎ้ข˜ๆ›ดไธฅ้‡๏ผ + - ๆญฃ็กฎๅšๆณ•๏ผšDECREASE output tokens๏ผŒไธบๆ›ดๅคšinput็•™ๅ‡บ็ฉบ้—ด + - ๆจกๅž‹ๅฏไปฅ็”จๆ›ด็ฎ€ๆด็š„่พ“ๅ‡บ่กจ่พพ็›ธๅŒๅ†…ๅฎน + """ + # ไปŽ้…็ฝฎๆ–‡ไปถ่ฏปๅ–retry token limit + _, retry_max_tokens = get_token_limits(config_path) + + # Tokenๅ‡ๅฐ‘็ญ–็•ฅ - ไธบinput่…พๅ‡บๆ›ดๅคš็ฉบ้—ด + if retry_count == 0: + # ็ฌฌไธ€ๆฌก้‡่ฏ•๏ผšไฝฟ็”จ้…็ฝฎ็š„retry_max_tokens + new_max_tokens = retry_max_tokens + elif retry_count == 1: + # ็ฌฌไบŒๆฌก้‡่ฏ•๏ผšๅ‡ๅฐ‘ๅˆฐretry_max_tokens็š„80% + new_max_tokens = int(retry_max_tokens * 0.9) + else: + # ็ฌฌไธ‰ๆฌกๅŠไปฅไธŠ๏ผšๅ‡ๅฐ‘ๅˆฐretry_max_tokens็š„60% + new_max_tokens = int(retry_max_tokens * 0.8) + + # ้š็€้‡่ฏ•ๆฌกๆ•ฐๅขžๅŠ ๏ผŒ้™ไฝŽtemperatureไปฅ่Žทๅพ—ๆ›ดไธ€่‡ดใ€ๆ›ดๅฏ้ข„ๆต‹็š„่พ“ๅ‡บ + new_temperature = max(params.temperature - (retry_count * 0.15), 0.05) + + print(f"๐Ÿ”ง Adjusting parameters for retry {retry_count + 1}:") + print(f" Token limit: {params.maxTokens} โ†’ {new_max_tokens}") + print(f" Temperature: {params.temperature:.2f} โ†’ {new_temperature:.2f}") + print( + " ๐Ÿ’ก Strategy: REDUCE output tokens to fit within model's total context limit" + ) + + # return RequestParams( + # maxTokens=new_max_tokens, # ๆณจๆ„๏ผšไฝฟ็”จ camelCase + # temperature=new_temperature, + # ) + return new_max_tokens, new_temperature + + +async def execute_requirement_analysis_workflow( + user_input: str, + analysis_mode: str, + user_answers: Optional[Dict[str, str]] = None, + logger=None, + progress_callback: Optional[Callable[[int, str], None]] = None, +) -> Dict[str, Any]: + """ + Lightweight orchestrator to run requirement-analysis-specific flows. + """ + + normalized_input = (user_input or "").strip() + if not normalized_input: + return { + "status": "error", + "error": "User requirement input cannot be empty.", + } + + user_answers = user_answers or {} + + try: + async with RequirementAnalysisAgent(logger=logger) as agent: + if progress_callback: + progress_callback(5, "๐Ÿค– Initializing requirement analysis agent...") + + if analysis_mode == "generate_questions": + questions = await agent.generate_guiding_questions(normalized_input) + if progress_callback: + progress_callback(100, "๐Ÿง  Guiding questions generated.") + return { + "status": "success", + "result": json.dumps(questions, ensure_ascii=False), + } + + if analysis_mode == "summarize_requirements": + summary = await agent.summarize_detailed_requirements( + normalized_input, user_answers + ) + if progress_callback: + progress_callback(100, "๐Ÿ“„ Requirement document created.") + return {"status": "success", "result": summary} + + raise ValueError(f"Unsupported analysis_mode: {analysis_mode}") + + except Exception as exc: + message = str(exc) + if logger: + try: + logger.error("Requirement analysis workflow failed: %s", message) + except Exception: + pass + return {"status": "error", "error": message} + + +def get_default_search_server(config_path: str = "mcp_agent.config.yaml"): + """ + Get the default search server from configuration. + + Args: + config_path: Path to the main configuration file + + Returns: + str: The default search server name ("brave" or "bocha-mcp") + """ + try: + if os.path.exists(config_path): + with open(config_path, "r", encoding="utf-8") as f: + config = yaml.safe_load(f) + + default_server = config.get("default_search_server", "brave") + print(f"๐Ÿ” Using search server: {default_server}") + return default_server + else: + print(f"โš ๏ธ Config file {config_path} not found, using default: brave") + return "brave" + except Exception as e: + print(f"โš ๏ธ Error reading config file {config_path}: {e}") + print("๐Ÿ” Falling back to default search server: brave") + return "brave" + + +def get_search_server_names( + additional_servers: Optional[List[str]] = None, +) -> List[str]: + """ + Get server names list with the configured default search server. + + Args: + additional_servers: Optional list of additional servers to include + + Returns: + List[str]: List of server names including the default search server + """ + default_search = get_default_search_server() + server_names = [default_search] + + if additional_servers: + # Add additional servers, avoiding duplicates + for server in additional_servers: + if server not in server_names: + server_names.append(server) + + return server_names + + +def extract_clean_json(llm_output: str) -> str: + """ + Extract clean JSON from LLM output, removing all extra text and formatting. + + Args: + llm_output: Raw LLM output + + Returns: + str: Clean JSON string + """ + try: + # Try to parse the entire output as JSON first + json.loads(llm_output.strip()) + return llm_output.strip() + except json.JSONDecodeError: + pass + + # Remove markdown code blocks + if "```json" in llm_output: + pattern = r"```json\s*(.*?)\s*```" + match = re.search(pattern, llm_output, re.DOTALL) + if match: + json_text = match.group(1).strip() + try: + json.loads(json_text) + return json_text + except json.JSONDecodeError: + pass + + # Find JSON object starting with { + lines = llm_output.split("\n") + json_lines = [] + in_json = False + brace_count = 0 + + for line in lines: + stripped = line.strip() + if not in_json and stripped.startswith("{"): + in_json = True + json_lines = [line] + brace_count = stripped.count("{") - stripped.count("}") + elif in_json: + json_lines.append(line) + brace_count += stripped.count("{") - stripped.count("}") + if brace_count == 0: + break + + if json_lines: + json_text = "\n".join(json_lines).strip() + try: + json.loads(json_text) + return json_text + except json.JSONDecodeError: + pass + + # Last attempt: use regex to find JSON + pattern = r"\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}" + matches = re.findall(pattern, llm_output, re.DOTALL) + for match in matches: + try: + json.loads(match) + return match + except json.JSONDecodeError: + continue + + # If all methods fail, return original output + return llm_output + + +async def run_research_analyzer(prompt_text: str, logger) -> str: + """ + Run the research analysis workflow using ResearchAnalyzerAgent. + + Args: + prompt_text: Input prompt text containing research information + logger: Logger instance for logging information + + Returns: + str: Analysis result from the agent + """ + try: + # Log input information for debugging + print("๐Ÿ“Š Starting research analysis...") + print(f"Input prompt length: {len(prompt_text) if prompt_text else 0}") + print(f"Input preview: {prompt_text[:200] if prompt_text else 'None'}...") + + if not prompt_text or prompt_text.strip() == "": + raise ValueError( + "Empty or None prompt_text provided to run_research_analyzer" + ) + + analyzer_agent = Agent( + name="ResearchAnalyzerAgent", + instruction=PAPER_INPUT_ANALYZER_PROMPT, + server_names=get_search_server_names(), + ) + + async with analyzer_agent: + print("analyzer: Connected to server, calling list_tools...") + try: + tools = await analyzer_agent.list_tools() + print( + "Tools available:", + tools.model_dump() if hasattr(tools, "model_dump") else str(tools), + ) + except Exception as e: + print(f"Failed to list tools: {e}") + + try: + analyzer = await analyzer_agent.attach_llm(get_preferred_llm_class()) + print("โœ… LLM attached successfully") + except Exception as e: + print(f"โŒ Failed to attach LLM: {e}") + raise + + # Set higher token output for research analysis + analysis_params = RequestParams( + maxTokens=6144, # ไฝฟ็”จ camelCase + temperature=0.3, + ) + + print( + f"๐Ÿ”„ Making LLM request with params: maxTokens={analysis_params.maxTokens}, temperature={analysis_params.temperature}" + ) + + try: + raw_result = await analyzer.generate_str( + message=prompt_text, request_params=analysis_params + ) + + print("โœ… LLM request completed") + print(f"Raw result type: {type(raw_result)}") + print(f"Raw result length: {len(raw_result) if raw_result else 0}") + + if not raw_result: + print("โŒ CRITICAL: raw_result is empty or None!") + print("This could indicate:") + print("1. LLM API call failed silently") + print("2. API rate limiting or quota exceeded") + print("3. Network connectivity issues") + print("4. MCP server communication problems") + raise ValueError("LLM returned empty result") + + except Exception as e: + print(f"โŒ LLM generation failed: {e}") + print(f"Exception type: {type(e)}") + raise + + # Clean LLM output to ensure only pure JSON is returned + try: + clean_result = extract_clean_json(raw_result) + print(f"Raw LLM output: {raw_result}") + print(f"Cleaned JSON output: {clean_result}") + + # Log to SimpleLLMLogger + if hasattr(logger, "log_response"): + logger.log_response( + clean_result, + model="ResearchAnalyzer", + agent="ResearchAnalyzerAgent", + ) + + if not clean_result or clean_result.strip() == "": + print("โŒ CRITICAL: clean_result is empty after JSON extraction!") + print(f"Original raw_result was: {raw_result}") + raise ValueError("JSON extraction resulted in empty output") + + return clean_result + + except Exception as e: + print(f"โŒ JSON extraction failed: {e}") + print(f"Raw result was: {raw_result}") + raise + + except Exception as e: + print(f"โŒ run_research_analyzer failed: {e}") + print(f"Exception details: {type(e).__name__}: {str(e)}") + raise + + +async def run_resource_processor(analysis_result: str, logger) -> str: + """ + Run the resource processing workflow - deterministic file operations without LLM. + + This function handles file downloading/moving using direct logic rather than LLM, + since the paper directory structure and ID are pre-computed and deterministic. + + Args: + analysis_result: Result from the research analyzer (contains file path/URL) + logger: Logger instance for logging information + + Returns: + str: Processing result with paper directory path + """ + # Pre-compute paper ID - deterministic, no LLM needed + papers_dir = "./deepcode_lab/papers" + os.makedirs(papers_dir, exist_ok=True) + existing_ids = [ + int(d) + for d in os.listdir(papers_dir) + if os.path.isdir(os.path.join(papers_dir, d)) and d.isdigit() + ] + next_id = max(existing_ids) + 1 if existing_ids else 1 + paper_dir = os.path.join(papers_dir, str(next_id)) + os.makedirs(paper_dir, exist_ok=True) + + logger.info(f"๐Ÿ“‹ Paper ID: {next_id}") + logger.info(f"๐Ÿ“‚ Paper directory: {paper_dir}") + + # Extract file path/URL from analysis_result - simple parsing, no LLM needed + # The analysis_result should contain the path/URL identified by the analyzer + try: + # Parse the analysis result to extract path + analysis_data = json.loads(analysis_result) + source_path = analysis_data.get("path") or analysis_data.get("input_path") + input_type = analysis_data.get("input_type", "unknown") + + logger.info(f"๐Ÿ“ฅ Processing {input_type}: {source_path}") + + # Try direct function calls first - no LLM needed for deterministic operations + direct_call_success = False + operation_result = None + + # 1. Handle local file - direct copy + if input_type == "file" and source_path and os.path.exists(source_path): + logger.info(f"๐Ÿ“„ Direct file copy: {source_path} -> {paper_dir}") + try: + operation_result = await move_file_to( + source=source_path, destination=paper_dir, filename=f"{next_id}.pdf" + ) + # Check if operation succeeded + if ( + "[SUCCESS]" in operation_result + and "[ERROR]" not in operation_result + ): + direct_call_success = True + logger.info(f"โœ… Direct file copy succeeded:\n{operation_result}") + else: + logger.warning(f"โš ๏ธ Direct file copy had issues: {operation_result}") + except Exception as e: + logger.warning(f"โš ๏ธ Direct file copy failed: {e}") + + # 2. Handle URL - direct download + elif input_type == "url" and source_path: + logger.info(f"๐ŸŒ Direct URL download: {source_path} -> {paper_dir}") + try: + operation_result = await download_file_to( + url=source_path, + destination=paper_dir, + filename=f"{next_id}.pdf", # Default to PDF, conversion will handle it + ) + # Check if operation succeeded + if ( + "[SUCCESS]" in operation_result + and "[ERROR]" not in operation_result + ): + direct_call_success = True + logger.info(f"โœ… Direct download succeeded:\n{operation_result}") + else: + logger.warning(f"โš ๏ธ Direct download had issues: {operation_result}") + except Exception as e: + logger.warning(f"โš ๏ธ Direct download failed: {e}") + + # 3. If direct call succeeded, format result + if direct_call_success: + dest_path = os.path.join(paper_dir, f"{next_id}.md") + result = json.dumps( + { + "status": "success", + "paper_id": next_id, + "paper_dir": paper_dir, + "file_path": dest_path, + "message": f"File successfully processed to {paper_dir}", + "operation_details": operation_result, + } + ) + else: + # 4. Fallback to LLM agent if direct call failed or unsupported type + logger.info( + f"๐Ÿค– Falling back to LLM agent for: {input_type} - {source_path}" + ) + processor_agent = Agent( + name="ResourceProcessorAgent", + instruction=PAPER_DOWNLOADER_PROMPT, + server_names=["file-downloader"], + ) + + async with processor_agent: + processor = await processor_agent.attach_llm(get_preferred_llm_class()) + processor_params = RequestParams( + maxTokens=4096, + temperature=0.2, + tool_filter={ + "file-downloader": {"download_file_to", "move_file_to"} + }, + ) + + # Provide context about what failed if available + context = ( + f"\nPrevious attempt result: {operation_result}" + if operation_result + else "" + ) + message = f"""Download/move the file to paper directory: {paper_dir} +Source: {source_path} +Input Type: {input_type} +Paper ID: {next_id} +Target filename: {next_id}.md (after conversion){context} + +Use the appropriate tool to complete this task.""" + + result = await processor.generate_str( + message=message, request_params=processor_params + ) + + return result + + except (json.JSONDecodeError, KeyError, Exception) as e: + logger.error(f"โŒ Error processing resource: {e}") + # Fallback - return paper directory for manual processing + return json.dumps( + { + "status": "partial", + "paper_id": next_id, + "paper_dir": paper_dir, + "message": f"Paper directory created at {paper_dir}, manual file placement may be needed", + } + ) + + +async def run_code_analyzer( + paper_dir: str, logger, use_segmentation: bool = True +) -> str: + """ + Run the adaptive code analysis workflow with optimized file reading. + + This function minimizes LLM tool calls by: + 1. Reading paper file directly (deterministic, no LLM needed) + 2. Passing paper content directly to agents + 3. LLM only used for analysis and search decisions + + Orchestrates three specialized agents: + - ConceptAnalysisAgent: Analyzes system architecture and conceptual framework + - AlgorithmAnalysisAgent: Extracts algorithms, formulas, and technical details + - CodePlannerAgent: Integrates outputs into a comprehensive implementation plan + + Args: + paper_dir: Directory path containing the research paper and related resources + logger: Logger instance for logging information + use_segmentation: Whether to use document segmentation capabilities + + Returns: + str: Comprehensive analysis result from the coordinated agents + """ + print( + f"๐Ÿ“Š Code analysis mode: {'Segmented' if use_segmentation else 'Traditional'}" + ) + print(" ๐Ÿ”ง Optimized workflow: Direct file reading, LLM only for analysis") + + # STEP 1: Read paper file directly - no LLM needed for deterministic file operations + paper_content = None + paper_file_path = None + + try: + # Find .md file in paper directory - simple file system operation + for filename in os.listdir(paper_dir): + if filename.endswith(".md"): + paper_file_path = os.path.join(paper_dir, filename) + with open(paper_file_path, "r", encoding="utf-8") as f: + paper_content = f.read() + logger.info( + f"๐Ÿ“„ Paper file loaded: {paper_file_path} ({len(paper_content)} chars)" + ) + break + + if not paper_content: + logger.warning( + f"โš ๏ธ No .md file found in {paper_dir}, agents will search for it" + ) + except Exception as e: + logger.warning(f"โš ๏ธ Error reading paper file: {e}, agents will search for it") + + # STEP 2: Configure agents with minimal tool access + search_server_names = get_search_server_names() + agent_config = get_adaptive_agent_config(use_segmentation, search_server_names) + prompts = get_adaptive_prompts(use_segmentation) + + if paper_content: + agent_config = { + "concept_analysis": [], + "algorithm_analysis": ["brave"], + "code_planner": [ + "brave" + ], # Empty list instead of None - code planner doesn't need tools when paper content is provided + } + # agent_config = { + # "concept_analysis": [], + # "algorithm_analysis": [], + # "code_planner": [], # Empty list instead of None - code planner doesn't need tools when paper content is provided + # } + else: + agent_config = { + "concept_analysis": ["filesystem"], + "algorithm_analysis": ["brave", "filesystem"], + "code_planner": ["brave", "filesystem"], + } + + print(f" Agent configurations: {agent_config}") + + concept_analysis_agent = Agent( + name="ConceptAnalysisAgent", + instruction=prompts["concept_analysis"], + server_names=agent_config["concept_analysis"], + ) + algorithm_analysis_agent = Agent( + name="AlgorithmAnalysisAgent", + instruction=prompts["algorithm_analysis"], + server_names=agent_config["algorithm_analysis"], + ) + code_planner_agent = Agent( + name="CodePlannerAgent", + instruction=prompts["code_planning"], + server_names=agent_config["code_planner"], + ) + + code_aggregator_agent = ParallelLLM( + fan_in_agent=code_planner_agent, + fan_out_agents=[concept_analysis_agent, algorithm_analysis_agent], + llm_factory=get_preferred_llm_class(), + ) + + base_max_tokens, _ = get_token_limits() + + # STEP 3: Configure parameters - minimal tool filter since paper content is provided + if use_segmentation: + max_tokens_limit = base_max_tokens + temperature = 0.2 + max_iterations = 5 + print( + f"๐Ÿง  Using SEGMENTED mode: max_tokens={base_max_tokens} for complete YAML output" + ) + + # Segmentation mode: Only use segmentation tools if needed (paper content already provided) + tool_filter = { + "document-segmentation": {"read_document_segments", "get_document_overview"} + if not paper_content + else set(), # Empty if paper already loaded + # "brave" not in filter = all brave tools available for searching + } + else: + max_tokens_limit = base_max_tokens + temperature = 0.3 + max_iterations = 2 + print( + f"๐Ÿง  Using TRADITIONAL mode: max_tokens={base_max_tokens} for complete YAML output" + ) + + # Traditional mode: No filesystem tools needed (paper content already provided) + if paper_content: + tool_filter = { + # Only brave search available - no filesystem tools needed + } + else: + tool_filter = { + "filesystem": { + "read_text_file", + "list_directory", + } + } + + enhanced_params = RequestParams( + maxTokens=max_tokens_limit, + temperature=temperature, + max_iterations=max_iterations, + tool_filter=tool_filter + if tool_filter + else None, # None = all tools, empty dict = no filtering + ) + + # STEP 4: Construct message with paper content directly included + if paper_content: + # Paper content provided directly - LLM only needs to analyze, not read files + message = f"""Analyze the research paper provided below. The paper file has been pre-loaded for you. + +=== PAPER CONTENT START === +{paper_content} +=== PAPER CONTENT END === + +Based on this paper, generate a comprehensive code reproduction plan that includes: + +1. Complete system architecture and component breakdown +2. All algorithms, formulas, and implementation details +3. Detailed file structure and implementation roadmap + +You may use web search (brave_web_search) if you need clarification on algorithms, methods, or concepts. + +The goal is to create a reproduction plan detailed enough for independent implementation.""" + else: + # Fallback: paper not found, agents will need to find it + message = f"""Analyze the research paper in directory: {paper_dir} + +Please locate and analyze the markdown (.md) file containing the research paper. Based on your analysis, generate a comprehensive code reproduction plan that includes: + +1. Complete system architecture and component breakdown +2. All algorithms, formulas, and implementation details +3. Detailed file structure and implementation roadmap + +The goal is to create a reproduction plan detailed enough for independent implementation.""" + + max_retries = 3 + retry_count = 0 + + while retry_count < max_retries: + try: + print( + f"๐Ÿš€ Attempting code analysis (attempt {retry_count + 1}/{max_retries})" + ) + result = await code_aggregator_agent.generate_str( + message=message, request_params=enhanced_params + ) + + print(f"๐Ÿ” Code analysis result:\n{result}") + + completeness_score = _assess_output_completeness( + result + ) # need to add file structure val + print(f"๐Ÿ“Š Output completeness score: {completeness_score:.2f}/1.0") + + if completeness_score >= 0.8: + print( + f"โœ… Code analysis completed successfully (length: {len(result)} chars)" + ) + return result + else: + print( + f"โš ๏ธ Output appears truncated (score: {completeness_score:.2f}), retrying with enhanced parameters..." + ) + new_max_tokens, new_temperature = _adjust_params_for_retry( + enhanced_params, retry_count + ) + enhanced_params = RequestParams( + maxTokens=new_max_tokens, + temperature=new_temperature, + max_iterations=max_iterations, + tool_filter=tool_filter + if tool_filter + else None, # None = all tools, empty dict = no filtering + ) + retry_count += 1 + + except Exception as e: + print(f"โŒ Error in code analysis attempt {retry_count + 1}: {e}") + retry_count += 1 + if retry_count >= max_retries: + raise + + print(f"โš ๏ธ Returning potentially incomplete result after {max_retries} attempts") + return result + + +async def github_repo_download(search_result: str, paper_dir: str, logger) -> str: + """ + Download GitHub repositories based on search results. + + Args: + search_result: Result from GitHub repository search + paper_dir: Directory where the paper and its code will be stored + logger: Logger instance for logging information + + Returns: + str: Download result + """ + github_download_agent = Agent( + name="GithubDownloadAgent", + instruction="Download github repo to the directory {paper_dir}/code_base".format( + paper_dir=paper_dir + ), + server_names=["filesystem", "github-downloader"], + ) + + async with github_download_agent: + print("GitHub downloader: Downloading repositories...") + downloader = await github_download_agent.attach_llm(get_preferred_llm_class()) + + # Set higher token output for GitHub download + github_params = RequestParams( + maxTokens=4096, # ไฝฟ็”จ camelCase + temperature=0.1, + ) + + return await downloader.generate_str( + message=search_result, request_params=github_params + ) + + +async def paper_reference_analyzer(paper_dir: str, logger) -> str: + """ + Run the paper reference analysis and GitHub repository workflow. + + Args: + analysis_result: Result from the paper analyzer + logger: Logger instance for logging information + + Returns: + str: Reference analysis result + """ + reference_analysis_agent = Agent( + name="ReferenceAnalysisAgent", + instruction=PAPER_REFERENCE_ANALYZER_PROMPT, + server_names=["filesystem", "fetch"], + ) + message = f"""Analyze the research paper in directory: {paper_dir} + +Please locate and analyze the markdown (.md) file containing the research paper. **Focus specifically on the References/Bibliography section** to identify and analyze the 5 most relevant references that have GitHub repositories. + +Goal: Find the most valuable GitHub repositories from the paper's reference list for code implementation reference.""" + + async with reference_analysis_agent: + print("Reference analyzer: Connected to server, analyzing references...") + analyzer = await reference_analysis_agent.attach_llm(get_preferred_llm_class()) + + # Filter tools to only essential ones for reference analysis + reference_params = RequestParams( + maxTokens=4096, + temperature=0.2, + tool_filter={ + "filesystem": {"read_text_file", "list_directory"}, + "fetch": {"fetch"}, + }, + ) + + reference_result = await analyzer.generate_str( + message=message, request_params=reference_params + ) + return reference_result + + +async def _process_input_source(input_source: str, logger) -> str: + """ + Process and validate input source (file path or URL). + + Args: + input_source: Input source (file path or analysis result) + logger: Logger instance + + Returns: + str: Processed input source + """ + if input_source.startswith("file://"): + file_path = input_source[7:] + if os.name == "nt" and file_path.startswith("/"): + file_path = file_path.lstrip("/") + return file_path + return input_source + + +async def orchestrate_research_analysis_agent( + input_source: str, logger, progress_callback: Optional[Callable] = None +) -> Tuple[str, str]: + """ + Orchestrate intelligent research analysis and resource processing automation. + + This agent coordinates multiple AI components to analyze research content + and process associated resources with automated workflow management. + + Args: + input_source: Research input source for analysis + logger: Logger instance for process tracking + progress_callback: Progress callback function for workflow monitoring + + Returns: + tuple: (analysis_result, resource_processing_result) + """ + # Step 1: Research Analysis + if progress_callback: + progress_callback( + 10, "๐Ÿ“Š Analyzing research content and extracting key information..." + ) + analysis_result = await run_research_analyzer(input_source, logger) + + # Add brief pause for system stability + await asyncio.sleep(5) + + # Step 2: Download Processing + if progress_callback: + progress_callback( + 25, "๐Ÿ“ฅ Processing downloads and preparing document structure..." + ) + download_result = await run_resource_processor(analysis_result, logger) + print("download result:", download_result) + + return analysis_result, download_result + + +async def synthesize_workspace_infrastructure_agent( + download_result: str, logger, workspace_dir: Optional[str] = None +) -> Dict[str, str]: + """ + Synthesize intelligent research workspace infrastructure with automated structure generation. + + This agent autonomously creates and configures the optimal workspace architecture + for research project implementation with AI-driven path optimization. + + Args: + download_result: Resource processing result from analysis agent + logger: Logger instance for infrastructure tracking + workspace_dir: Optional workspace directory path for environment customization + + Returns: + dict: Comprehensive workspace infrastructure metadata + """ + # Parse download result to get file information + result = await FileProcessor.process_file_input( + download_result, base_dir=workspace_dir + ) + paper_dir = result["paper_dir"] + + # Log workspace infrastructure synthesis + print("๐Ÿ—๏ธ Intelligent workspace infrastructure synthesized:") + print(f" Base workspace environment: {workspace_dir or 'auto-detected'}") + print(f" Research workspace: {paper_dir}") + print(" AI-driven path optimization: active") + + return { + "paper_dir": paper_dir, + "standardized_text": result["standardized_text"], + "reference_path": os.path.join(paper_dir, "reference.txt"), + "initial_plan_path": os.path.join(paper_dir, "initial_plan.txt"), + "download_path": os.path.join(paper_dir, "github_download.txt"), + "index_report_path": os.path.join(paper_dir, "codebase_index_report.txt"), + "implementation_report_path": os.path.join( + paper_dir, "code_implementation_report.txt" + ), + "workspace_dir": workspace_dir, + } + + +async def orchestrate_reference_intelligence_agent( + dir_info: Dict[str, str], logger, progress_callback: Optional[Callable] = None +) -> str: + """ + Orchestrate intelligent reference analysis with automated research discovery. + + This agent autonomously processes research references and discovers + related work using advanced AI-powered analysis algorithms. + + Args: + dir_info: Workspace infrastructure metadata + logger: Logger instance for intelligence tracking + progress_callback: Progress callback function for monitoring + + Returns: + str: Comprehensive reference intelligence analysis result + """ + if progress_callback: + progress_callback(50, "๐Ÿง  Orchestrating reference intelligence discovery...") + + reference_path = dir_info["reference_path"] + + # Check if reference analysis already exists + if os.path.exists(reference_path): + print(f"Found existing reference analysis at {reference_path}") + with open(reference_path, "r", encoding="utf-8") as f: + return f.read() + + # Execute reference analysis + reference_result = await paper_reference_analyzer(dir_info["paper_dir"], logger) + + # Save reference analysis result + with open(reference_path, "w", encoding="utf-8") as f: + f.write(reference_result) + print(f"Reference analysis saved to {reference_path}") + + return reference_result + + +async def orchestrate_document_preprocessing_agent( + dir_info: Dict[str, str], logger +) -> Dict[str, Any]: + """ + Orchestrate adaptive document preprocessing with intelligent segmentation control. + + This agent autonomously determines whether to use document segmentation based on + configuration settings and document size, then applies the appropriate processing strategy. + + Args: + dir_info: Workspace infrastructure metadata + logger: Logger instance for preprocessing tracking + + Returns: + dict: Document preprocessing result with segmentation metadata + """ + + try: + print("๐Ÿ” Starting adaptive document preprocessing...") + print(f" Paper directory: {dir_info['paper_dir']}") + + # Step 1: Check if any markdown files exist + md_files = [] + try: + md_files = [ + f for f in os.listdir(dir_info["paper_dir"]) if f.endswith(".md") + ] + except Exception as e: + print(f"โš ๏ธ Error reading paper directory: {e}") + + if not md_files: + print("โ„น๏ธ No markdown files found - skipping document preprocessing") + dir_info["segments_ready"] = False + dir_info["use_segmentation"] = False + return { + "status": "skipped", + "reason": "no_markdown_files", + "paper_dir": dir_info["paper_dir"], + "segments_ready": False, + "use_segmentation": False, + } + + # Step 2: Read document content to determine size + md_path = os.path.join(dir_info["paper_dir"], md_files[0]) + try: + # Check if file is actually a PDF by reading the first few bytes + with open(md_path, "rb") as f: + header = f.read(8) + if header.startswith(b"%PDF"): + raise IOError( + f"File {md_path} is a PDF file, not a text file. Please convert it to markdown format or use PDF processing tools." + ) + + with open(md_path, "r", encoding="utf-8") as f: + document_content = f.read() + except Exception as e: + print(f"โš ๏ธ Error reading document content: {e}") + dir_info["segments_ready"] = False + dir_info["use_segmentation"] = False + return { + "status": "error", + "error_message": f"Failed to read document: {str(e)}", + "paper_dir": dir_info["paper_dir"], + "segments_ready": False, + "use_segmentation": False, + } + + # Step 3: Determine if segmentation should be used + should_segment, reason = should_use_document_segmentation(document_content) + print(f"๐Ÿ“Š Segmentation decision: {should_segment}") + print(f" Reason: {reason}") + + # Store decision in dir_info for downstream agents + dir_info["use_segmentation"] = should_segment + + if should_segment: + print("๐Ÿ”ง Using intelligent document segmentation workflow...") + + # Prepare document segments using the segmentation agent + segmentation_result = await prepare_document_segments( + paper_dir=dir_info["paper_dir"], logger=logger + ) + + if segmentation_result["status"] == "success": + print("โœ… Document segmentation completed successfully!") + print(f" Segments directory: {segmentation_result['segments_dir']}") + print(" ๐Ÿง  Intelligent segments ready for planning agents") + + # Add segment information to dir_info for downstream agents + dir_info["segments_dir"] = segmentation_result["segments_dir"] + dir_info["segments_ready"] = True + + return segmentation_result + + else: + print( + f"โš ๏ธ Document segmentation failed: {segmentation_result.get('error_message', 'Unknown error')}" + ) + print(" Falling back to traditional full-document processing...") + dir_info["segments_ready"] = False + dir_info["use_segmentation"] = False + + return { + "status": "fallback_to_traditional", + "original_error": segmentation_result.get( + "error_message", "Unknown error" + ), + "paper_dir": dir_info["paper_dir"], + "segments_ready": False, + "use_segmentation": False, + "fallback_reason": "segmentation_failed", + } + else: + print("๐Ÿ“– Using traditional full-document reading workflow...") + dir_info["segments_ready"] = False + + return { + "status": "traditional", + "reason": reason, + "paper_dir": dir_info["paper_dir"], + "segments_ready": False, + "use_segmentation": False, + "document_size": len(document_content), + } + + except Exception as e: + print(f"โŒ Error during document preprocessing: {e}") + print(" Continuing with traditional full-document processing...") + + # Ensure fallback settings + dir_info["segments_ready"] = False + dir_info["use_segmentation"] = False + + return { + "status": "error", + "paper_dir": dir_info["paper_dir"], + "segments_ready": False, + "use_segmentation": False, + "error_message": str(e), + } + + +async def orchestrate_code_planning_agent( + dir_info: Dict[str, str], logger, progress_callback: Optional[Callable] = None +): + """ + Orchestrate intelligent code planning with automated design analysis. + + This agent autonomously generates optimal code reproduction plans and implementation + strategies using AI-driven code analysis and planning principles. + + Args: + dir_info: Workspace infrastructure metadata + logger: Logger instance for planning tracking + progress_callback: Progress callback function for monitoring + """ + if progress_callback: + progress_callback(40, "๐Ÿ—๏ธ Synthesizing intelligent code architecture...") + + initial_plan_path = dir_info["initial_plan_path"] + + # Check if initial plan already exists + if not os.path.exists(initial_plan_path): + # Use segmentation setting from preprocessing phase + use_segmentation = dir_info.get("use_segmentation", True) + print(f"๐Ÿ“Š Planning mode: {'Segmented' if use_segmentation else 'Traditional'}") + + initial_plan_result = await run_code_analyzer( + dir_info["paper_dir"], logger, use_segmentation=use_segmentation + ) + with open(initial_plan_path, "w", encoding="utf-8") as f: + f.write(initial_plan_result) + print(f"Initial plan saved to {initial_plan_path}") + + +async def automate_repository_acquisition_agent( + reference_result: str, + dir_info: Dict[str, str], + logger, + progress_callback: Optional[Callable] = None, +): + """ + Automate intelligent repository acquisition with AI-guided selection. + + This agent autonomously identifies, evaluates, and acquires relevant + repositories using intelligent filtering and automated download protocols. + + Args: + reference_result: Reference intelligence analysis result + dir_info: Workspace infrastructure metadata + logger: Logger instance for acquisition tracking + progress_callback: Progress callback function for monitoring + """ + if progress_callback: + progress_callback(60, "๐Ÿค– Automating intelligent repository acquisition...") + + await asyncio.sleep(5) # Brief pause for stability + + try: + download_result = await github_repo_download( + reference_result, dir_info["paper_dir"], logger + ) + + # Save download results + with open(dir_info["download_path"], "w", encoding="utf-8") as f: + f.write(download_result) + print(f"GitHub download results saved to {dir_info['download_path']}") + + # Verify if any repositories were actually downloaded + code_base_path = os.path.join(dir_info["paper_dir"], "code_base") + if os.path.exists(code_base_path): + downloaded_repos = [ + d + for d in os.listdir(code_base_path) + if os.path.isdir(os.path.join(code_base_path, d)) + and not d.startswith(".") + ] + + if downloaded_repos: + print( + f"Successfully downloaded {len(downloaded_repos)} repositories: {downloaded_repos}" + ) + else: + print( + "GitHub download phase completed, but no repositories were found in the code_base directory" + ) + print("This might indicate:") + print( + "1. No relevant repositories were identified in the reference analysis" + ) + print( + "2. Repository downloads failed due to access permissions or network issues" + ) + print( + "3. The download agent encountered errors during the download process" + ) + else: + print(f"Code base directory was not created: {code_base_path}") + + except Exception as e: + print(f"Error during GitHub repository download: {e}") + # Still save the error information + error_message = f"GitHub download failed: {str(e)}" + with open(dir_info["download_path"], "w", encoding="utf-8") as f: + f.write(error_message) + print(f"GitHub download error saved to {dir_info['download_path']}") + raise e # Re-raise to be handled by the main pipeline + + +async def orchestrate_codebase_intelligence_agent( + dir_info: Dict[str, str], logger, progress_callback: Optional[Callable] = None +) -> Dict: + """ + Orchestrate intelligent codebase analysis with automated knowledge extraction. + + This agent autonomously processes and indexes codebases using advanced + AI algorithms for intelligent relationship mapping and knowledge synthesis. + + Args: + dir_info: Workspace infrastructure metadata + logger: Logger instance for intelligence tracking + progress_callback: Progress callback function for monitoring + + Returns: + dict: Comprehensive codebase intelligence analysis result + """ + if progress_callback: + progress_callback(70, "๐Ÿงฎ Orchestrating codebase intelligence analysis...") + + print( + "Initiating intelligent codebase analysis with AI-powered relationship mapping..." + ) + await asyncio.sleep(2) # Brief pause before starting indexing + + # Check if code_base directory exists and has content + code_base_path = os.path.join(dir_info["paper_dir"], "code_base") + if not os.path.exists(code_base_path): + print(f"Code base directory not found: {code_base_path}") + return { + "status": "skipped", + "message": "No code base directory found - skipping indexing", + } + + # Check if there are any repositories in the code_base directory + try: + repo_dirs = [ + d + for d in os.listdir(code_base_path) + if os.path.isdir(os.path.join(code_base_path, d)) and not d.startswith(".") + ] + + if not repo_dirs: + print(f"No repositories found in {code_base_path}") + print("This might be because:") + print("1. GitHub download phase didn't complete successfully") + print("2. No relevant repositories were identified for download") + print("3. Repository download failed due to access issues") + print("Continuing with code implementation without codebase indexing...") + + # Save a report about the skipped indexing + skip_report = { + "status": "skipped", + "reason": "no_repositories_found", + "message": f"No repositories found in {code_base_path}", + "suggestions": [ + "Check if GitHub download phase completed successfully", + "Verify if relevant repositories were identified in reference analysis", + "Check network connectivity and GitHub access permissions", + ], + } + + with open(dir_info["index_report_path"], "w", encoding="utf-8") as f: + f.write(str(skip_report)) + print(f"Indexing skip report saved to {dir_info['index_report_path']}") + + return skip_report + + except Exception as e: + print(f"Error checking code base directory: {e}") + return { + "status": "error", + "message": f"Error checking code base directory: {str(e)}", + } + + try: + from workflows.codebase_index_workflow import run_codebase_indexing + + print(f"Found {len(repo_dirs)} repositories to index: {repo_dirs}") + + # Run codebase index workflow + index_result = await run_codebase_indexing( + paper_dir=dir_info["paper_dir"], + initial_plan_path=dir_info["initial_plan_path"], + config_path="mcp_agent.secrets.yaml", + logger=logger, + ) + + # Log indexing results + if index_result["status"] == "success": + print("Code indexing completed successfully!") + print( + f"Indexed {index_result['statistics']['total_repositories'] if index_result.get('statistics') else len(index_result['output_files'])} repositories" + ) + print(f"Generated {len(index_result['output_files'])} index files") + + # Save indexing results to file + with open(dir_info["index_report_path"], "w", encoding="utf-8") as f: + f.write(str(index_result)) + print(f"Indexing report saved to {dir_info['index_report_path']}") + + elif index_result["status"] == "warning": + print(f"Code indexing completed with warnings: {index_result['message']}") + else: + print(f"Code indexing failed: {index_result['message']}") + + return index_result + + except Exception as e: + print(f"Error during codebase indexing workflow: {e}") + print("Continuing with code implementation despite indexing failure...") + + # Save error report + error_report = { + "status": "error", + "message": str(e), + "phase": "codebase_indexing", + "recovery_action": "continuing_with_code_implementation", + } + + with open(dir_info["index_report_path"], "w", encoding="utf-8") as f: + f.write(str(error_report)) + print(f"Indexing error report saved to {dir_info['index_report_path']}") + + return error_report + + +async def synthesize_code_implementation_agent( + dir_info: Dict[str, str], + logger, + progress_callback: Optional[Callable] = None, + enable_indexing: bool = True, +) -> Dict: + """ + Synthesize intelligent code implementation with automated development. + + This agent autonomously generates high-quality code implementations using + AI-powered development strategies and intelligent code synthesis algorithms. + + Args: + dir_info: Workspace infrastructure metadata + logger: Logger instance for implementation tracking + progress_callback: Progress callback function for monitoring + enable_indexing: Whether to enable code reference indexing for enhanced implementation + + Returns: + dict: Comprehensive code implementation synthesis result + """ + if progress_callback: + progress_callback(85, "๐Ÿ”ฌ Synthesizing intelligent code implementation...") + + print( + "Launching intelligent code synthesis with AI-driven implementation strategies..." + ) + await asyncio.sleep(3) # Brief pause before starting implementation + + try: + # Create code implementation workflow instance based on indexing preference + if enable_indexing: + print( + "๐Ÿ” Using enhanced code implementation workflow with reference indexing..." + ) + code_workflow = CodeImplementationWorkflowWithIndex() + else: + print("โšก Using standard code implementation workflow (fast mode)...") + code_workflow = CodeImplementationWorkflow() + + # Check if initial plan file exists + if os.path.exists(dir_info["initial_plan_path"]): + print(f"Using initial plan from {dir_info['initial_plan_path']}") + + # Run code implementation workflow with pure code mode + implementation_result = await code_workflow.run_workflow( + plan_file_path=dir_info["initial_plan_path"], + target_directory=dir_info["paper_dir"], + pure_code_mode=True, # Focus on code implementation, skip testing + ) + + # Log implementation results + if implementation_result["status"] == "success": + print("Code implementation completed successfully!") + print(f"Code directory: {implementation_result['code_directory']}") + + # Save implementation results to file + with open( + dir_info["implementation_report_path"], "w", encoding="utf-8" + ) as f: + f.write(str(implementation_result)) + print( + f"Implementation report saved to {dir_info['implementation_report_path']}" + ) + + else: + print( + f"Code implementation failed: {implementation_result.get('message', 'Unknown error')}" + ) + + return implementation_result + else: + print( + f"Initial plan file not found at {dir_info['initial_plan_path']}, skipping code implementation" + ) + return { + "status": "warning", + "message": "Initial plan not found - code implementation skipped", + } + + except Exception as e: + print(f"Error during code implementation workflow: {e}") + return {"status": "error", "message": str(e)} + + +async def run_chat_planning_agent(user_input: str, logger) -> str: + """ + Run the chat-based planning agent for user-provided coding requirements. + + This agent transforms user's coding description into a comprehensive implementation plan + that can be directly used for code generation. It handles both academic and engineering + requirements with intelligent context adaptation. + + Args: + user_input: User's coding requirements and description + logger: Logger instance for logging information + + Returns: + str: Comprehensive implementation plan in YAML format + """ + try: + print("๐Ÿ’ฌ Starting chat-based planning agent...") + print(f"Input length: {len(user_input) if user_input else 0}") + print(f"Input preview: {user_input[:200] if user_input else 'None'}...") + + if not user_input or user_input.strip() == "": + raise ValueError( + "Empty or None user_input provided to run_chat_planning_agent" + ) + + # Create the chat planning agent + chat_planning_agent = Agent( + name="ChatPlanningAgent", + instruction=CHAT_AGENT_PLANNING_PROMPT, + server_names=get_search_server_names(), # Dynamic search server configuration + ) + + async with chat_planning_agent: + print("chat_planning: Connected to server, calling list_tools...") + try: + tools = await chat_planning_agent.list_tools() + print( + "Tools available:", + tools.model_dump() if hasattr(tools, "model_dump") else str(tools), + ) + except Exception as e: + print(f"Failed to list tools: {e}") + + try: + planner = await chat_planning_agent.attach_llm( + get_preferred_llm_class() + ) + print("โœ… LLM attached successfully") + except Exception as e: + print(f"โŒ Failed to attach LLM: {e}") + raise + + # Set higher token output for comprehensive planning + planning_params = RequestParams( + maxTokens=8192, # ไฝฟ็”จ camelCase - Higher token limit for detailed plans + temperature=0.2, # Lower temperature for more structured output + ) + + print( + f"๐Ÿ”„ Making LLM request with params: maxTokens={planning_params.maxTokens}, temperature={planning_params.temperature}" + ) + + # Format the input message for the agent + formatted_message = f"""Please analyze the following coding requirements and generate a comprehensive implementation plan: + +User Requirements: +{user_input} + +Please provide a detailed implementation plan that covers all aspects needed for successful development.""" + + try: + raw_result = await planner.generate_str( + message=formatted_message, request_params=planning_params + ) + + print("โœ… Planning request completed") + print(f"Raw result type: {type(raw_result)}") + print(f"Raw result length: {len(raw_result) if raw_result else 0}") + + if not raw_result: + print("โŒ CRITICAL: raw_result is empty or None!") + raise ValueError("Chat planning agent returned empty result") + + except Exception as e: + print(f"โŒ Planning generation failed: {e}") + print(f"Exception type: {type(e)}") + raise + + # Log to SimpleLLMLogger + if hasattr(logger, "log_response"): + logger.log_response( + raw_result, model="ChatPlanningAgent", agent="ChatPlanningAgent" + ) + + if not raw_result or raw_result.strip() == "": + print("โŒ CRITICAL: Planning result is empty!") + raise ValueError("Chat planning agent produced empty output") + + print("๐ŸŽฏ Chat planning completed successfully") + print(f"Planning result preview: {raw_result[:500]}...") + + return raw_result + + except Exception as e: + print(f"โŒ run_chat_planning_agent failed: {e}") + print(f"Exception details: {type(e).__name__}: {str(e)}") + raise + + +async def execute_multi_agent_research_pipeline( + input_source: str, + logger, + progress_callback: Optional[Callable] = None, + enable_indexing: bool = True, +) -> str: + """ + Execute the complete intelligent multi-agent research orchestration pipeline. + + This is the main AI orchestration engine that coordinates autonomous research workflow agents: + - Local workspace automation for seamless environment management + - Intelligent research analysis with automated content processing + - AI-driven code architecture synthesis and design automation + - Reference intelligence discovery with automated knowledge extraction (optional) + - Codebase intelligence orchestration with automated relationship analysis (optional) + - Intelligent code implementation synthesis with AI-powered development + + Args: + input_source: Research input source (file path, URL, or preprocessed analysis) + logger: Logger instance for comprehensive workflow intelligence tracking + progress_callback: Progress callback function for real-time monitoring + enable_indexing: Whether to enable advanced intelligence analysis (default: True) + + Returns: + str: The comprehensive pipeline execution result with status and outcomes + """ + try: + # Phase 0: Workspace Setup + if progress_callback: + progress_callback(5, "๐Ÿ”„ Setting up workspace for file processing...") + + print("๐Ÿš€ Initializing intelligent multi-agent research orchestration system") + + # Setup local workspace directory + workspace_dir = os.path.join(os.getcwd(), "deepcode_lab") + os.makedirs(workspace_dir, exist_ok=True) + + print("๐Ÿ“ Working environment: local") + print(f"๐Ÿ“‚ Workspace directory: {workspace_dir}") + print("โœ… Workspace status: ready") + + # Log intelligence functionality status + if enable_indexing: + print("๐Ÿง  Advanced intelligence analysis enabled - comprehensive workflow") + else: + print("โšก Optimized mode - advanced intelligence analysis disabled") + + # Phase 1: Input Processing and Validation + input_source = await _process_input_source(input_source, logger) + + # Phase 2: Research Analysis and Resource Processing (if needed) + if isinstance(input_source, str) and ( + input_source.endswith((".pdf", ".docx", ".txt", ".html", ".md")) + or input_source.startswith(("http", "file://")) + ): + ( + analysis_result, + download_result, + ) = await orchestrate_research_analysis_agent( + input_source, logger, progress_callback + ) + else: + download_result = input_source # Use input directly if already processed + + # Phase 3: Workspace Infrastructure Synthesis + if progress_callback: + progress_callback( + 40, "๐Ÿ—๏ธ Synthesizing intelligent workspace infrastructure..." + ) + + dir_info = await synthesize_workspace_infrastructure_agent( + download_result, logger, workspace_dir + ) + await asyncio.sleep(5) + + # Phase 3.5: Document Segmentation and Preprocessing + + segmentation_result = await orchestrate_document_preprocessing_agent( + dir_info, logger + ) + + # Handle segmentation result + if segmentation_result["status"] == "success": + print("โœ… Document preprocessing completed successfully!") + print( + f" ๐Ÿ“Š Using segmentation: {dir_info.get('use_segmentation', False)}" + ) + if dir_info.get("segments_ready", False): + print( + f" ๐Ÿ“ Segments directory: {segmentation_result.get('segments_dir', 'N/A')}" + ) + elif segmentation_result["status"] == "fallback_to_traditional": + print("โš ๏ธ Document segmentation failed, using traditional processing") + print( + f" Original error: {segmentation_result.get('original_error', 'Unknown')}" + ) + else: + print( + f"โš ๏ธ Document preprocessing encountered issues: {segmentation_result.get('error_message', 'Unknown')}" + ) + + # Phase 4: Code Planning Orchestration + await orchestrate_code_planning_agent(dir_info, logger, progress_callback) + + # Phase 5: Reference Intelligence (only when indexing is enabled) + if enable_indexing: + reference_result = await orchestrate_reference_intelligence_agent( + dir_info, logger, progress_callback + ) + else: + print("๐Ÿ”ถ Skipping reference intelligence analysis (fast mode enabled)") + # Create empty reference analysis result to maintain file structure consistency + reference_result = "Reference intelligence analysis skipped - fast mode enabled for optimized processing" + with open(dir_info["reference_path"], "w", encoding="utf-8") as f: + f.write(reference_result) + + # Phase 6: Repository Acquisition Automation (optional) + if enable_indexing: + await automate_repository_acquisition_agent( + reference_result, dir_info, logger, progress_callback + ) + else: + print("๐Ÿ”ถ Skipping automated repository acquisition (fast mode enabled)") + # Create empty download result file to maintain file structure consistency + with open(dir_info["download_path"], "w", encoding="utf-8") as f: + f.write( + "Automated repository acquisition skipped - fast mode enabled for optimized processing" + ) + + # Phase 7: Codebase Intelligence Orchestration (optional) + if enable_indexing: + index_result = await orchestrate_codebase_intelligence_agent( + dir_info, logger, progress_callback + ) + else: + print("๐Ÿ”ถ Skipping codebase intelligence orchestration (fast mode enabled)") + # Create a skipped indexing result + index_result = { + "status": "skipped", + "reason": "fast_mode_enabled", + "message": "Codebase intelligence orchestration skipped for optimized processing", + } + with open(dir_info["index_report_path"], "w", encoding="utf-8") as f: + f.write(str(index_result)) + + # Phase 8: Code Implementation Synthesis + implementation_result = await synthesize_code_implementation_agent( + dir_info, logger, progress_callback, enable_indexing + ) + + # Final Status Report + if enable_indexing: + pipeline_summary = ( + f"Multi-agent research pipeline completed for {dir_info['paper_dir']}" + ) + else: + pipeline_summary = f"Multi-agent research pipeline completed (fast mode) for {dir_info['paper_dir']}" + + # Add indexing status to summary + if not enable_indexing: + pipeline_summary += ( + "\nโšก Fast mode: GitHub download and codebase indexing skipped" + ) + elif index_result["status"] == "skipped": + pipeline_summary += f"\n๐Ÿ”ถ Codebase indexing: {index_result['message']}" + elif index_result["status"] == "error": + pipeline_summary += ( + f"\nโŒ Codebase indexing failed: {index_result['message']}" + ) + elif index_result["status"] == "success": + pipeline_summary += "\nโœ… Codebase indexing completed successfully" + + # Add implementation status to summary + if implementation_result["status"] == "success": + pipeline_summary += "\n๐ŸŽ‰ Code implementation completed successfully!" + pipeline_summary += ( + f"\n๐Ÿ“ Code generated in: {implementation_result['code_directory']}" + ) + return pipeline_summary + elif implementation_result["status"] == "warning": + pipeline_summary += ( + f"\nโš ๏ธ Code implementation: {implementation_result['message']}" + ) + return pipeline_summary + else: + pipeline_summary += ( + f"\nโŒ Code implementation failed: {implementation_result['message']}" + ) + return pipeline_summary + + except Exception as e: + print(f"Error in execute_multi_agent_research_pipeline: {e}") + raise e + + +# Backward compatibility alias (deprecated) +async def paper_code_preparation( + input_source: str, logger, progress_callback: Optional[Callable] = None +) -> str: + """ + Deprecated: Use execute_multi_agent_research_pipeline instead. + + Args: + input_source: Input source + logger: Logger instance + progress_callback: Progress callback function + + Returns: + str: Pipeline result + """ + print( + "paper_code_preparation is deprecated. Use execute_multi_agent_research_pipeline instead." + ) + return await execute_multi_agent_research_pipeline( + input_source, logger, progress_callback + ) + + +async def execute_chat_based_planning_pipeline( + user_input: str, + logger, + progress_callback: Optional[Callable] = None, + enable_indexing: bool = True, +) -> str: + """ + Execute the chat-based planning and implementation pipeline. + + This pipeline is designed for users who provide coding requirements directly through chat, + bypassing the traditional paper analysis phases (Phase 0-7) and jumping directly to + planning and code implementation. + + Pipeline Flow: + - Chat Planning: Transform user input into implementation plan + - Workspace Setup: Create necessary directory structure + - Code Implementation: Generate code based on the plan + + Args: + user_input: User's coding requirements and description + logger: Logger instance for comprehensive workflow tracking + progress_callback: Progress callback function for real-time monitoring + enable_indexing: Whether to enable code reference indexing for enhanced implementation + + Returns: + str: The pipeline execution result with status and outcomes + """ + try: + print("๐Ÿš€ Initializing chat-based planning and implementation pipeline") + print("๐Ÿ’ฌ Chat mode: Direct user requirements to code implementation") + + # Phase 0: Workspace Setup + if progress_callback: + progress_callback(5, "๐Ÿ”„ Setting up workspace for file processing...") + + # Setup local workspace directory + workspace_dir = os.path.join(os.getcwd(), "deepcode_lab") + os.makedirs(workspace_dir, exist_ok=True) + + print("๐Ÿ“ Working environment: local") + print(f"๐Ÿ“‚ Workspace directory: {workspace_dir}") + print("โœ… Workspace status: ready") + + # Phase 1: Chat-Based Planning + if progress_callback: + progress_callback( + 30, + "๐Ÿ’ฌ Generating comprehensive implementation plan from user requirements...", + ) + + print("๐Ÿง  Running chat-based planning agent...") + planning_result = await run_chat_planning_agent(user_input, logger) + + # Phase 2: Workspace Infrastructure Synthesis + if progress_callback: + progress_callback( + 50, "๐Ÿ—๏ธ Synthesizing intelligent workspace infrastructure..." + ) + + # Create workspace directory structure for chat mode + # First, let's create a temporary directory structure that mimics a paper workspace + import time + + # Generate a unique paper directory name + timestamp = str(int(time.time())) + paper_name = f"chat_project_{timestamp}" + + # Use workspace directory + chat_paper_dir = os.path.join(workspace_dir, "papers", paper_name) + + os.makedirs(chat_paper_dir, exist_ok=True) + + # Create a synthetic markdown file with user requirements + markdown_content = f"""# User Coding Requirements + +## Project Description +This is a coding project generated from user requirements via chat interface. + +## User Requirements +{user_input} + +## Generated Implementation Plan +The following implementation plan was generated by the AI chat planning agent: + +```yaml +{planning_result} +``` + +## Project Metadata +- **Input Type**: Chat Input +- **Generation Method**: AI Chat Planning Agent +- **Timestamp**: {timestamp} +""" + + # Save the markdown file + markdown_file_path = os.path.join(chat_paper_dir, f"{paper_name}.md") + with open(markdown_file_path, "w", encoding="utf-8") as f: + f.write(markdown_content) + + print(f"๐Ÿ’พ Created chat project workspace: {chat_paper_dir}") + print(f"๐Ÿ“„ Saved requirements to: {markdown_file_path}") + + # Create a download result that matches FileProcessor expectations + synthetic_download_result = json.dumps( + { + "status": "success", + "paper_path": markdown_file_path, + "input_type": "chat_input", + "paper_info": { + "title": "User-Provided Coding Requirements", + "source": "chat_input", + "description": "Implementation plan generated from user requirements", + }, + } + ) + + dir_info = await synthesize_workspace_infrastructure_agent( + synthetic_download_result, logger, workspace_dir + ) + await asyncio.sleep(10) # Brief pause for file system operations + + # Phase 3: Save Planning Result + if progress_callback: + progress_callback(70, "๐Ÿ“ Saving implementation plan...") + + # Save the planning result to the initial_plan.txt file (same location as Phase 4 in original pipeline) + initial_plan_path = dir_info["initial_plan_path"] + with open(initial_plan_path, "w", encoding="utf-8") as f: + f.write(planning_result) + print(f"๐Ÿ’พ Implementation plan saved to {initial_plan_path}") + + # Phase 4: Code Implementation Synthesis (same as Phase 8 in original pipeline) + if progress_callback: + progress_callback(85, "๐Ÿ”ฌ Synthesizing intelligent code implementation...") + + implementation_result = await synthesize_code_implementation_agent( + dir_info, logger, progress_callback, enable_indexing + ) + + # Final Status Report + pipeline_summary = f"Chat-based planning and implementation pipeline completed for {dir_info['paper_dir']}" + + # Add implementation status to summary + if implementation_result["status"] == "success": + pipeline_summary += "\n๐ŸŽ‰ Code implementation completed successfully!" + pipeline_summary += ( + f"\n๐Ÿ“ Code generated in: {implementation_result['code_directory']}" + ) + pipeline_summary += ( + "\n๐Ÿ’ฌ Generated from user requirements via chat interface" + ) + return pipeline_summary + elif implementation_result["status"] == "warning": + pipeline_summary += ( + f"\nโš ๏ธ Code implementation: {implementation_result['message']}" + ) + return pipeline_summary + else: + pipeline_summary += ( + f"\nโŒ Code implementation failed: {implementation_result['message']}" + ) + return pipeline_summary + + except Exception as e: + print(f"Error in execute_chat_based_planning_pipeline: {e}") + raise e diff --git a/DeepCode/workflows/agents/__init__.py b/DeepCode/workflows/agents/__init__.py new file mode 100644 index 00000000..e247e994 --- /dev/null +++ b/DeepCode/workflows/agents/__init__.py @@ -0,0 +1,12 @@ +""" +Agents Package for Code Implementation Workflow + +This package contains specialized agents for different aspects of code implementation: +- CodeImplementationAgent: Handles file-by-file code generation +- ConciseMemoryAgent: Manages memory optimization and consistency across phases +""" + +from .code_implementation_agent import CodeImplementationAgent +from .memory_agent_concise import ConciseMemoryAgent as MemoryAgent + +__all__ = ["CodeImplementationAgent", "MemoryAgent"] diff --git a/DeepCode/workflows/agents/code_implementation_agent.py b/DeepCode/workflows/agents/code_implementation_agent.py new file mode 100644 index 00000000..7bf79e43 --- /dev/null +++ b/DeepCode/workflows/agents/code_implementation_agent.py @@ -0,0 +1,1117 @@ +""" +Code Implementation Agent for File-by-File Development + +Handles systematic code implementation with progress tracking and +memory optimization for long-running development sessions. +""" + +import json +import time +import logging +from typing import Dict, Any, List, Optional + +# Import tiktoken for token calculation +try: + import tiktoken + + TIKTOKEN_AVAILABLE = True +except ImportError: + TIKTOKEN_AVAILABLE = False + +# Import prompts from code_prompts +import sys +import os + +sys.path.insert( + 0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +from prompts.code_prompts import ( + GENERAL_CODE_IMPLEMENTATION_SYSTEM_PROMPT, +) + + +class CodeImplementationAgent: + """ + Code Implementation Agent for systematic file-by-file development + + Responsibilities: + - Track file implementation progress + - Execute MCP tool calls for code generation + - Monitor implementation status + - Coordinate with Summary Agent for memory optimization + - Calculate token usage for context management + """ + + def __init__( + self, + mcp_agent, + logger: Optional[logging.Logger] = None, + enable_read_tools: bool = True, + ): + """ + Initialize Code Implementation Agent + + Args: + mcp_agent: MCP agent instance for tool calls + logger: Logger instance for tracking operations + enable_read_tools: Whether to enable read_file and read_code_mem tools (default: True) + """ + self.mcp_agent = mcp_agent + self.logger = logger or self._create_default_logger() + self.enable_read_tools = enable_read_tools # Control read tools execution + + self.implementation_summary = { + "completed_files": [], + "technical_decisions": [], + "important_constraints": [], + "architecture_notes": [], + "dependency_analysis": [], # Track dependency analysis and file reads + } + self.files_implemented_count = 0 + self.implemented_files_set = ( + set() + ) # Track unique file paths to avoid duplicate counting + self.files_read_for_dependencies = ( + set() + ) # Track files read for dependency analysis + self.last_summary_file_count = ( + 0 # Track the file count when last summary was triggered + ) + + # Token calculation settings + self.max_context_tokens = ( + 200000 # Default max context tokens for Claude-3.5-Sonnet + ) + self.token_buffer = 10000 # Safety buffer before reaching max + self.summary_trigger_tokens = ( + self.max_context_tokens - self.token_buffer + ) # Trigger summary when approaching limit + self.last_summary_token_count = ( + 0 # Track token count when last summary was triggered + ) + + # Initialize tokenizer + if TIKTOKEN_AVAILABLE: + try: + # Use Claude-3 tokenizer (approximation with OpenAI's o200k_base) + self.tokenizer = tiktoken.get_encoding("o200k_base") + self.logger.info("Token calculation enabled with o200k_base encoding") + except Exception as e: + self.tokenizer = None + self.logger.warning(f"Failed to initialize tokenizer: {e}") + else: + self.tokenizer = None + self.logger.warning( + "tiktoken not available, token-based summary triggering disabled" + ) + + # Analysis loop detection + self.recent_tool_calls = [] # Track recent tool calls to detect analysis loops + self.max_read_without_write = 5 # Max read_file calls without write_file + + # Memory agent integration + self.memory_agent = None # Will be set externally + self.llm_client = None # Will be set externally + self.llm_client_type = None # Will be set externally + + # Log read tools configuration + read_tools_status = "ENABLED" if self.enable_read_tools else "DISABLED" + self.logger.info( + f"๐Ÿ”ง Code Implementation Agent initialized - Read tools: {read_tools_status}" + ) + if not self.enable_read_tools: + self.logger.info( + "๐Ÿšซ Testing mode: read_file and read_code_mem will be skipped when called" + ) + + def _create_default_logger(self) -> logging.Logger: + """Create default logger if none provided""" + logger = logging.getLogger(f"{__name__}.CodeImplementationAgent") + # Don't add handlers to child loggers - let them propagate to root + logger.setLevel(logging.INFO) + return logger + + def get_system_prompt(self) -> str: + """ + Get the system prompt for code implementation + """ + return GENERAL_CODE_IMPLEMENTATION_SYSTEM_PROMPT + + def set_memory_agent(self, memory_agent, llm_client=None, llm_client_type=None): + """ + Set memory agent for code summary generation + + Args: + memory_agent: Memory agent instance + llm_client: LLM client for summary generation + llm_client_type: Type of LLM client ("anthropic" or "openai") + """ + self.memory_agent = memory_agent + self.llm_client = llm_client + self.llm_client_type = llm_client_type + self.logger.info("Memory agent integration configured") + + async def execute_tool_calls(self, tool_calls: List[Dict]) -> List[Dict]: + """ + Execute MCP tool calls and track implementation progress + + Args: + tool_calls: List of tool calls to execute + + Returns: + List of tool execution results + """ + results = [] + + for tool_call in tool_calls: + tool_name = tool_call["name"] + tool_input = tool_call["input"] + + self.logger.info(f"Executing MCP tool: {tool_name}") + + try: + # Check if read tools are disabled + if not self.enable_read_tools and tool_name in [ + "read_file", + "read_code_mem", + ]: + # self.logger.info(f"๐Ÿšซ SKIPPING {tool_name} - Read tools disabled for testing") + # Return a mock result indicating the tool was skipped + mock_result = json.dumps( + { + "status": "skipped", + "message": f"{tool_name} tool disabled for testing", + "tool_disabled": True, + "original_input": tool_input, + }, + ensure_ascii=False, + ) + + results.append( + { + "tool_id": tool_call["id"], + "tool_name": tool_name, + "result": mock_result, + } + ) + continue + + # read_code_mem is now a proper MCP tool, no special handling needed + + # INTERCEPT read_file calls - redirect to read_code_mem first if memory agent is available + if tool_name == "read_file": + file_path = tool_call["input"].get("file_path", "unknown") + self.logger.info(f"๐Ÿ” READ_FILE CALL DETECTED: {file_path}") + self.logger.info( + f"๐Ÿ“Š Files implemented count: {self.files_implemented_count}" + ) + self.logger.info( + f"๐Ÿง  Memory agent available: {self.memory_agent is not None}" + ) + + # Enable optimization if memory agent is available (more aggressive approach) + if self.memory_agent is not None: + self.logger.info( + f"๐Ÿ”„ INTERCEPTING read_file call for {file_path} (memory agent available)" + ) + result = await self._handle_read_file_with_memory_optimization( + tool_call + ) + results.append(result) + continue + else: + self.logger.info( + "๐Ÿ“ NO INTERCEPTION: no memory agent available" + ) + + if self.mcp_agent: + # Execute tool call through MCP protocol + result = await self.mcp_agent.call_tool(tool_name, tool_input) + + # Track file implementation progress + if tool_name == "write_file": + await self._track_file_implementation_with_summary( + tool_call, result + ) + elif tool_name == "read_file": + self._track_dependency_analysis(tool_call, result) + + # Track tool calls for analysis loop detection + self._track_tool_call_for_loop_detection(tool_name) + + results.append( + { + "tool_id": tool_call["id"], + "tool_name": tool_name, + "result": result, + } + ) + else: + results.append( + { + "tool_id": tool_call["id"], + "tool_name": tool_name, + "result": json.dumps( + { + "status": "error", + "message": "MCP agent not initialized", + }, + ensure_ascii=False, + ), + } + ) + + except Exception as e: + self.logger.error(f"MCP tool execution failed: {e}") + results.append( + { + "tool_id": tool_call["id"], + "tool_name": tool_name, + "result": json.dumps( + {"status": "error", "message": str(e)}, ensure_ascii=False + ), + } + ) + + return results + + # _handle_read_code_mem method removed - read_code_mem is now a proper MCP tool + + async def _handle_read_file_with_memory_optimization(self, tool_call: Dict) -> Dict: + """ + Intercept read_file calls and redirect to read_code_mem if a summary exists. + This prevents unnecessary file reads if the summary is already available. + """ + file_path = tool_call["input"].get("file_path") + if not file_path: + return { + "tool_id": tool_call["id"], + "tool_name": "read_file", + "result": json.dumps( + {"status": "error", "message": "file_path parameter is required"}, + ensure_ascii=False, + ), + } + + # Check if a summary exists for this file using read_code_mem MCP tool + should_use_summary = False + if self.memory_agent and self.mcp_agent: + try: + # Use read_code_mem MCP tool to check if summary exists (pass file path as list) + read_code_mem_result = await self.mcp_agent.call_tool( + "read_code_mem", {"file_paths": [file_path]} + ) + + # Parse the result to check if summary was found + import json + + if isinstance(read_code_mem_result, str): + try: + result_data = json.loads(read_code_mem_result) + # Check if any summaries were found in the results + should_use_summary = ( + result_data.get("status") + in ["all_summaries_found", "partial_summaries_found"] + and result_data.get("summaries_found", 0) > 0 + ) + except json.JSONDecodeError: + should_use_summary = False + except Exception as e: + self.logger.debug(f"read_code_mem check failed for {file_path}: {e}") + should_use_summary = False + + if should_use_summary: + self.logger.info(f"๐Ÿ”„ READ_FILE INTERCEPTED: Using summary for {file_path}") + + # Use the MCP agent to call read_code_mem tool + if self.mcp_agent: + result = await self.mcp_agent.call_tool( + "read_code_mem", {"file_paths": [file_path]} + ) + + # Modify the result to indicate it was originally a read_file call + import json + + try: + result_data = ( + json.loads(result) if isinstance(result, str) else result + ) + if isinstance(result_data, dict): + # Extract the specific file result for the single file we requested + file_results = result_data.get("results", []) + if file_results and len(file_results) > 0: + specific_result = file_results[ + 0 + ] # Get the first (and only) result + # Transform to match the old single-file format for backward compatibility + transformed_result = { + "status": specific_result.get("status", "no_summary"), + "file_path": specific_result.get( + "file_path", file_path + ), + "summary_content": specific_result.get( + "summary_content" + ), + "message": specific_result.get("message", ""), + "original_tool": "read_file", + "optimization": "redirected_to_read_code_mem", + } + final_result = json.dumps( + transformed_result, ensure_ascii=False + ) + else: + # Fallback if no results + result_data["original_tool"] = "read_file" + result_data["optimization"] = "redirected_to_read_code_mem" + final_result = json.dumps(result_data, ensure_ascii=False) + else: + final_result = result + except (json.JSONDecodeError, TypeError): + final_result = result + + return { + "tool_id": tool_call["id"], + "tool_name": "read_file", # Keep original tool name for tracking + "result": final_result, + } + else: + self.logger.warning( + "MCP agent not available for read_code_mem optimization" + ) + else: + self.logger.info( + f"๐Ÿ“ READ_FILE: No summary for {file_path}, using actual file" + ) + + # Execute the original read_file call + if self.mcp_agent: + result = await self.mcp_agent.call_tool("read_file", tool_call["input"]) + + # Track dependency analysis for the actual file read + self._track_dependency_analysis(tool_call, result) + + # Track tool calls for analysis loop detection + self._track_tool_call_for_loop_detection("read_file") + + return { + "tool_id": tool_call["id"], + "tool_name": "read_file", + "result": result, + } + else: + return { + "tool_id": tool_call["id"], + "tool_name": "read_file", + "result": json.dumps( + {"status": "error", "message": "MCP agent not initialized"}, + ensure_ascii=False, + ), + } + + async def _track_file_implementation_with_summary( + self, tool_call: Dict, result: Any + ): + """ + Track file implementation and create code summary + + Args: + tool_call: The write_file tool call + result: Result of the tool execution + """ + # First do the regular tracking + self._track_file_implementation(tool_call, result) + + # Then create and save code summary if memory agent is available + if self.memory_agent and self.llm_client and self.llm_client_type: + try: + file_path = tool_call["input"].get("file_path") + file_content = tool_call["input"].get("content", "") + + if file_path and file_content: + # Create code implementation summary + summary = await self.memory_agent.create_code_implementation_summary( + self.llm_client, + self.llm_client_type, + file_path, + file_content, + self.get_files_implemented_count(), # Pass the current file count + ) + + self.logger.info( + f"Created code summary for implemented file: {file_path}, summary: {summary[:100]}..." + ) + else: + self.logger.warning( + "Missing file path or content for summary generation" + ) + + except Exception as e: + self.logger.error(f"Failed to create code summary: {e}") + + def _track_file_implementation(self, tool_call: Dict, result: Any): + """ + Track file implementation progress + """ + try: + # Handle different result types from MCP + result_data = None + + # Check if result is a CallToolResult object + if hasattr(result, "content"): + # Extract content from CallToolResult + if hasattr(result.content, "text"): + result_content = result.content.text + else: + result_content = str(result.content) + + # Try to parse as JSON + try: + result_data = json.loads(result_content) + except json.JSONDecodeError: + # If not JSON, create a structure + result_data = { + "status": "success", + "file_path": tool_call["input"].get("file_path", "unknown"), + } + elif isinstance(result, str): + # Try to parse string result + try: + result_data = json.loads(result) + except json.JSONDecodeError: + result_data = { + "status": "success", + "file_path": tool_call["input"].get("file_path", "unknown"), + } + elif isinstance(result, dict): + # Direct dictionary result + result_data = result + else: + # Fallback: assume success and extract file path from input + result_data = { + "status": "success", + "file_path": tool_call["input"].get("file_path", "unknown"), + } + + # Extract file path for tracking + file_path = None + if result_data and result_data.get("status") == "success": + file_path = result_data.get( + "file_path", tool_call["input"].get("file_path", "unknown") + ) + else: + file_path = tool_call["input"].get("file_path") + + # Only count unique files, not repeated tool calls on same file + if file_path and file_path not in self.implemented_files_set: + # This is a new file implementation + self.implemented_files_set.add(file_path) + self.files_implemented_count += 1 + # self.logger.info(f"New file implementation tracked: count={self.files_implemented_count}, file={file_path}") + # print(f"New file implementation tracked: count={self.files_implemented_count}, file={file_path}") + + # Add to completed files list + self.implementation_summary["completed_files"].append( + { + "file": file_path, + "iteration": self.files_implemented_count, + "timestamp": time.time(), + "size": result_data.get("size", 0) if result_data else 0, + } + ) + + # self.logger.info( + # f"New file implementation tracked: count={self.files_implemented_count}, file={file_path}" + # ) + # print(f"๐Ÿ“ NEW FILE IMPLEMENTED: count={self.files_implemented_count}, file={file_path}") + # print(f"๐Ÿ”ง OPTIMIZATION NOW ENABLED: files_implemented_count > 0 = {self.files_implemented_count > 0}") + elif file_path and file_path in self.implemented_files_set: + # This file was already implemented (duplicate tool call) + self.logger.debug( + f"File already tracked, skipping duplicate count: {file_path}" + ) + else: + # No valid file path found + self.logger.warning("No valid file path found for tracking") + + except Exception as e: + self.logger.warning(f"Failed to track file implementation: {e}") + # Even if tracking fails, try to count based on tool input (but check for duplicates) + + file_path = tool_call["input"].get("file_path") + if file_path and file_path not in self.implemented_files_set: + self.implemented_files_set.add(file_path) + self.files_implemented_count += 1 + self.logger.info( + f"File implementation counted (emergency fallback): count={self.files_implemented_count}, file={file_path}" + ) + + def _track_dependency_analysis(self, tool_call: Dict, result: Any): + """ + Track dependency analysis through read_file calls + """ + try: + file_path = tool_call["input"].get("file_path") + if file_path: + # Track unique files read for dependency analysis + if file_path not in self.files_read_for_dependencies: + self.files_read_for_dependencies.add(file_path) + + # Add to dependency analysis summary + self.implementation_summary["dependency_analysis"].append( + { + "file_read": file_path, + "timestamp": time.time(), + "purpose": "dependency_analysis", + } + ) + + self.logger.info( + f"Dependency analysis tracked: file_read={file_path}" + ) + + except Exception as e: + self.logger.warning(f"Failed to track dependency analysis: {e}") + + def calculate_messages_token_count(self, messages: List[Dict]) -> int: + """ + Calculate total token count for a list of messages + + Args: + messages: List of chat messages with 'role' and 'content' keys + + Returns: + Total token count + """ + if not self.tokenizer: + # Fallback: rough estimation based on character count + total_chars = sum(len(str(msg.get("content", ""))) for msg in messages) + # Rough approximation: 1 token โ‰ˆ 4 characters + return total_chars // 4 + + try: + total_tokens = 0 + for message in messages: + content = str(message.get("content", "")) + role = message.get("role", "") + + # Count tokens for content + if content: + content_tokens = len( + self.tokenizer.encode(content, disallowed_special=()) + ) + total_tokens += content_tokens + + # Add tokens for role and message structure + role_tokens = len(self.tokenizer.encode(role, disallowed_special=())) + total_tokens += role_tokens + 4 # Extra tokens for message formatting + + return total_tokens + + except Exception as e: + self.logger.warning(f"Token calculation failed: {e}") + # Fallback estimation + total_chars = sum(len(str(msg.get("content", ""))) for msg in messages) + return total_chars // 4 + + def should_trigger_summary_by_tokens(self, messages: List[Dict]) -> bool: + """ + Check if summary should be triggered based on token count + + Args: + messages: Current conversation messages + + Returns: + True if summary should be triggered based on token count + """ + if not messages: + return False + + # Calculate current token count / ่ฎก็ฎ—ๅฝ“ๅ‰tokenๆ•ฐ + current_token_count = self.calculate_messages_token_count(messages) + + # Check if we should trigger summary / ๆฃ€ๆŸฅๆ˜ฏๅฆๅบ”่งฆๅ‘ๆ€ป็ป“ + should_trigger = ( + current_token_count > self.summary_trigger_tokens + and current_token_count + > self.last_summary_token_count + + 10000 # Minimum 10k tokens between summaries / ๆ€ป็ป“้—ดๆœ€ๅฐ‘10k tokens + ) + + if should_trigger: + self.logger.info( + f"Token-based summary trigger: current={current_token_count:,}, " + f"threshold={self.summary_trigger_tokens:,}, " + f"last_summary={self.last_summary_token_count:,}" + ) + + return should_trigger + + def should_trigger_summary( + self, summary_trigger: int = 5, messages: List[Dict] = None + ) -> bool: + """ + Check if summary should be triggered based on token count (preferred) or file count (fallback) + ๆ นๆฎtokenๆ•ฐ๏ผˆ้ฆ–้€‰๏ผ‰ๆˆ–ๆ–‡ไปถๆ•ฐ๏ผˆๅ›ž้€€๏ผ‰ๆฃ€ๆŸฅๆ˜ฏๅฆๅบ”่งฆๅ‘ๆ€ป็ป“ + + Args: + summary_trigger: Number of files after which to trigger summary (fallback) + messages: Current conversation messages for token calculation + + Returns: + True if summary should be triggered + """ + # Primary: Token-based triggering / ไธป่ฆ๏ผšๅŸบไบŽtoken็š„่งฆๅ‘ + if messages and self.tokenizer: + return self.should_trigger_summary_by_tokens(messages) + + # Fallback: File-based triggering (original logic) / ๅ›ž้€€๏ผšๅŸบไบŽๆ–‡ไปถ็š„่งฆๅ‘๏ผˆๅŽŸๅง‹้€ป่พ‘๏ผ‰ + self.logger.info("Using fallback file-based summary triggering") + should_trigger = ( + self.files_implemented_count > 0 + and self.files_implemented_count % summary_trigger == 0 + and self.files_implemented_count > self.last_summary_file_count + ) + + return should_trigger + + def mark_summary_triggered(self, messages: List[Dict] = None): + """ + Mark that summary has been triggered for current state + ๆ ‡่ฎฐๅฝ“ๅ‰็Šถๆ€็š„ๆ€ป็ป“ๅทฒ่ขซ่งฆๅ‘ + + Args: + messages: Current conversation messages for token tracking + """ + # Update file-based tracking / ๆ›ดๆ–ฐๅŸบไบŽๆ–‡ไปถ็š„่ทŸ่ธช + self.last_summary_file_count = self.files_implemented_count + + # Update token-based tracking / ๆ›ดๆ–ฐๅŸบไบŽtoken็š„่ทŸ่ธช + if messages and self.tokenizer: + self.last_summary_token_count = self.calculate_messages_token_count( + messages + ) + self.logger.info( + f"Summary marked as triggered - file_count: {self.files_implemented_count}, " + f"token_count: {self.last_summary_token_count:,}" + ) + else: + self.logger.info( + f"Summary marked as triggered for file count: {self.files_implemented_count}" + ) + + def get_implementation_summary(self) -> Dict[str, Any]: + """ + Get current implementation summary + ่Žทๅ–ๅฝ“ๅ‰ๅฎž็Žฐๆ€ป็ป“ + """ + return self.implementation_summary.copy() + + def get_files_implemented_count(self) -> int: + """ + Get the number of files implemented so far + ่Žทๅ–ๅˆฐ็›ฎๅ‰ไธบๆญขๅฎž็Žฐ็š„ๆ–‡ไปถๆ•ฐ้‡ + """ + return self.files_implemented_count + + def get_read_tools_status(self) -> Dict[str, Any]: + """ + Get read tools configuration status + ่Žทๅ–่ฏปๅ–ๅทฅๅ…ท้…็ฝฎ็Šถๆ€ + + Returns: + Dictionary with read tools status information + """ + return { + "read_tools_enabled": self.enable_read_tools, + "status": "ENABLED" if self.enable_read_tools else "DISABLED", + "tools_affected": ["read_file", "read_code_mem"], + "description": "Read tools configuration for testing purposes", + } + + def add_technical_decision(self, decision: str, context: str = ""): + """ + Add a technical decision to the implementation summary + ๅ‘ๅฎž็Žฐๆ€ป็ป“ๆทปๅŠ ๆŠ€ๆœฏๅ†ณ็ญ– + + Args: + decision: Description of the technical decision + context: Additional context for the decision + """ + self.implementation_summary["technical_decisions"].append( + {"decision": decision, "context": context, "timestamp": time.time()} + ) + self.logger.info(f"Technical decision recorded: {decision}") + + def add_constraint(self, constraint: str, impact: str = ""): + """ + Add an important constraint to the implementation summary + ๅ‘ๅฎž็Žฐๆ€ป็ป“ๆทปๅŠ ้‡่ฆ็บฆๆŸ + + Args: + constraint: Description of the constraint + impact: Impact of the constraint on implementation + """ + self.implementation_summary["important_constraints"].append( + {"constraint": constraint, "impact": impact, "timestamp": time.time()} + ) + self.logger.info(f"Constraint recorded: {constraint}") + + def add_architecture_note(self, note: str, component: str = ""): + """ + Add an architecture note to the implementation summary + ๅ‘ๅฎž็Žฐๆ€ป็ป“ๆทปๅŠ ๆžถๆž„ๆณจ้‡Š + + Args: + note: Architecture note description + component: Related component or module + """ + self.implementation_summary["architecture_notes"].append( + {"note": note, "component": component, "timestamp": time.time()} + ) + self.logger.info(f"Architecture note recorded: {note}") + + def get_implementation_statistics(self) -> Dict[str, Any]: + """ + Get comprehensive implementation statistics + ่Žทๅ–ๅ…จ้ข็š„ๅฎž็Žฐ็ปŸ่ฎกไฟกๆฏ + """ + return { + "total_files_implemented": self.files_implemented_count, + "files_implemented_count": self.files_implemented_count, + "technical_decisions_count": len( + self.implementation_summary["technical_decisions"] + ), + "constraints_count": len( + self.implementation_summary["important_constraints"] + ), + "architecture_notes_count": len( + self.implementation_summary["architecture_notes"] + ), + "dependency_analysis_count": len( + self.implementation_summary["dependency_analysis"] + ), + "files_read_for_dependencies": len(self.files_read_for_dependencies), + "unique_files_implemented": len(self.implemented_files_set), + "completed_files_list": [ + f["file"] for f in self.implementation_summary["completed_files"] + ], + "dependency_files_read": list(self.files_read_for_dependencies), + "last_summary_file_count": self.last_summary_file_count, + "read_tools_status": self.get_read_tools_status(), # Include read tools configuration + } + + def force_enable_optimization(self): + """ + Force enable optimization for testing purposes + ๅผบๅˆถๅฏ็”จไผ˜ๅŒ–็”จไบŽๆต‹่ฏ•็›ฎ็š„ + """ + self.files_implemented_count = 1 + self.logger.info( + f"๐Ÿ”ง OPTIMIZATION FORCE ENABLED: files_implemented_count set to {self.files_implemented_count}" + ) + print( + f"๐Ÿ”ง OPTIMIZATION FORCE ENABLED: files_implemented_count set to {self.files_implemented_count}" + ) + + def reset_implementation_tracking(self): + """ + Reset implementation tracking (useful for new sessions) + ้‡็ฝฎๅฎž็Žฐ่ทŸ่ธช๏ผˆๅฏนๆ–ฐไผš่ฏๆœ‰็”จ๏ผ‰ + """ + self.implementation_summary = { + "completed_files": [], + "technical_decisions": [], + "important_constraints": [], + "architecture_notes": [], + "dependency_analysis": [], # Reset dependency analysis and file reads + } + self.files_implemented_count = 0 + self.implemented_files_set = ( + set() + ) # Reset the unique files set / ้‡็ฝฎๅ”ฏไธ€ๆ–‡ไปถ้›†ๅˆ + self.files_read_for_dependencies = ( + set() + ) # Reset files read for dependency analysis / ้‡็ฝฎไธบไพ่ต–ๅˆ†ๆž่€Œ่ฏปๅ–็š„ๆ–‡ไปถ + self.last_summary_file_count = 0 # Reset the file count when last summary was triggered / ้‡็ฝฎไธŠๆฌก่งฆๅ‘ๆ€ป็ป“ๆ—ถ็š„ๆ–‡ไปถๆ•ฐ + self.last_summary_token_count = 0 # Reset token count when last summary was triggered / ้‡็ฝฎไธŠๆฌก่งฆๅ‘ๆ€ป็ป“ๆ—ถ็š„tokenๆ•ฐ + self.logger.info("Implementation tracking reset") + + # Reset analysis loop detection / ้‡็ฝฎๅˆ†ๆžๅพช็Žฏๆฃ€ๆต‹ + self.recent_tool_calls = [] + self.logger.info("Analysis loop detection reset") + + def _track_tool_call_for_loop_detection(self, tool_name: str): + """ + Track tool calls for analysis loop detection + ่ทŸ่ธชๅทฅๅ…ท่ฐƒ็”จไปฅๆฃ€ๆต‹ๅˆ†ๆžๅพช็Žฏ + + Args: + tool_name: Name of the tool called + """ + self.recent_tool_calls.append(tool_name) + if len(self.recent_tool_calls) > self.max_read_without_write: + self.recent_tool_calls.pop(0) + + if len(set(self.recent_tool_calls)) == 1: + self.logger.warning("Analysis loop detected") + + def is_in_analysis_loop(self) -> bool: + """ + Check if the agent is in an analysis loop (only reading files, not writing) + ๆฃ€ๆŸฅไปฃ็†ๆ˜ฏๅฆๅœจๅˆ†ๆžๅพช็Žฏไธญ๏ผˆๅช่ฏปๆ–‡ไปถ๏ผŒไธๅ†™ๆ–‡ไปถ๏ผ‰ + + Returns: + True if in analysis loop + """ + if len(self.recent_tool_calls) < self.max_read_without_write: + return False + + # Check if recent calls are all read_file or search_reference_code / ๆฃ€ๆŸฅๆœ€่ฟ‘็š„่ฐƒ็”จๆ˜ฏๅฆ้ƒฝๆ˜ฏread_fileๆˆ–search_reference_code + analysis_tools = { + "read_file", + "search_reference_code", + "get_all_available_references", + } + recent_calls_set = set(self.recent_tool_calls) + + # If all recent calls are analysis tools, we're in an analysis loop / ๅฆ‚ๆžœๆœ€่ฟ‘็š„่ฐƒ็”จ้ƒฝๆ˜ฏๅˆ†ๆžๅทฅๅ…ท๏ผŒๆˆ‘ไปฌๅœจๅˆ†ๆžๅพช็Žฏไธญ + in_loop = ( + recent_calls_set.issubset(analysis_tools) and len(recent_calls_set) >= 1 + ) + + if in_loop: + self.logger.warning( + f"Analysis loop detected! Recent calls: {self.recent_tool_calls}" + ) + + return in_loop + + def get_analysis_loop_guidance(self) -> str: + """ + Get guidance to break out of analysis loop + + Returns: + Guidance message to encourage implementation + """ + return f"""๐Ÿšจ **ANALYSIS LOOP DETECTED - IMMEDIATE ACTION REQUIRED** + +**Problem**: You've been reading/analyzing files for {len(self.recent_tool_calls)} consecutive calls without writing code. +**Recent tool calls**: {' โ†’ '.join(self.recent_tool_calls)} + +**SOLUTION - IMPLEMENT CODE NOW**: +1. **STOP ANALYZING** - You have enough information +2. **Use write_file** to create the next code file according to the implementation plan +3. **Choose ANY file** from the plan that hasn't been implemented yet +4. **Write complete, working code** - don't ask for permission or clarification + +**Files implemented so far**: {self.files_implemented_count} +**Your goal**: Implement MORE files, not analyze existing ones! + +**CRITICAL**: Your next response MUST use write_file to create a new code file!""" + + async def test_summary_functionality(self, test_file_path: str = None): + """ + Test if the code summary functionality is working correctly + ๆต‹่ฏ•ไปฃ็ ๆ€ป็ป“ๅŠŸ่ƒฝๆ˜ฏๅฆๆญฃๅธธๅทฅไฝœ + + Args: + test_file_path: Specific file to test, if None will test all implemented files + """ + if not self.memory_agent: + self.logger.warning("No memory agent available for testing") + return + + if test_file_path: + files_to_test = [test_file_path] + else: + # Use implemented files from tracking + files_to_test = list(self.implemented_files_set)[ + :3 + ] # Limit to first 3 files + + if not files_to_test: + self.logger.warning("No implemented files to test") + return + + # Test each file silently + summary_files_found = 0 + + for file_path in files_to_test: + if self.mcp_agent: + try: + result = await self.mcp_agent.call_tool( + "read_code_mem", {"file_paths": [file_path]} + ) + + # Parse the result to check if summary was found + import json + + result_data = ( + json.loads(result) if isinstance(result, str) else result + ) + + if ( + result_data.get("status") + in ["all_summaries_found", "partial_summaries_found"] + and result_data.get("summaries_found", 0) > 0 + ): + summary_files_found += 1 + except Exception as e: + self.logger.warning( + f"Failed to test read_code_mem for {file_path}: {e}" + ) + else: + self.logger.warning("MCP agent not available for testing") + + self.logger.info( + f"๐Ÿ“‹ Summary testing: {summary_files_found}/{len(files_to_test)} files have summaries" + ) + + async def test_automatic_read_file_optimization(self): + """ + Test the automatic read_file optimization that redirects to read_code_mem + ๆต‹่ฏ•่‡ชๅŠจread_fileไผ˜ๅŒ–๏ผŒ้‡ๅฎšๅ‘ๅˆฐread_code_mem + """ + print("=" * 80) + print("๐Ÿ”„ TESTING AUTOMATIC READ_FILE OPTIMIZATION") + print("=" * 80) + + # Simulate that at least one file has been implemented (to trigger optimization) + self.files_implemented_count = 1 + + # Test with a generic config file that should have a summary + test_file = "config.py" + + print(f"๐Ÿ“ Testing automatic optimization for: {test_file}") + print(f"๐Ÿ“Š Files implemented count: {self.files_implemented_count}") + print( + f"๐Ÿ”ง Optimization should be: {'ENABLED' if self.files_implemented_count > 0 else 'DISABLED'}" + ) + + # Create a simulated read_file tool call + simulated_read_file_call = { + "id": "test_read_file_optimization", + "name": "read_file", + "input": {"file_path": test_file}, + } + + print("\n๐Ÿ”„ Simulating read_file call:") + print(f" Tool: {simulated_read_file_call['name']}") + print(f" File: {simulated_read_file_call['input']['file_path']}") + + # Execute the tool call (this should trigger automatic optimization) + results = await self.execute_tool_calls([simulated_read_file_call]) + + if results: + result = results[0] + print("\nโœ… Tool execution completed:") + print(f" Tool name: {result.get('tool_name', 'N/A')}") + print(f" Tool ID: {result.get('tool_id', 'N/A')}") + + # Parse the result to check if optimization occurred + import json + + try: + result_data = json.loads(result.get("result", "{}")) + if result_data.get("optimization") == "redirected_to_read_code_mem": + print("๐ŸŽ‰ SUCCESS: read_file was automatically optimized!") + print( + f" Original tool: {result_data.get('original_tool', 'N/A')}" + ) + print(f" Status: {result_data.get('status', 'N/A')}") + elif result_data.get("status") == "summary_found": + print("๐ŸŽ‰ SUCCESS: Summary was found and returned!") + else: + print("โ„น๏ธ INFO: No optimization occurred (no summary available)") + except json.JSONDecodeError: + print("โš ๏ธ WARNING: Could not parse result as JSON") + else: + print("โŒ ERROR: No results returned from tool execution") + + print("\n" + "=" * 80) + print("๐Ÿ”„ AUTOMATIC READ_FILE OPTIMIZATION TEST COMPLETE") + print("=" * 80) + + async def test_summary_optimization(self, test_file_path: str = "config.py"): + """ + Test the summary optimization functionality with a specific file + ๆต‹่ฏ•็‰นๅฎšๆ–‡ไปถ็š„ๆ€ป็ป“ไผ˜ๅŒ–ๅŠŸ่ƒฝ + + Args: + test_file_path: File path to test (default: config.py which should be in summary) + """ + if not self.mcp_agent: + return False + + try: + # Use MCP agent to call read_code_mem tool + result = await self.mcp_agent.call_tool( + "read_code_mem", {"file_paths": [test_file_path]} + ) + + # Parse the result to check if summary was found + import json + + result_data = json.loads(result) if isinstance(result, str) else result + + return ( + result_data.get("status") + in ["all_summaries_found", "partial_summaries_found"] + and result_data.get("summaries_found", 0) > 0 + ) + except Exception as e: + self.logger.warning(f"Failed to test read_code_mem optimization: {e}") + return False + + async def test_read_tools_configuration(self): + """ + Test the read tools configuration to verify enabling/disabling works correctly + ๆต‹่ฏ•่ฏปๅ–ๅทฅๅ…ท้…็ฝฎไปฅ้ชŒ่ฏๅฏ็”จ/็ฆ็”จๆ˜ฏๅฆๆญฃๅธธๅทฅไฝœ + """ + print("=" * 60) + print("๐Ÿงช TESTING READ TOOLS CONFIGURATION") + print("=" * 60) + + status = self.get_read_tools_status() + print(f"Read tools enabled: {status['read_tools_enabled']}") + print(f"Status: {status['status']}") + print(f"Tools affected: {status['tools_affected']}") + + # Test with mock tool calls + test_tools = [ + { + "id": "test_read_file", + "name": "read_file", + "input": {"file_path": "test.py"}, + }, + { + "id": "test_read_code_mem", + "name": "read_code_mem", + "input": {"file_path": "test.py"}, + }, + { + "id": "test_write_file", + "name": "write_file", + "input": {"file_path": "test.py", "content": "# test"}, + }, + ] + + print( + f"\n๐Ÿ”„ Testing tool execution with read_tools_enabled={self.enable_read_tools}" + ) + + for tool_call in test_tools: + tool_name = tool_call["name"] + if not self.enable_read_tools and tool_name in [ + "read_file", + "read_code_mem", + ]: + print(f"๐Ÿšซ {tool_name}: Would be SKIPPED (disabled)") + else: + print(f"โœ… {tool_name}: Would be EXECUTED") + + print("=" * 60) + print("๐Ÿงช READ TOOLS CONFIGURATION TEST COMPLETE") + print("=" * 60) + + return status diff --git a/DeepCode/workflows/agents/document_segmentation_agent.py b/DeepCode/workflows/agents/document_segmentation_agent.py new file mode 100644 index 00000000..06cd03bc --- /dev/null +++ b/DeepCode/workflows/agents/document_segmentation_agent.py @@ -0,0 +1,353 @@ +""" +Document Segmentation Agent + +A lightweight agent that coordinates with the document segmentation MCP server +to analyze document structure and prepare segments for other agents. +""" + +import os +import logging +from typing import Dict, Any, Optional + +from mcp_agent.agents.agent import Agent +from utils.llm_utils import get_preferred_llm_class + + +class DocumentSegmentationAgent: + """ + Intelligent document segmentation agent with semantic analysis capabilities. + + This enhanced agent provides: + 1. **Semantic Document Classification**: Content-based document type identification + 2. **Adaptive Segmentation Strategy**: Algorithm integrity and semantic coherence preservation + 3. **Planning Agent Optimization**: Segment preparation specifically optimized for downstream agents + 4. **Quality Intelligence Validation**: Advanced metrics for completeness and technical accuracy + 5. **Algorithm Completeness Protection**: Ensures critical algorithms and formulas remain intact + + Key improvements over traditional segmentation: + - Semantic content analysis vs mechanical structure splitting + - Dynamic character limits based on content complexity + - Enhanced relevance scoring for planning agents + - Algorithm and formula integrity preservation + - Content type-aware segmentation strategies + """ + + def __init__(self, logger: Optional[logging.Logger] = None): + self.logger = logger or self._create_default_logger() + self.mcp_agent = None + + def _create_default_logger(self) -> logging.Logger: + """Create default logger if none provided""" + logger = logging.getLogger(f"{__name__}.DocumentSegmentationAgent") + logger.setLevel(logging.INFO) + return logger + + async def __aenter__(self): + """Async context manager entry""" + await self.initialize() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Async context manager exit""" + await self.cleanup() + + async def initialize(self): + """Initialize the MCP agent connection""" + try: + self.mcp_agent = Agent( + name="DocumentSegmentationCoordinator", + instruction="""You are an intelligent document segmentation coordinator that leverages advanced semantic analysis for optimal document processing. + +Your enhanced capabilities include: +1. **Semantic Content Analysis**: Coordinate intelligent document type classification based on content semantics rather than structural patterns +2. **Algorithm Integrity Protection**: Ensure algorithm blocks, formulas, and related content maintain logical coherence +3. **Adaptive Segmentation Strategy**: Select optimal segmentation approaches (semantic_research_focused, algorithm_preserve_integrity, concept_implementation_hybrid, etc.) +4. **Quality Intelligence Validation**: Assess segmentation quality using enhanced metrics for completeness, relevance, and technical accuracy +5. **Planning Agent Optimization**: Ensure segments are specifically optimized for ConceptAnalysisAgent, AlgorithmAnalysisAgent, and CodePlannerAgent needs + +**Key Principles**: +- Prioritize content semantics over mechanical structure +- Preserve algorithm and formula completeness +- Optimize for downstream agent token efficiency +- Ensure technical content integrity +- Provide actionable quality assessments + +Use the enhanced document-segmentation tools to deliver superior segmentation results that significantly improve planning agent performance.""", + server_names=["document-segmentation"], + ) + + # Initialize the agent context + await self.mcp_agent.__aenter__() + + # Attach LLM + self.llm = await self.mcp_agent.attach_llm(get_preferred_llm_class()) + + self.logger.info("DocumentSegmentationAgent initialized successfully") + + except Exception as e: + self.logger.error(f"Failed to initialize DocumentSegmentationAgent: {e}") + raise + + async def cleanup(self): + """Cleanup resources""" + if self.mcp_agent: + try: + await self.mcp_agent.__aexit__(None, None, None) + except Exception as e: + self.logger.warning(f"Error during cleanup: {e}") + + async def analyze_and_prepare_document( + self, paper_dir: str, force_refresh: bool = False + ) -> Dict[str, Any]: + """ + Perform intelligent semantic analysis and create optimized document segments. + + This method coordinates with the enhanced document segmentation server to: + - Classify document type using semantic content analysis + - Select optimal segmentation strategy (semantic_research_focused, algorithm_preserve_integrity, etc.) + - Preserve algorithm and formula integrity + - Optimize segments for downstream planning agents + + Args: + paper_dir: Path to the paper directory + force_refresh: Whether to force re-analysis with latest algorithms + + Returns: + Dict containing enhanced analysis results and intelligent segment information + """ + try: + self.logger.info(f"Starting document analysis for: {paper_dir}") + + # Check if markdown file exists + md_files = [f for f in os.listdir(paper_dir) if f.endswith(".md")] + if not md_files: + raise ValueError(f"No markdown file found in {paper_dir}") + + # Use the enhanced document segmentation tool + message = f"""Please perform intelligent semantic analysis and segmentation for the document in directory: {paper_dir} + +Use the analyze_and_segment_document tool with these parameters: +- paper_dir: {paper_dir} +- force_refresh: {force_refresh} + +**Focus on these enhanced objectives**: +1. **Semantic Document Classification**: Identify document type using content semantics (research_paper, algorithm_focused, technical_doc, etc.) +2. **Intelligent Segmentation Strategy**: Select the optimal strategy based on content analysis: + - `semantic_research_focused` for research papers with high algorithm density + - `algorithm_preserve_integrity` for algorithm-heavy documents + - `concept_implementation_hybrid` for mixed concept/implementation content +3. **Algorithm Completeness**: Ensure algorithm blocks, formulas, and related descriptions remain logically connected +4. **Planning Agent Optimization**: Create segments that maximize effectiveness for ConceptAnalysisAgent, AlgorithmAnalysisAgent, and CodePlannerAgent + +After segmentation, get a document overview and provide: +- Quality assessment of semantic segmentation approach +- Algorithm/formula integrity verification +- Recommendations for planning agent optimization +- Technical content completeness evaluation""" + + result = await self.llm.generate_str(message=message) + + self.logger.info("Document analysis completed successfully") + + # Parse the result and return structured information + return { + "status": "success", + "paper_dir": paper_dir, + "analysis_result": result, + "segments_available": True, + } + + except Exception as e: + self.logger.error(f"Error in document analysis: {e}") + return { + "status": "error", + "paper_dir": paper_dir, + "error_message": str(e), + "segments_available": False, + } + + async def get_document_overview(self, paper_dir: str) -> Dict[str, Any]: + """ + Get overview of document structure and segments. + + Args: + paper_dir: Path to the paper directory + + Returns: + Dict containing document overview information + """ + try: + message = f"""Please provide an intelligent overview of the enhanced document segmentation for: {paper_dir} + +Use the get_document_overview tool to retrieve: +- **Semantic Document Classification**: Document type and confidence score +- **Adaptive Segmentation Strategy**: Strategy used and reasoning +- **Segment Intelligence**: Total segments with enhanced metadata +- **Content Type Distribution**: Breakdown by algorithm, concept, formula, implementation content +- **Quality Intelligence Assessment**: Completeness, coherence, and planning agent optimization + +Provide a comprehensive analysis focusing on: +1. Semantic vs structural segmentation quality +2. Algorithm and formula integrity preservation +3. Segment relevance for downstream planning agents +4. Technical content distribution and completeness""" + + result = await self.llm.generate_str(message=message) + + return { + "status": "success", + "paper_dir": paper_dir, + "overview_result": result, + } + + except Exception as e: + self.logger.error(f"Error getting document overview: {e}") + return {"status": "error", "paper_dir": paper_dir, "error_message": str(e)} + + async def validate_segmentation_quality(self, paper_dir: str) -> Dict[str, Any]: + """ + Validate the quality of document segmentation. + + Args: + paper_dir: Path to the paper directory + + Returns: + Dict containing validation results + """ + try: + # Get overview first + overview_result = await self.get_document_overview(paper_dir) + + if overview_result["status"] != "success": + return overview_result + + # Analyze enhanced segmentation quality + message = f"""Based on the intelligent document overview for {paper_dir}, please evaluate the enhanced segmentation quality using advanced criteria. + +**Enhanced Quality Assessment Factors**: +1. **Semantic Coherence**: Do segments maintain logical content boundaries vs mechanical structural splits? +2. **Algorithm Integrity**: Are algorithm blocks, formulas, and related explanations kept together? +3. **Content Type Optimization**: Are different content types (algorithm, concept, formula, implementation) properly identified and scored? +4. **Planning Agent Effectiveness**: Will ConceptAnalysisAgent, AlgorithmAnalysisAgent, and CodePlannerAgent receive optimal information? +5. **Dynamic Sizing**: Are segments adaptively sized based on content complexity rather than fixed limits? +6. **Technical Completeness**: Are critical technical details preserved without fragmentation? + +**Provide specific recommendations for**: +- Semantic segmentation improvements +- Algorithm/formula integrity enhancements +- Planning agent optimization opportunities +- Content distribution balance adjustments""" + + validation_result = await self.llm.generate_str(message=message) + + return { + "status": "success", + "paper_dir": paper_dir, + "validation_result": validation_result, + "overview_data": overview_result, + } + + except Exception as e: + self.logger.error(f"Error validating segmentation quality: {e}") + return {"status": "error", "paper_dir": paper_dir, "error_message": str(e)} + + +async def run_document_segmentation_analysis( + paper_dir: str, logger: Optional[logging.Logger] = None, force_refresh: bool = False +) -> Dict[str, Any]: + """ + Convenience function to run document segmentation analysis. + + Args: + paper_dir: Path to the paper directory + logger: Optional logger instance + force_refresh: Whether to force re-analysis + + Returns: + Dict containing analysis results + """ + async with DocumentSegmentationAgent(logger=logger) as agent: + # Analyze and prepare document + analysis_result = await agent.analyze_and_prepare_document( + paper_dir, force_refresh=force_refresh + ) + + if analysis_result["status"] == "success": + # Validate segmentation quality + validation_result = await agent.validate_segmentation_quality(paper_dir) + analysis_result["validation"] = validation_result + + return analysis_result + + +# Utility function for integration with existing workflow +async def prepare_document_segments( + paper_dir: str, logger: Optional[logging.Logger] = None +) -> Dict[str, Any]: + """ + Prepare intelligent document segments optimized for planning agents. + + This enhanced function leverages semantic analysis to create segments that: + - Preserve algorithm and formula integrity + - Optimize for ConceptAnalysisAgent, AlgorithmAnalysisAgent, and CodePlannerAgent + - Use adaptive character limits based on content complexity + - Maintain technical content completeness + + Called from the orchestration engine (Phase 3.5) to prepare documents + before the planning phase with superior segmentation quality. + + Args: + paper_dir: Path to the paper directory containing markdown file + logger: Optional logger instance for tracking + + Returns: + Dict containing enhanced preparation results and intelligent metadata + """ + try: + logger = logger or logging.getLogger(__name__) + logger.info(f"Preparing document segments for: {paper_dir}") + + # Run analysis + result = await run_document_segmentation_analysis( + paper_dir=paper_dir, + logger=logger, + force_refresh=False, # Use cached analysis if available + ) + + if result["status"] == "success": + logger.info("Document segments prepared successfully") + + # Create metadata for downstream agents + segments_dir = os.path.join(paper_dir, "document_segments") + + return { + "status": "success", + "paper_dir": paper_dir, + "segments_dir": segments_dir, + "segments_ready": True, + "analysis_summary": result.get("analysis_result", ""), + "validation_summary": result.get("validation", {}).get( + "validation_result", "" + ), + } + else: + logger.error( + f"Document segmentation failed: {result.get('error_message', 'Unknown error')}" + ) + return { + "status": "error", + "paper_dir": paper_dir, + "segments_ready": False, + "error_message": result.get( + "error_message", "Document segmentation failed" + ), + } + + except Exception as e: + logger.error(f"Error preparing document segments: {e}") + return { + "status": "error", + "paper_dir": paper_dir, + "segments_ready": False, + "error_message": str(e), + } diff --git a/DeepCode/workflows/agents/memory_agent_concise.py b/DeepCode/workflows/agents/memory_agent_concise.py new file mode 100644 index 00000000..e55301d3 --- /dev/null +++ b/DeepCode/workflows/agents/memory_agent_concise.py @@ -0,0 +1,2155 @@ +""" +Concise Memory Agent for Code Implementation Workflow + +This memory agent implements a focused approach: +1. Before first file: Normal conversation flow +2. After first file: Keep only system_prompt + initial_plan + current round tool results +3. Clean slate for each new code file generation + +Key Features: +- Preserves system prompt and initial plan always +- After first file generation, discards previous conversation history +- Keeps only current round tool results from essential tools: + * read_code_mem, read_file, write_file + * execute_python, execute_bash + * search_code, search_reference_code, get_file_structure +- Provides clean, focused input for next write_file operation +""" + +import json +import logging +import os +import time +from datetime import datetime +from typing import Dict, Any, List, Optional + + +class ConciseMemoryAgent: + """ + Concise Memory Agent - Focused Information Retention + + Core Philosophy: + - Preserve essential context (system prompt + initial plan) + - After first file generation, use clean slate approach + - Keep only current round tool results from all essential MCP tools + - Remove conversational clutter and previous tool calls + + Essential Tools Tracked: + - File Operations: read_code_mem, read_file, write_file + - Code Analysis: search_code, search_reference_code, get_file_structure + - Execution: execute_python, execute_bash + """ + + def __init__( + self, + initial_plan_content: str, + logger: Optional[logging.Logger] = None, + target_directory: Optional[str] = None, + default_models: Optional[Dict[str, str]] = None, + code_directory: Optional[str] = None, + ): + """ + Initialize Concise Memory Agent + + Args: + initial_plan_content: Content of initial_plan.txt + logger: Logger instance + target_directory: Target directory for saving summaries + default_models: Default models configuration from workflow + code_directory: Generated code directory path (e.g., target_directory/generate_code) + """ + self.logger = logger or self._create_default_logger() + self.initial_plan = initial_plan_content + + # Store default models configuration + self.default_models = default_models or { + "anthropic": "claude-sonnet-4-20250514", + "openai": "o3-mini", + "google": "gemini-2.0-flash", + } + + # Memory state tracking - new logic: trigger after each write_file + self.last_write_file_detected = ( + False # Track if write_file was called in current iteration + ) + self.should_clear_memory_next = False # Flag to clear memory in next round + self.current_round = 0 + + # Parse phase structure from initial plan + self.phase_structure = self._parse_phase_structure() + + # Memory configuration + if target_directory: + self.save_path = target_directory + else: + self.save_path = "./deepcode_lab/papers/1/" + + # Store code directory for file extraction + self.code_directory = code_directory or os.path.join( + self.save_path, "generate_code" + ) + + # Extract all files - prioritize generated directory over plan parsing + self.all_files_list = self._extract_all_files() + + # Code summary file path + self.code_summary_path = os.path.join( + self.save_path, "implement_code_summary.md" + ) + + # Current round tool results storage + self.current_round_tool_results = [] + + # Track all implemented files + self.implemented_files = [] + + # Store Next Steps information temporarily (not saved to file) + self.current_next_steps = "" + + self.logger.info( + f"Concise Memory Agent initialized with target directory: {self.save_path}" + ) + self.logger.info(f"Code directory: {self.code_directory}") + self.logger.info(f"Code summary will be saved to: {self.code_summary_path}") + # self.logger.info(f"๐Ÿค– Using models - Anthropic: {self.default_models['anthropic']}, OpenAI: {self.default_models['openai']}") + self.logger.info( + "๐Ÿ“ NEW LOGIC: Memory clearing triggered after each write_file call" + ) + + def _create_default_logger(self) -> logging.Logger: + """Create default logger""" + logger = logging.getLogger(f"{__name__}.ConciseMemoryAgent") + logger.setLevel(logging.INFO) + return logger + + def _parse_phase_structure(self) -> Dict[str, List[str]]: + """Parse implementation phases from initial plan""" + try: + phases = {} + lines = self.initial_plan.split("\n") + current_phase = None + + for line in lines: + if "Phase" in line and ":" in line: + # Extract phase name + phase_parts = line.split(":") + if len(phase_parts) >= 2: + current_phase = phase_parts[0].strip() + phases[current_phase] = [] + elif current_phase and line.strip().startswith("-"): + # This is a file in the current phase + file_line = line.strip()[1:].strip() + if file_line.startswith("`") and file_line.endswith("`"): + file_name = file_line[1:-1] + phases[current_phase].append(file_name) + elif current_phase and not line.strip(): + # Empty line might indicate end of phase + continue + elif current_phase and line.strip().startswith("###"): + # New section, end current phase + current_phase = None + + return phases + + except Exception as e: + self.logger.warning(f"Failed to parse phase structure: {e}") + return {} + + def _extract_all_files(self) -> List[str]: + """ + Extract all code files - prioritizes generated directory over plan parsing + + Strategy: + 1. First try to extract from the generated code directory (reliable) + 2. Fall back to plan parsing if directory doesn't exist yet + + Returns: + List of all file paths that should be implemented + """ + # Try extracting from generated directory first (more reliable) + if os.path.exists(self.code_directory): + files_from_dir = self._extract_files_from_generated_directory() + if files_from_dir: + self.logger.info( + f"๐Ÿ“ Extracted {len(files_from_dir)} files from generated directory" + ) + return files_from_dir + + # Fall back to plan parsing + self.logger.info( + "๐Ÿ“ Generated directory not found, extracting from plan (less reliable)" + ) + return self._extract_all_files_from_plan() + + def _extract_files_from_generated_directory(self) -> List[str]: + """ + Extract all code files from the generated code directory + This is more reliable than parsing the LLM-generated plan + + Returns: + List of relative file paths within the code directory + """ + code_files = [] + + # Define code file extensions to track + code_extensions = { + ".py", + ".js", + ".ts", + ".jsx", + ".tsx", + ".vue", + ".html", + ".css", + ".scss", + ".sass", + ".less", + ".json", + ".yaml", + ".yml", + ".toml", + ".xml", + ".ini", + ".cfg", + ".md", + ".rst", + ".txt", + ".sh", + ".bash", + ".zsh", + ".bat", + ".ps1", + ".cmd", + ".c", + ".cpp", + ".h", + ".hpp", + ".cc", + ".cxx", + ".java", + ".kt", + ".scala", + ".go", + ".rs", + ".php", + ".rb", + ".pl", + ".lua", + ".r", + ".sql", + } + + # Files and directories to exclude + exclude_patterns = { + "__pycache__", + ".pyc", + "node_modules", + ".git", + ".vscode", + ".idea", + "dist", + "build", + "output", + ".egg-info", + "venv", + ".venv", + "env", + ".env", + } + + try: + for root, dirs, files in os.walk(self.code_directory): + # Filter out excluded directories + dirs[:] = [ + d + for d in dirs + if d not in exclude_patterns and not d.startswith(".") + ] + + for file in files: + # Skip hidden files and excluded patterns + if file.startswith("."): + continue + + # Check if file has a code extension + has_code_ext = any( + file.lower().endswith(ext) for ext in code_extensions + ) + if not has_code_ext: + continue + + # Get full path and convert to relative path + full_path = os.path.join(root, file) + relative_path = os.path.relpath(full_path, self.code_directory) + + # Normalize path separators + relative_path = relative_path.replace(os.sep, "/") + + code_files.append(relative_path) + + # Sort for consistency + code_files = sorted(code_files) + + if code_files: + self.logger.info(f"๐Ÿ“„ Found {len(code_files)} code files in directory") + self.logger.info(f"๐Ÿ“„ Sample files: {code_files[:3]}...") + + return code_files + + except Exception as e: + self.logger.error(f"Failed to extract files from directory: {e}") + return [] + + def _extract_all_files_from_plan(self) -> List[str]: + """ + Extract all file paths from the file_structure section in initial plan + Handles multiple formats: tree structure, YAML, and simple lists + + Returns: + List of all file paths that should be implemented + """ + try: + lines = self.initial_plan.split("\n") + files = [] + + # Method 1: Try to extract from tree structure in file_structure section + files.extend(self._extract_from_tree_structure(lines)) + + # Method 2: If no files found, try to extract from simple list format + if not files: + files.extend(self._extract_from_simple_list(lines)) + + # Method 3: If still no files, try to extract from anywhere in the plan + if not files: + files.extend(self._extract_from_plan_content(lines)) + + # Clean and validate file paths + cleaned_files = self._clean_and_validate_files(files) + + # Log the extracted files + self.logger.info( + f"๐Ÿ“ Extracted {len(cleaned_files)} files from initial plan" + ) + if cleaned_files: + self.logger.info(f"๐Ÿ“ Sample files: {cleaned_files[:3]}...") + + return cleaned_files + + except Exception as e: + self.logger.error(f"Failed to extract files from initial plan: {e}") + return [] + + def _extract_from_tree_structure(self, lines: List[str]) -> List[str]: + """ + Extract files from tree structure format - Advanced algorithm with multi-strategy approach + + Strategy: + 1. Precise indentation-based depth calculation + 2. Smart directory vs file detection using multiple heuristics + 3. Robust path stack management with depth tracking + 4. Fallback to regex pattern matching if tree parsing fails + """ + files = [] + in_file_structure = False + + # Enhanced path tracking: store (depth, name) pairs + path_stack = [] # [(depth, dir_name), ...] + root_dir = None + + # Track the base indentation of tree structure + base_indent = None + + for line_num, line in enumerate(lines): + # === Section Boundary Detection === + if "file_structure:" in line or "file_structure |" in line: + in_file_structure = True + continue + + # End of file_structure section (next YAML key without indentation) + if ( + in_file_structure + and line.strip() + and not line.startswith(" ") + and ":" in line + ): + break + + if not in_file_structure: + continue + + if not line.strip(): + continue + + # Skip YAML comments and keys that are clearly not files + stripped = line.strip() + if stripped.startswith("#") or ( + stripped.endswith(":") and "/" not in stripped + ): + continue + + # === Root Directory Detection === + # Pattern: "project-name/" at minimal indentation, no tree chars + if stripped.endswith("/") and not any( + c in line for c in ["โ”œ", "โ””", "โ”‚", "โ”€"] + ): + indent = len(line) - len(line.lstrip()) + if indent <= 4: # Root level + root_dir = stripped.rstrip("/") + path_stack = [] + base_indent = None + self.logger.debug(f"๐ŸŒณ Detected root directory: {root_dir}") + continue + + # === Tree Structure Line Detection === + has_tree_chars = any(c in line for c in ["โ”œ", "โ””", "โ”‚", "โ”€"]) + if not has_tree_chars: + continue + + # === Calculate Precise Depth === + # Method: Count the actual tree structure symbols to determine hierarchy + indent = len(line) - len(line.lstrip()) + + # Set base indent on first tree line + if base_indent is None: + base_indent = indent + + # Count tree depth indicators + # Each "โ”‚ " or " " block represents one level + # "โ”œโ”€โ”€ " or "โ””โ”€โ”€ " marks the current item + tree_prefix = line[ + : line.find("โ”œ") + if "โ”œ" in line + else line.find("โ””") + if "โ””" in line + else len(line) + ] + + # Count depth by analyzing tree prefix structure + # Pattern: " โ”‚ โ”‚ โ”œโ”€โ”€ filename" -> depth 3 + # Pattern: " โ”œโ”€โ”€ filename" -> depth 1 + # Pattern: " โ”‚ โ”œโ”€โ”€ filename" -> depth 2 + + depth = 0 + i = 0 + while i < len(tree_prefix): + # Look for pipe or tree junction + if i + 4 <= len(tree_prefix): + chunk = tree_prefix[i : i + 4] + if "โ”‚" in chunk or all(c == " " for c in chunk): + depth += 1 + i += 4 + else: + i += 1 + else: + break + + # Fallback: use relative indentation + if depth == 0: + depth = max(1, (indent - base_indent) // 4 + 1) + + # === Clean and Extract Item Name === + item_name = line + # Remove all tree characters + for pattern in ["โ”œโ”€โ”€", "โ””โ”€โ”€", "โ”‚", "โ”œ", "โ””", "โ”€"]: + item_name = item_name.replace(pattern, "") + item_name = item_name.strip() + + # Remove inline comments + if "#" in item_name: + item_name = item_name.split("#")[0].strip() + + if not item_name or ":" in item_name: + continue + + # === Smart Directory vs File Detection === + is_directory = self._is_directory(item_name) + + # === Update Path Stack === + # Remove items deeper than current depth + path_stack = [(d, n) for d, n in path_stack if d < depth] + + if is_directory: + dir_name = item_name.rstrip("/") + path_stack.append((depth, dir_name)) + self.logger.debug(f" {' ' * depth}๐Ÿ“ {dir_name} (depth={depth})") + else: + # Construct full file path + path_parts = [root_dir] if root_dir else [] + path_parts.extend([name for _, name in path_stack]) + path_parts.append(item_name) + + full_path = "/".join(path_parts) + files.append(full_path) + self.logger.debug(f" {' ' * depth}๐Ÿ“„ {full_path}") + + return files + + def _is_directory(self, name: str) -> bool: + """ + Advanced directory detection using multiple heuristics + + Returns True if the name represents a directory, False if it's a file + """ + # Rule 1: Explicit directory marker + if name.endswith("/"): + return True + + # Rule 2: Has file extension -> definitely a file + basename = name.split("/")[-1] + if "." in basename: + # Check if it's a known file extension + known_extensions = [ + ".py", + ".js", + ".ts", + ".jsx", + ".tsx", + ".vue", + ".html", + ".css", + ".scss", + ".sass", + ".json", + ".yaml", + ".yml", + ".xml", + ".toml", + ".md", + ".txt", + ".rst", + ".sh", + ".bat", + ".ps1", + ".c", + ".cpp", + ".h", + ".hpp", + ".java", + ".go", + ".rs", + ".sql", + ".db", + ".env", + ".gitignore", + ".dockerignore", + ".lock", + ".sum", + ".mod", + ] + if any(basename.lower().endswith(ext) for ext in known_extensions): + return False + + # Has extension but not recognized -> might be config file, treat as file + if basename.count(".") == 1: + return False + + # Rule 3: Known special files without extensions + special_files = [ + "README", + "LICENSE", + "CHANGELOG", + "CONTRIBUTING", + "Makefile", + "Dockerfile", + "Vagrantfile", + "requirements.txt", + "setup.py", + "setup.cfg", + "package.json", + "package-lock.json", + "Cargo.toml", + "go.mod", + ] + if basename in special_files or basename.upper() in special_files: + return False + + # Rule 4: Common directory names (even without trailing /) + common_dirs = [ + "src", + "lib", + "app", + "core", + "api", + "web", + "client", + "server", + "config", + "configs", + "settings", + "data", + "datasets", + "models", + "model", + "utils", + "helpers", + "common", + "shared", + "tests", + "test", + "testing", + "__tests__", + "docs", + "documentation", + "scripts", + "bin", + "tools", + "assets", + "static", + "public", + "resources", + "components", + "views", + "pages", + "routes", + "services", + "controllers", + "handlers", + "middleware", + "middlewares", + "types", + "interfaces", + "schemas", + "experiments", + "notebooks", + "dist", + "build", + "output", + "node_modules", + "vendor", + "packages", + "__pycache__", + ".git", + ".vscode", + "training", + "evaluation", + "inference", + ] + if basename.lower() in common_dirs: + return True + + # Rule 5: Plural forms often indicate directories + if basename.endswith("s") and len(basename) > 3: + singular = basename[:-1] + if singular in common_dirs: + return True + + # Rule 6: Python package indicators + if basename == "__init__.py": + return False # This is a file + + # Default: if no extension and not a known file, likely a directory + return "." not in basename + + def _extract_from_simple_list(self, lines: List[str]) -> List[str]: + """Extract files from simple list format (- filename)""" + files = [] + + for line in lines: + line = line.strip() + if line.startswith("- ") and not line.startswith('- "'): + # Remove leading "- " and clean up + filename = line[2:].strip() + + # Remove quotes if present + if filename.startswith('"') and filename.endswith('"'): + filename = filename[1:-1] + + # Check if it looks like a file (has extension) + if "." in filename and "/" in filename: + files.append(filename) + + return files + + def _extract_from_plan_content(self, lines: List[str]) -> List[str]: + """ + Advanced fallback extraction: Extract files from anywhere in the plan content + Uses multiple regex patterns and intelligent filtering + """ + files = [] + import re + + # === Pattern 1: Standard file paths === + # Matches: path/to/file.py, src/model/apt_layer.py + pattern1 = r"([a-zA-Z0-9_\-]+(?:/[a-zA-Z0-9_\-]+)+\.[a-zA-Z0-9]+)" + + # === Pattern 2: Quoted file paths === + # Matches: "path/to/file.py", 'src/utils.py' + pattern2 = r'["\']([a-zA-Z0-9_\-]+(?:/[a-zA-Z0-9_\-]+)+\.[a-zA-Z0-9]+)["\']' + + # === Pattern 3: File paths with special characters === + # Matches: data/data_loader.py, __init__.py paths + pattern3 = r"([a-zA-Z0-9_\-]+(?:/[a-zA-Z0-9_\-]+)*/__init__\.py)" + pattern4 = r"([a-zA-Z0-9_\-]+(?:/[a-zA-Z0-9_\-]+)+\.(?:py|js|ts|jsx|tsx|html|css|md|txt|json|yaml|yml|xml|sql|sh|bat))" + + # === Pattern 5: Backtick-wrapped paths (in code blocks) === + pattern5 = r"`([a-zA-Z0-9_\-]+(?:/[a-zA-Z0-9_\-]+)+\.[a-zA-Z0-9]+)`" + + all_patterns = [pattern1, pattern2, pattern3, pattern4, pattern5] + + # Collect all potential matches + potential_files = set() + + for line in lines: + # Skip comment-only lines + stripped = line.strip() + if stripped.startswith("#") and not ("/" in stripped and "." in stripped): + continue + + # Apply all patterns + for pattern in all_patterns: + matches = re.findall(pattern, line) + potential_files.update(matches) + + # === Filter and validate matches === + code_extensions = { + ".py", + ".js", + ".ts", + ".jsx", + ".tsx", + ".vue", + ".html", + ".css", + ".scss", + ".sass", + ".less", + ".json", + ".yaml", + ".yml", + ".toml", + ".xml", + ".ini", + ".cfg", + ".md", + ".rst", + ".txt", + ".sh", + ".bash", + ".zsh", + ".bat", + ".ps1", + ".cmd", + ".c", + ".cpp", + ".h", + ".hpp", + ".cc", + ".cxx", + ".java", + ".kt", + ".scala", + ".go", + ".rs", + ".php", + ".rb", + ".pl", + ".lua", + ".r", + ".sql", + ".db", + ".dockerfile", + ".env", + ".gitignore", + ".lock", + ".sum", + ".mod", + } + + for file_path in potential_files: + # Must have path separator + if "/" not in file_path: + continue + + # Must have valid extension + has_valid_ext = any( + file_path.lower().endswith(ext) for ext in code_extensions + ) + if not has_valid_ext: + continue + + # Filter out obvious non-files + if any( + bad in file_path.lower() + for bad in [ + "http://", + "https://", + ".png", + ".jpg", + ".jpeg", + ".gif", + ".svg", + ".ico", + ] + ): + continue + + # Must not be too short (avoid false positives) + if len(file_path) < 5: + continue + + # Path components should be reasonable + parts = file_path.split("/") + if any(len(part) == 0 for part in parts): + continue + + files.append(file_path) + + # Sort for consistency + files = sorted(list(set(files))) + + return files + + def _clean_and_validate_files(self, files: List[str]) -> List[str]: + """ + Clean and validate extracted file paths - advanced filtering and deduplication + + Features: + 1. Remove duplicates while preserving order + 2. Normalize paths (handle ../, ./, double slashes) + 3. Filter out non-code files + 4. Smart deduplication (recognize same file with different path prefixes) + """ + cleaned_files = [] + seen_normalized = set() + + # Define code file extensions we want to track + code_extensions = { + ".py", + ".js", + ".ts", + ".jsx", + ".tsx", + ".vue", + ".html", + ".css", + ".scss", + ".sass", + ".less", + ".json", + ".yaml", + ".yml", + ".toml", + ".xml", + ".ini", + ".cfg", + ".md", + ".rst", + ".txt", + ".sh", + ".bash", + ".zsh", + ".bat", + ".ps1", + ".cmd", + ".c", + ".cpp", + ".h", + ".hpp", + ".cc", + ".cxx", + ".java", + ".kt", + ".scala", + ".go", + ".rs", + ".php", + ".rb", + ".pl", + ".lua", + ".r", + ".sql", + ".db", + ".dockerfile", + ".env", + ".gitignore", + ".lock", + ".sum", + ".mod", + } + + for file_path in files: + # === Step 1: Basic Cleaning === + cleaned_path = file_path.strip().strip('"').strip("'").strip("`") + + if not cleaned_path: + continue + + # Remove leading/trailing slashes + cleaned_path = cleaned_path.strip("/") + + # === Step 2: Path Normalization === + # Remove double slashes + while "//" in cleaned_path: + cleaned_path = cleaned_path.replace("//", "/") + + # Handle relative paths (remove ./ prefix) + if cleaned_path.startswith("./"): + cleaned_path = cleaned_path[2:] + + # === Step 3: Validate File Structure === + # Must have filename (not just directory) + if not cleaned_path or "/" not in cleaned_path: + # Single file without path - only accept if it has extension + if "." not in cleaned_path: + continue + + # Extract basename + basename = cleaned_path.split("/")[-1] + + # Skip directories (no file extension in basename) + if "." not in basename: + continue + + # === Step 4: Extension Validation === + # Only include files with code extensions + has_code_extension = any( + cleaned_path.lower().endswith(ext) for ext in code_extensions + ) + if not has_code_extension: + continue + + # === Step 5: Filter Invalid Patterns === + # Skip files that look like YAML keys or config entries + if ":" in cleaned_path and not any( + cleaned_path.endswith(ext) for ext in [".yaml", ".yml"] + ): + continue + + # Skip paths with invalid characters + if any( + char in cleaned_path for char in ['"', "'", "|", "<", ">", "*", "?"] + ): + continue + + # Skip obvious build/temp artifacts + if any( + part in cleaned_path + for part in [ + "__pycache__", + ".pyc", + "node_modules", + ".git/", + "dist/build", + ] + ): + continue + + # === Step 6: Smart Deduplication === + # Normalize for comparison (lowercase, remove common prefixes) + normalized_for_comparison = cleaned_path.lower() + + # Check if we've already seen this file (exact match) + if normalized_for_comparison in seen_normalized: + continue + + # Check for duplicate with different path (e.g., "src/model/apt_layer.py" vs "model/apt_layer.py") + # Keep the longer (more specific) path + is_duplicate = False + paths_to_remove = [] + + for existing_normalized in seen_normalized: + # If current path is suffix of existing, it's a shorter version - skip it + if existing_normalized.endswith("/" + normalized_for_comparison): + is_duplicate = True + break + + # If existing path is suffix of current, current is longer - replace existing + if normalized_for_comparison.endswith("/" + existing_normalized): + paths_to_remove.append(existing_normalized) + + if is_duplicate: + continue + + # Remove shorter versions + for path_to_remove in paths_to_remove: + seen_normalized.discard(path_to_remove) + # Also remove from cleaned_files list + cleaned_files = [ + f for f in cleaned_files if f.lower() != path_to_remove + ] + + # === Step 7: Add to Results === + seen_normalized.add(normalized_for_comparison) + cleaned_files.append(cleaned_path) + + return sorted(cleaned_files) + + def record_file_implementation( + self, file_path: str, implementation_content: str = "" + ): + """ + Record a newly implemented file (simplified version) + NEW LOGIC: File implementation is tracked via write_file tool detection + + Args: + file_path: Path of the implemented file + implementation_content: Content of the implemented file + """ + # Add file to implemented files list if not already present + if file_path not in self.implemented_files: + self.implemented_files.append(file_path) + + self.logger.info(f"๐Ÿ“ File implementation recorded: {file_path}") + + async def create_code_implementation_summary( + self, + client, + client_type: str, + file_path: str, + implementation_content: str, + files_implemented: int, + ) -> str: + """ + Create LLM-based code implementation summary after writing a file + Uses LLM to analyze and summarize the implemented code + + Args: + client: LLM client instance + client_type: Type of LLM client ("anthropic" or "openai") + file_path: Path of the implemented file + implementation_content: Content of the implemented file + files_implemented: Number of files implemented so far + + Returns: + LLM-generated formatted code implementation summary + """ + try: + # Record the file implementation first + self.record_file_implementation(file_path, implementation_content) + + # Create prompt for LLM summary + summary_prompt = self._create_code_summary_prompt( + file_path, implementation_content, files_implemented + ) + summary_messages = [{"role": "user", "content": summary_prompt}] + + # Get LLM-generated summary + llm_response = await self._call_llm_for_summary( + client, client_type, summary_messages + ) + llm_summary = llm_response.get("content", "") + + # Extract different sections from LLM summary + sections = self._extract_summary_sections(llm_summary) + + # Store Next Steps in temporary variable (not saved to file) + self.current_next_steps = sections.get("next_steps", "") + if self.current_next_steps: + self.logger.info("๐Ÿ“ Next Steps stored temporarily (not saved to file)") + + # Format summary with only Implementation Progress and Dependencies for file saving + file_summary_content = "" + if sections.get("core_purpose"): + file_summary_content += sections["core_purpose"] + "\n\n" + if sections.get("public_interface"): + file_summary_content += sections["public_interface"] + "\n\n" + if sections.get("internal_dependencies"): + file_summary_content += sections["internal_dependencies"] + "\n\n" + if sections.get("external_dependencies"): + file_summary_content += sections["external_dependencies"] + "\n\n" + if sections.get("implementation_notes"): + file_summary_content += sections["implementation_notes"] + "\n\n" + + # Create the formatted summary for file saving (without Next Steps) + formatted_summary = self._format_code_implementation_summary( + file_path, file_summary_content.strip(), files_implemented + ) + + # Save to implement_code_summary.md (append mode) - only Implementation Progress and Dependencies + await self._save_code_summary_to_file(formatted_summary, file_path) + + self.logger.info(f"Created and saved code summary for: {file_path}") + return formatted_summary + + except Exception as e: + self.logger.error( + f"Failed to create LLM-based code implementation summary: {e}" + ) + # Fallback to simple summary + return self._create_fallback_code_summary( + file_path, implementation_content, files_implemented + ) + + def _create_code_summary_prompt( + self, file_path: str, implementation_content: str, files_implemented: int + ) -> str: + """ + Create prompt for LLM to generate code implementation summary + + Args: + file_path: Path of the implemented file + implementation_content: Content of the implemented file + files_implemented: Number of files implemented so far + + Returns: + Prompt for LLM summarization + """ + current_round = self.current_round + + # Get formatted file lists + file_lists = self.get_formatted_files_lists() + implemented_files_list = file_lists["implemented"] + unimplemented_files_list = file_lists["unimplemented"] + + prompt = f"""You are an expert code implementation summarizer. Analyze the implemented code file and create a structured summary. + +**๐Ÿšจ CRITICAL: The files listed below are ALREADY IMPLEMENTED - DO NOT suggest them in Next Steps! ๐Ÿšจ** + +**All Previously Implemented Files:** +{implemented_files_list} + +**Remaining Unimplemented Files (choose ONLY from these for Next Steps):** +{unimplemented_files_list} + +**Current Implementation Context:** +- **File Implemented**: {file_path} +- **Current Round**: {current_round} +- **Total Files Implemented**: {files_implemented} + + +**Initial Plan Reference:** +{self.initial_plan[:]} + +**Implemented Code Content:** +``` +{implementation_content[:]} +``` + +**Required Summary Format:** + +**Core Purpose** (provide a general overview of the file's main responsibility): +- {{1-2 sentence description of file's main responsibility}} + +**Public Interface** (what other files can use, if any): +- Class {{ClassName}}: {{purpose}} | Key methods: {{method_names}} | Constructor params: {{params}} +- Function {{function_name}}({{params}}): {{purpose}} -> {{return_type}}: {{purpose}} +- Constants/Types: {{name}}: {{value/description}} + +**Internal Dependencies** (what this file imports/requires, if any): +- From {{module/file}}: {{specific_imports}} +- External packages: {{package_name}} - {{usage_context}} + +**External Dependencies** (what depends on this file, if any): +- Expected to be imported by: {{likely_consumer_files}} +- Key exports used elsewhere: {{main_interfaces}} + +**Implementation Notes**: (if any) +- Architecture decisions: {{key_choices_made}} +- Cross-File Relationships: {{how_files_work_together}} + +**Next Steps**: List the code file (ONLY ONE) that will be implemented in the next round (MUST choose from "Remaining Unimplemented Files" above) + Format: Code will be implemented: {{file_path}} + **NEVER suggest any file from the "All Previously Implemented Files" list!** + +**Instructions:** +- Be precise and concise +- Focus on function interfaces that other files will need +- Extract actual function signatures from the code +- **CRITICAL: For Next Steps, ONLY choose ONE file from the "Remaining Unimplemented Files" list above** +- **NEVER suggest implementing a file that is already in the implemented files list** +- Choose the next file based on logical dependencies and implementation order +- Use the exact format specified above + +**Summary:**""" + + return prompt + + # TODO: The prompt is not good, need to be improved + # **Implementation Progress**: List the code file completed in current round and core implementation ideas + # Format: {{file_path}}: {{core implementation ideas}} + + # **Dependencies**: According to the File Structure and initial plan, list functions that may be called by other files + # Format: {{file_path}}: Function {{function_name}}: core ideas--{{ideas}}; Required parameters--{{params}}; Return parameters--{{returns}} + # Required packages: {{packages}} + + def _extract_summary_sections(self, llm_summary: str) -> Dict[str, str]: + """ + Extract different sections from LLM-generated summary + + Args: + llm_summary: Raw LLM-generated summary text + + Returns: + Dictionary with extracted sections: core_purpose, public_interface, internal_dependencies, + external_dependencies, implementation_notes, next_steps + """ + sections = { + "core_purpose": "", + "public_interface": "", + "internal_dependencies": "", + "external_dependencies": "", + "implementation_notes": "", + "next_steps": "", + } + + try: + lines = llm_summary.split("\n") + current_section = None + current_content = [] + + for line in lines: + line_lower = line.lower().strip() + + # Check for section headers + if "core purpose" in line_lower: + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + current_section = "core_purpose" + current_content = [line] # Include the header + elif "public interface" in line_lower: + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + current_section = "public_interface" + current_content = [line] # Include the header + elif "internal dependencies" in line_lower: + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + current_section = "internal_dependencies" + current_content = [line] # Include the header + elif "external dependencies" in line_lower: + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + current_section = "external_dependencies" + current_content = [line] # Include the header + elif "implementation notes" in line_lower: + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + current_section = "implementation_notes" + current_content = [line] # Include the header + elif "next steps" in line_lower: + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + current_section = "next_steps" + current_content = [line] # Include the header + else: + # Add content to current section + if current_section: + current_content.append(line) + + # Don't forget the last section + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + + self.logger.info(f"๐Ÿ“‹ Extracted sections: {list(sections.keys())}") + + except Exception as e: + self.logger.error(f"Failed to extract summary sections: {e}") + # Fallback: put everything in core_purpose + sections["core_purpose"] = llm_summary + + return sections + + def _format_code_implementation_summary( + self, file_path: str, llm_summary: str, files_implemented: int + ) -> str: + """ + Format the LLM-generated summary into the final structure + + Args: + file_path: Path of the implemented file + llm_summary: LLM-generated summary content + files_implemented: Number of files implemented so far + + Returns: + Formatted summary + """ + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + # # Create formatted list of implemented files + # implemented_files_list = ( + # "\n".join([f"- {file}" for file in self.implemented_files]) + # if self.implemented_files + # else "- None yet" + # ) + + # formatted_summary = f"""# Code Implementation Summary + # **All Previously Implemented Files:** + # {implemented_files_list} + # **Generated**: {timestamp} + # **File Implemented**: {file_path} + # **Total Files Implemented**: {files_implemented} + + # {llm_summary} + + # --- + # *Auto-generated by Memory Agent* + # """ + formatted_summary = f"""# Code Implementation Summary +**Generated**: {timestamp} +**File Implemented**: {file_path} + +{llm_summary} + +--- +*Auto-generated by Memory Agent* +""" + return formatted_summary + + def _create_fallback_code_summary( + self, file_path: str, implementation_content: str, files_implemented: int + ) -> str: + """ + Create fallback summary when LLM is unavailable + + Args: + file_path: Path of the implemented file + implementation_content: Content of the implemented file + files_implemented: Number of files implemented so far + + Returns: + Fallback summary + """ + # Create formatted list of implemented files + implemented_files_list = ( + "\n".join([f"- {file}" for file in self.implemented_files]) + if self.implemented_files + else "- None yet" + ) + + summary = f"""# Code Implementation Summary +**All Previously Implemented Files:** +{implemented_files_list} +**Generated**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} +**File Implemented**: {file_path} +**Total Files Implemented**: {files_implemented} +**Summary failed to generate.** + +--- +*Auto-generated by Concise Memory Agent (Fallback Mode)* +""" + return summary + + async def _save_code_summary_to_file(self, new_summary: str, file_path: str): + """ + Append code implementation summary to implement_code_summary.md + Accumulates all implementations with clear separators + + Args: + new_summary: New summary content to append + file_path: Path of the file for which the summary was generated + """ + try: + # Create directory if it doesn't exist + os.makedirs(os.path.dirname(self.code_summary_path), exist_ok=True) + + # Check if file exists to determine if we need header + file_exists = os.path.exists(self.code_summary_path) + + # Open in append mode to accumulate all implementations + with open(self.code_summary_path, "a", encoding="utf-8") as f: + if not file_exists: + # Write header for new file + f.write("# Code Implementation Progress Summary\n") + f.write("*Accumulated implementation progress for all files*\n\n") + + # Add clear separator between implementations + f.write("\n" + "=" * 80 + "\n") + f.write( + f"## IMPLEMENTATION File {file_path}; ROUND {self.current_round} \n" + ) + f.write("=" * 80 + "\n\n") + + # Write the new summary + f.write(new_summary) + f.write("\n\n") + + self.logger.info( + f"Appended LLM-based code implementation summary to: {self.code_summary_path}" + ) + + except Exception as e: + self.logger.error(f"Failed to save code implementation summary: {e}") + + async def _call_llm_for_summary( + self, client, client_type: str, summary_messages: List[Dict] + ) -> Dict[str, Any]: + """ + Call LLM for code implementation summary generation ONLY + + This method is used only for creating code implementation summaries, + NOT for conversation summarization which has been removed. + """ + if client_type == "anthropic": + response = await client.messages.create( + model=self.default_models["anthropic"], + system="You are an expert code implementation summarizer. Create structured summaries of implemented code files that preserve essential information about functions, dependencies, and implementation approaches.", + messages=summary_messages, + max_tokens=5000, + temperature=0.2, + ) + + content = "" + if response and hasattr(response, "content") and response.content: + for block in response.content: + if block.type == "text": + content += block.text + else: + self.logger.warning("Anthropic response is empty or malformed") + + return {"content": content} + + elif client_type == "openai": + openai_messages = [ + { + "role": "system", + "content": "You are an expert code implementation summarizer. Create structured summaries of implemented code files that preserve essential information about functions, dependencies, and implementation approaches.", + } + ] + openai_messages.extend(summary_messages) + + # Try max_tokens and temperature first, fallback to max_completion_tokens without temperature if unsupported + try: + response = await client.chat.completions.create( + model=self.default_models["openai"], + messages=openai_messages, + max_tokens=5000, + temperature=0.2, + ) + except Exception as e: + if "max_tokens" in str(e) and "max_completion_tokens" in str(e): + # Retry with max_completion_tokens and no temperature for models that require it + response = await client.chat.completions.create( + model=self.default_models["openai"], + messages=openai_messages, + max_completion_tokens=5000, + ) + else: + raise + + # Safely extract content from response + if response and hasattr(response, "choices") and response.choices: + return {"content": response.choices[0].message.content or ""} + else: + self.logger.warning("OpenAI response is empty or malformed") + return {"content": ""} + + elif client_type == "google": + from google.genai import types + + # Convert messages to Gemini format + system_instruction = "You are an expert code implementation summarizer. Create structured summaries of implemented code files that preserve essential information about functions, dependencies, and implementation approaches." + + gemini_messages = [] + for msg in summary_messages: + role = msg.get("role", "user") + content = msg.get("content", "") + + # Convert role names: "assistant" -> "model" + if role == "assistant": + role = "model" + elif role not in ["user", "model"]: + role = "user" + + gemini_messages.append( + types.Content(role=role, parts=[types.Part.from_text(text=content)]) + ) + + config = types.GenerateContentConfig( + max_output_tokens=5000, + temperature=0.2, + system_instruction=system_instruction, + ) + + response = await client.aio.models.generate_content( + model=self.default_models.get("google", "gemini-2.0-flash"), + contents=gemini_messages, + config=config, + ) + + # Extract content from Gemini response + content = "" + if response and hasattr(response, "candidates") and response.candidates: + candidate = response.candidates[0] + if hasattr(candidate, "content") and candidate.content: + if hasattr(candidate.content, "parts") and candidate.content.parts: + for part in candidate.content.parts: + if hasattr(part, "text") and part.text: + content += part.text + + if not content: + self.logger.warning("Google response is empty or malformed") + + return {"content": content} + + else: + raise ValueError(f"Unsupported client type: {client_type}") + + def start_new_round(self, iteration: Optional[int] = None): + """Start a new dialogue round and reset tool results + + Args: + iteration: Optional iteration number from workflow to sync with current_round + """ + if iteration is not None: + # Sync with workflow iteration + self.current_round = iteration + # self.logger.info(f"๐Ÿ”„ Synced round with workflow iteration {iteration}") + else: + # Default behavior: increment round counter + self.current_round += 1 + self.logger.info(f"๐Ÿ”„ Started new round {self.current_round}") + + self.current_round_tool_results = [] # Clear previous round results + # Note: Don't reset last_write_file_detected and should_clear_memory_next here + # These flags persist across rounds until memory optimization is applied + # self.logger.info(f"๐Ÿ”„ Round {self.current_round} - Tool results cleared, memory flags preserved") + + def record_tool_result( + self, tool_name: str, tool_input: Dict[str, Any], tool_result: Any + ): + """ + Record tool result for current round and detect write_file calls + + Args: + tool_name: Name of the tool called + tool_input: Input parameters for the tool + tool_result: Result returned by the tool + """ + # Detect write_file calls to trigger memory clearing + if tool_name == "write_file": + self.last_write_file_detected = True + self.should_clear_memory_next = True + + # self.logger.info(f"๐Ÿ”„ WRITE_FILE DETECTED: {file_path} - Memory will be cleared in next round") + + # Only record specific tools that provide essential information + essential_tools = [ + "read_code_mem", # Read code summary from implement_code_summary.md + "read_file", # Read file contents + "write_file", # Write file contents (important for tracking implementations) + "execute_python", # Execute Python code (for testing/validation) + "execute_bash", # Execute bash commands (for build/execution) + "search_code", # Search code patterns + "search_reference_code", # Search reference code (if available) + "get_file_structure", # Get file structure (for understanding project layout) + ] + + if tool_name in essential_tools: + tool_record = { + "tool_name": tool_name, + "tool_input": tool_input, + "tool_result": tool_result, + "timestamp": time.time(), + } + self.current_round_tool_results.append(tool_record) + # self.logger.info(f"๐Ÿ“Š Essential tool result recorded: {tool_name} ({len(self.current_round_tool_results)} total)") + + def should_use_concise_mode(self) -> bool: + """ + Check if concise memory mode should be used + + Returns: + True if first file has been generated and concise mode should be active + """ + return self.last_write_file_detected + + def create_concise_messages( + self, + system_prompt: str, + messages: List[Dict[str, Any]], + files_implemented: int, + ) -> List[Dict[str, Any]]: + """ + Create concise message list for LLM input + NEW LOGIC: Always clear after write_file, keep system_prompt + initial_plan + current round tools + + Args: + system_prompt: Current system prompt + messages: Original message list + files_implemented: Number of files implemented so far + + Returns: + Concise message list containing only essential information + """ + if not self.last_write_file_detected: + # Before any write_file, use normal flow + self.logger.info( + "๐Ÿ”„ Using normal conversation flow (before any write_file)" + ) + return messages + + # After write_file detection, use concise approach with clean slate + self.logger.info( + f"๐ŸŽฏ Using CONCISE memory mode - Clear slate after write_file, Round {self.current_round}" + ) + + concise_messages = [] + + # Get formatted file lists + file_lists = self.get_formatted_files_lists() + implemented_files_list = file_lists["implemented"] + unimplemented_files_list = file_lists["unimplemented"] + + # Debug output for unimplemented files (clean format without dashes) + unimplemented_files = self.get_unimplemented_files() + print("โœ… Unimplemented Files:") + for file_path in unimplemented_files: + print(f"{file_path}") + if self.current_next_steps.strip(): + print(f"\n๐Ÿ“‹ {self.current_next_steps}") + + # 1. Add initial plan message (always preserved) + initial_plan_message = { + "role": "user", + "content": f"""**Task: Implement code based on the following reproduction plan** + +**Code Reproduction Plan:** +{self.initial_plan} + +**Working Directory:** Current workspace + +**All Previously Implemented Files:** +{implemented_files_list} + +**Current Status:** {files_implemented} files implemented + +**Remaining Files to Implement:** +{unimplemented_files_list} + +**IMPORTANT:** If the remaining files list shows "All files implemented!", you MUST reply with "All files implemented" to complete the task. Do NOT continue calling tools. + +**Objective:** {"Reply 'All files implemented' to finish" if not unimplemented_files else "Continue implementation by analyzing dependencies and implementing the next required file according to the plan's priority order."}""", + } + + # Append Next Steps information if available + # if self.current_next_steps.strip(): + # initial_plan_message["content"] += ( + # f"\n\n**Next Steps (from previous analysis):**\n{self.current_next_steps}" + # ) + + concise_messages.append(initial_plan_message) + + # 2. Add Knowledge Base + knowledge_base_message = { + "role": "user", + "content": f"""**Below is the Knowledge Base of the LATEST implemented code file:** +{self._read_code_knowledge_base()} + +**Development Cycle - START HERE:** + +**FIRST - Check completion status:** +- If "Remaining Files to Implement" above shows "All files implemented!", reply "All files implemented" immediately + +**For NEW file implementation (if remaining files exist):** +Write_file can be used to implement the new component + +**Remember:** Stop and declare completion when all files are done!""", + } + if self.current_next_steps.strip(): + knowledge_base_message["content"] += ( + f"\n\n**Next Steps (from previous analysis):**\n{self.current_next_steps}" + ) + concise_messages.append(knowledge_base_message) + + # # 3. Add current tool results (essential information for next file generation) + # if self.current_round_tool_results: + # tool_results_content = self._format_tool_results() + + # # # Append Next Steps information if available + # # if self.current_next_steps.strip(): + # # tool_results_content += f"\n\n**Next Steps (from previous analysis):**\n{self.current_next_steps}" + + # tool_results_message = { + # "role": "user", + # "content": f"""**Current Tool Results:** + # {tool_results_content}""", + # } + # concise_messages.append(tool_results_message) + # else: + # # If no tool results yet, add guidance for next steps + # guidance_content = f"""**Current Round:** {self.current_round} + + # **Development Cycle - START HERE:** + + # **For NEW file implementation:** + # Write_file can be used to implement the new component""" + + # # # Append Next Steps information if available (even when no tool results) + # # if self.current_next_steps.strip(): + # # guidance_content += f"\n\n**Next Steps (from previous analysis):**\n{self.current_next_steps}" + + # guidance_message = { + # "role": "user", + # "content": guidance_content, + # } + # concise_messages.append(guidance_message) + # # **Available Essential Tools:** read_code_mem, write_file, execute_python, execute_bash + # # **Remember:** Start with read_code_mem when implementing NEW files to understand existing code. When all files are implemented, focus on testing and completion. Implement according to the original paper's specifications - any reference code is for inspiration only.""" + # # self.logger.info(f"โœ… Concise messages created: {len(concise_messages)} messages (original: {len(messages)})") + return concise_messages + + def _read_code_knowledge_base(self) -> Optional[str]: + """ + Read the implement_code_summary.md file as code knowledge base + Returns all content from the file + + Returns: + Full content of the file if it exists, None otherwise + """ + try: + if os.path.exists(self.code_summary_path): + with open(self.code_summary_path, "r", encoding="utf-8") as f: + content = f.read().strip() + + if content: + # Return all content instead of just the latest entry + return content + else: + return None + else: + return None + + except Exception as e: + self.logger.error(f"Failed to read code knowledge base: {e}") + return None + + def _extract_latest_implementation_entry(self, content: str) -> Optional[str]: + """ + Extract the latest/final implementation entry from the implement_code_summary.md content + Uses a simpler approach to find the last implementation section + + Args: + content: Full content of implement_code_summary.md + + Returns: + Latest implementation entry content, or None if not found + """ + try: + import re + + # Pattern to match the start of implementation sections + section_pattern = ( + r"={80}\s*\n## IMPLEMENTATION File .+?; ROUND \d+\s*\n={80}" + ) + + # Find all implementation section starts + matches = list(re.finditer(section_pattern, content)) + + if not matches: + # No implementation sections found + lines = content.split("\n") + fallback_content = ( + "\n".join(lines[:10]) + "\n... (truncated for brevity)" + if len(lines) > 10 + else content + ) + self.logger.info( + "๐Ÿ“– No implementation sections found, using fallback content" + ) + return fallback_content + + # Get the start position of the last implementation section + last_match = matches[-1] + start_pos = last_match.start() + + # Take everything from the last section start to the end of content + latest_entry = content[start_pos:].strip() + + # self.logger.info(f"๐Ÿ“– Extracted latest implementation entry from knowledge base") + # print(f"DEBUG: Extracted content length: {len(latest_entry)}") + # print(f"DEBUG: First 200 chars: {latest_entry[:]}") + + return latest_entry + + except Exception as e: + self.logger.error(f"Failed to extract latest implementation entry: {e}") + # Return last 1000 characters as fallback + return content[-500:] if len(content) > 500 else content + + def _format_tool_results(self) -> str: + """ + Format current round tool results for LLM input + + Returns: + Formatted string of tool results + """ + if not self.current_round_tool_results: + return "No tool results in current round." + + formatted_results = [] + + for result in self.current_round_tool_results: + tool_name = result["tool_name"] + tool_input = result["tool_input"] + tool_result = result["tool_result"] + + # Format based on tool type + if tool_name == "read_code_mem": + file_path = tool_input.get("file_path", "unknown") + formatted_results.append(f""" +**read_code_mem Result for {file_path}:** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "read_file": + file_path = tool_input.get("file_path", "unknown") + formatted_results.append(f""" +**read_file Result for {file_path}:** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "write_file": + file_path = tool_input.get("file_path", "unknown") + formatted_results.append(f""" +**write_file Result for {file_path}:** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "execute_python": + code_snippet = ( + tool_input.get("code", "")[:50] + "..." + if len(tool_input.get("code", "")) > 50 + else tool_input.get("code", "") + ) + formatted_results.append(f""" +**execute_python Result (code: {code_snippet}):** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "execute_bash": + command = tool_input.get("command", "unknown") + formatted_results.append(f""" +**execute_bash Result (command: {command}):** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "search_code": + pattern = tool_input.get("pattern", "unknown") + file_pattern = tool_input.get("file_pattern", "") + formatted_results.append(f""" +**search_code Result (pattern: {pattern}, files: {file_pattern}):** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "search_reference_code": + target_file = tool_input.get("target_file", "unknown") + keywords = tool_input.get("keywords", "") + formatted_results.append(f""" +**search_reference_code Result for {target_file} (keywords: {keywords}):** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "get_file_structure": + directory = tool_input.get( + "directory_path", tool_input.get("path", "current") + ) + formatted_results.append(f""" +**get_file_structure Result for {directory}:** +{self._format_tool_result_content(tool_result)} +""") + + return "\n".join(formatted_results) + + def _format_tool_result_content(self, tool_result: Any) -> str: + """ + Format tool result content for display + + Args: + tool_result: Tool result to format + + Returns: + Formatted string representation + """ + if isinstance(tool_result, str): + # Try to parse as JSON for better formatting + try: + result_data = json.loads(tool_result) + if isinstance(result_data, dict): + # Format key information + if result_data.get("status") == "summary_found": + return ( + f"Summary found:\n{result_data.get('summary_content', '')}" + ) + elif result_data.get("status") == "no_summary": + return "No summary available" + else: + return json.dumps(result_data, indent=2) + else: + return str(result_data) + except json.JSONDecodeError: + return tool_result + else: + return str(tool_result) + + def get_memory_statistics(self, files_implemented: int = 0) -> Dict[str, Any]: + """Get memory agent statistics""" + unimplemented_files = self.get_unimplemented_files() + return { + "last_write_file_detected": self.last_write_file_detected, + "should_clear_memory_next": self.should_clear_memory_next, + "current_round": self.current_round, + "concise_mode_active": self.should_use_concise_mode(), + "current_round_tool_results": len(self.current_round_tool_results), + "essential_tools_recorded": [ + r["tool_name"] for r in self.current_round_tool_results + ], + "implemented_files_tracked": files_implemented, + "implemented_files_list": self.implemented_files.copy(), + "phases_parsed": len(self.phase_structure), + "next_steps_available": bool(self.current_next_steps.strip()), + "next_steps_length": len(self.current_next_steps.strip()) + if self.current_next_steps + else 0, + # File tracking statistics + "total_files_in_plan": len(self.all_files_list), + "files_implemented_count": len(self.implemented_files), + "files_remaining_count": len(unimplemented_files), + "all_files_list": self.all_files_list.copy(), + "unimplemented_files_list": unimplemented_files, + "implementation_progress_percent": ( + len(self.implemented_files) / len(self.all_files_list) * 100 + ) + if self.all_files_list + else 0, + } + + def get_implemented_files(self) -> List[str]: + """Get list of all implemented files""" + return self.implemented_files.copy() + + def get_all_files_list(self) -> List[str]: + """Get list of all files that should be implemented according to the plan""" + return self.all_files_list.copy() + + def refresh_files_list_from_directory(self) -> bool: + """ + Refresh the files list by extracting from the generated directory + Useful when the directory structure has been updated after initialization + + Returns: + True if successfully refreshed from directory, False if fell back to plan + """ + if os.path.exists(self.code_directory): + files_from_dir = self._extract_files_from_generated_directory() + if files_from_dir: + old_count = len(self.all_files_list) + self.all_files_list = files_from_dir + new_count = len(self.all_files_list) + self.logger.info( + f"๐Ÿ”„ Files list refreshed from directory: {old_count} โ†’ {new_count} files" + ) + return True + + self.logger.warning("Cannot refresh from directory, keeping current list") + return False + + def get_unimplemented_files(self) -> List[str]: + """ + Get list of files that haven't been implemented yet + Uses fuzzy path matching to handle partial paths + + Returns: + List of file paths that still need to be implemented + """ + + # def is_implemented(plan_file: str) -> bool: + # """Check if a file from plan is implemented (with fuzzy matching)""" + # # Normalize paths for comparison + # plan_file_normalized = plan_file.replace("\\", "/").strip("/") + # plan_filename = plan_file_normalized.split("/")[-1] # Extract filename + + # for impl_file in self.implemented_files: + # impl_file_normalized = impl_file.replace("\\", "/").strip("/") + # impl_filename = impl_file_normalized.split("/")[-1] # Extract filename + + # # Strategy 1: Exact path match + # if plan_file_normalized == impl_file_normalized: + # return True + + # # Strategy 2: One path ends with the other (partial path match) + # if plan_file_normalized.endswith( + # impl_file_normalized + # ) or impl_file_normalized.endswith(plan_file_normalized): + # # Ensure match is at a path boundary (not middle of directory name) + # if ( + # plan_file_normalized.endswith("/" + impl_file_normalized) + # or impl_file_normalized.endswith("/" + plan_file_normalized) + # ): + # return True + + # # Strategy 3: Same filename (fallback for different directory structures) + # # Only match if filenames are identical and reasonably unique (length > 5) + # if (plan_filename == impl_filename and len(plan_filename) > 5): + # return True + + # return False + def is_implemented(plan_file: str) -> bool: + """Check if a file from plan is implemented (with fuzzy matching)""" + # Normalize paths for comparison + plan_file_normalized = plan_file.replace("\\", "/").strip("/") + + for impl_file in self.implemented_files: + impl_file_normalized = impl_file.replace("\\", "/").strip("/") + + # Check if plan_file ends with impl_file (partial path match) + # or impl_file ends with plan_file (reverse partial match) + if plan_file_normalized.endswith( + impl_file_normalized + ) or impl_file_normalized.endswith(plan_file_normalized): + # Ensure match is at a path boundary (not middle of directory name) + if ( + plan_file_normalized.endswith("/" + impl_file_normalized) + or plan_file_normalized == impl_file_normalized + or impl_file_normalized.endswith("/" + plan_file_normalized) + ): + return True + return False + + # unimplemented = [f for f in self.all_files_list if not is_implemented(f)] + # return unimplemented + + unimplemented = [f for f in self.all_files_list if not is_implemented(f)] + return unimplemented + + def get_formatted_files_lists(self) -> Dict[str, str]: + """ + Get formatted strings for implemented and unimplemented files + + Returns: + Dictionary with 'implemented' and 'unimplemented' formatted lists + """ + implemented_list = ( + "\n".join([f"- {file}" for file in self.implemented_files]) + if self.implemented_files + else "- None yet" + ) + + unimplemented_files = self.get_unimplemented_files() + unimplemented_list = ( + "\n".join([f"- {file}" for file in unimplemented_files]) + if unimplemented_files + else "- All files implemented!" + ) + + return {"implemented": implemented_list, "unimplemented": unimplemented_list} + + def get_current_next_steps(self) -> str: + """Get the current Next Steps information""" + return self.current_next_steps + + def clear_next_steps(self): + """Clear the stored Next Steps information""" + if self.current_next_steps.strip(): + self.logger.info("๐Ÿงน Next Steps information cleared") + self.current_next_steps = "" + + def set_next_steps(self, next_steps: str): + """Manually set Next Steps information""" + self.current_next_steps = next_steps + self.logger.info( + f"๐Ÿ“ Next Steps manually set ({len(next_steps.strip())} chars)" + ) + + def should_trigger_memory_optimization( + self, messages: List[Dict[str, Any]], files_implemented: int = 0 + ) -> bool: + """ + Check if memory optimization should be triggered + NEW LOGIC: Trigger after write_file has been detected + + Args: + messages: Current message list + files_implemented: Number of files implemented so far + + Returns: + True if concise mode should be applied + """ + # Trigger if we detected write_file and should clear memory + if self.should_clear_memory_next: + # self.logger.info(f"๐ŸŽฏ Triggering CONCISE memory optimization (write_file detected, files: {files_implemented})") + return True + + # No optimization before any write_file + return False + + def apply_memory_optimization( + self, system_prompt: str, messages: List[Dict[str, Any]], files_implemented: int + ) -> List[Dict[str, Any]]: + """ + Apply memory optimization using concise approach + NEW LOGIC: Clear all history after write_file, keep only system_prompt + initial_plan + current tools + + Args: + system_prompt: Current system prompt + messages: Original message list + files_implemented: Number of files implemented so far + + Returns: + Optimized message list + """ + if not self.should_clear_memory_next: + # Before any write_file, return original messages + return messages + + # Apply concise memory optimization after write_file detection + # self.logger.info(f"๐Ÿงน CLEARING MEMORY after write_file - creating clean slate") + optimized_messages = self.create_concise_messages( + system_prompt, messages, files_implemented + ) + + # Clear the flag after applying optimization + self.should_clear_memory_next = False + + compression_ratio = ( + ((len(messages) - len(optimized_messages)) / len(messages) * 100) + if messages + else 0 + ) + print( + f"๐ŸŽฏ CONCISE optimization applied: {len(messages)} โ†’ {len(optimized_messages)} messages ({compression_ratio:.1f}% compression)" + ) + + return optimized_messages + + def clear_current_round_tool_results(self): + """Clear current round tool results (called when starting new round)""" + self.current_round_tool_results = [] + self.logger.info("๐Ÿงน Current round tool results cleared") + + def debug_concise_state(self, files_implemented: int = 0): + """Debug method to show current concise memory state""" + stats = self.get_memory_statistics(files_implemented) + + print("=" * 60) + print("๐ŸŽฏ CONCISE MEMORY AGENT STATE (Write-File-Based)") + print("=" * 60) + print(f"Last write_file detected: {stats['last_write_file_detected']}") + print(f"Should clear memory next: {stats['should_clear_memory_next']}") + print(f"Files implemented: {stats['implemented_files_tracked']}") + print(f"Current round: {stats['current_round']}") + print(f"Concise mode active: {stats['concise_mode_active']}") + print(f"Current round tool results: {stats['current_round_tool_results']}") + print(f"Essential tools recorded: {stats['essential_tools_recorded']}") + print(f"Implemented files tracked: {len(self.implemented_files)}") + print(f"Implemented files list: {self.implemented_files}") + print(f"Code summary file exists: {os.path.exists(self.code_summary_path)}") + print(f"Next Steps available: {stats['next_steps_available']}") + print(f"Next Steps length: {stats['next_steps_length']} chars") + if self.current_next_steps.strip(): + print(f"Next Steps preview: {self.current_next_steps[:100]}...") + print("") + print("๐Ÿ“‹ FILE TRACKING:") + print(f" Total files in plan: {stats['total_files_in_plan']}") + print(f" Files implemented: {stats['files_implemented_count']}") + print(f" Files remaining: {stats['files_remaining_count']}") + print(f" Progress: {stats['implementation_progress_percent']:.1f}%") + if stats["unimplemented_files_list"]: + print(f" Next possible files: {stats['unimplemented_files_list'][:3]}...") + print("") + print( + "๐Ÿ“Š NEW LOGIC: write_file โ†’ clear memory โ†’ accumulate tools โ†’ next write_file" + ) + print("๐Ÿ“Š NEXT STEPS: Stored separately from file, included in tool results") + print( + "๐Ÿ“Š FILE TRACKING: All files extracted from plan, unimplemented files guide LLM decisions" + ) + print("๐Ÿ“Š Essential Tools Tracked:") + essential_tools = [ + "read_code_mem", + "read_file", + "write_file", + "execute_python", + "execute_bash", + "search_code", + "search_reference_code", + "get_file_structure", + ] + for tool in essential_tools: + tool_count = sum( + 1 for r in self.current_round_tool_results if r["tool_name"] == tool + ) + print(f" - {tool}: {tool_count} calls") + print("=" * 60) diff --git a/DeepCode/workflows/agents/memory_agent_concise_index.py b/DeepCode/workflows/agents/memory_agent_concise_index.py new file mode 100644 index 00000000..abdcd9e9 --- /dev/null +++ b/DeepCode/workflows/agents/memory_agent_concise_index.py @@ -0,0 +1,2157 @@ +""" +Concise Memory Agent for Code Implementation Workflow + +This memory agent implements a focused approach: +1. Before first file: Normal conversation flow +2. After first file: Keep only system_prompt + initial_plan + current round tool results +3. Clean slate for each new code file generation + +Key Features: +- Preserves system prompt and initial plan always +- After first file generation, discards previous conversation history +- Keeps only current round tool results from essential tools: + * read_code_mem, read_file, write_file + * execute_python, execute_bash + * search_code, search_reference_code, get_file_structure +- Provides clean, focused input for next write_file operation +""" + +import json +import logging +import os +import time +from datetime import datetime +from typing import Dict, Any, List, Optional + + +class ConciseMemoryAgent: + """ + Concise Memory Agent - Focused Information Retention + + Core Philosophy: + - Preserve essential context (system prompt + initial plan) + - After first file generation, use clean slate approach + - Keep only current round tool results from all essential MCP tools + - Remove conversational clutter and previous tool calls + + Essential Tools Tracked: + - File Operations: read_code_mem, read_file, write_file + - Code Analysis: search_code, search_reference_code, get_file_structure + - Execution: execute_python, execute_bash + """ + + def __init__( + self, + initial_plan_content: str, + logger: Optional[logging.Logger] = None, + target_directory: Optional[str] = None, + default_models: Optional[Dict[str, str]] = None, + code_directory: Optional[str] = None, + ): + """ + Initialize Concise Memory Agent + + Args: + initial_plan_content: Content of initial_plan.txt + logger: Logger instance + target_directory: Target directory for saving summaries + default_models: Default models configuration from workflow + code_directory: Generated code directory path (e.g., target_directory/generate_code) + """ + self.logger = logger or self._create_default_logger() + self.initial_plan = initial_plan_content + + # Store default models configuration + self.default_models = default_models or { + "anthropic": "claude-sonnet-4-20250514", + "openai": "o3-mini", + "google": "gemini-2.0-flash", + } + + # Memory state tracking - new logic: trigger after each write_file + self.last_write_file_detected = ( + False # Track if write_file was called in current iteration + ) + self.should_clear_memory_next = False # Flag to clear memory in next round + self.current_round = 0 + + # Parse phase structure from initial plan + self.phase_structure = self._parse_phase_structure() + + # Memory configuration + if target_directory: + self.save_path = target_directory + else: + self.save_path = "./deepcode_lab/papers/1/" + + # Store code directory for file extraction + self.code_directory = code_directory or os.path.join( + self.save_path, "generate_code" + ) + + # Extract all files - prioritize generated directory over plan parsing + self.all_files_list = self._extract_all_files() + + # Code summary file path + self.code_summary_path = os.path.join( + self.save_path, "implement_code_summary.md" + ) + + # Current round tool results storage + self.current_round_tool_results = [] + + # Track all implemented files + self.implemented_files = [] + + # Store Next Steps information temporarily (not saved to file) + self.current_next_steps = "" + + self.logger.info( + f"Concise Memory Agent initialized with target directory: {self.save_path}" + ) + self.logger.info(f"Code directory: {self.code_directory}") + self.logger.info(f"Code summary will be saved to: {self.code_summary_path}") + # self.logger.info(f"๐Ÿค– Using models - Anthropic: {self.default_models['anthropic']}, OpenAI: {self.default_models['openai']}") + self.logger.info( + "๐Ÿ“ NEW LOGIC: Memory clearing triggered after each write_file call" + ) + + def _create_default_logger(self) -> logging.Logger: + """Create default logger""" + logger = logging.getLogger(f"{__name__}.ConciseMemoryAgent") + logger.setLevel(logging.INFO) + return logger + + def _parse_phase_structure(self) -> Dict[str, List[str]]: + """Parse implementation phases from initial plan""" + try: + phases = {} + lines = self.initial_plan.split("\n") + current_phase = None + + for line in lines: + if "Phase" in line and ":" in line: + # Extract phase name + phase_parts = line.split(":") + if len(phase_parts) >= 2: + current_phase = phase_parts[0].strip() + phases[current_phase] = [] + elif current_phase and line.strip().startswith("-"): + # This is a file in the current phase + file_line = line.strip()[1:].strip() + if file_line.startswith("`") and file_line.endswith("`"): + file_name = file_line[1:-1] + phases[current_phase].append(file_name) + elif current_phase and not line.strip(): + # Empty line might indicate end of phase + continue + elif current_phase and line.strip().startswith("###"): + # New section, end current phase + current_phase = None + + return phases + + except Exception as e: + self.logger.warning(f"Failed to parse phase structure: {e}") + return {} + + def _extract_all_files(self) -> List[str]: + """ + Extract all code files - prioritizes generated directory over plan parsing + + Strategy: + 1. First try to extract from the generated code directory (reliable) + 2. Fall back to plan parsing if directory doesn't exist yet + + Returns: + List of all file paths that should be implemented + """ + # Try extracting from generated directory first (more reliable) + if os.path.exists(self.code_directory): + files_from_dir = self._extract_files_from_generated_directory() + if files_from_dir: + self.logger.info( + f"๐Ÿ“ Extracted {len(files_from_dir)} files from generated directory" + ) + return files_from_dir + + # Fall back to plan parsing + self.logger.info( + "๐Ÿ“ Generated directory not found, extracting from plan (less reliable)" + ) + return self._extract_all_files_from_plan() + + def _extract_files_from_generated_directory(self) -> List[str]: + """ + Extract all code files from the generated code directory + This is more reliable than parsing the LLM-generated plan + + Returns: + List of relative file paths within the code directory + """ + code_files = [] + + # Define code file extensions to track + code_extensions = { + ".py", + ".js", + ".ts", + ".jsx", + ".tsx", + ".vue", + ".html", + ".css", + ".scss", + ".sass", + ".less", + ".json", + ".yaml", + ".yml", + ".toml", + ".xml", + ".ini", + ".cfg", + ".md", + ".rst", + ".txt", + ".sh", + ".bash", + ".zsh", + ".bat", + ".ps1", + ".cmd", + ".c", + ".cpp", + ".h", + ".hpp", + ".cc", + ".cxx", + ".java", + ".kt", + ".scala", + ".go", + ".rs", + ".php", + ".rb", + ".pl", + ".lua", + ".r", + ".sql", + } + + # Files and directories to exclude + exclude_patterns = { + "__pycache__", + ".pyc", + "node_modules", + ".git", + ".vscode", + ".idea", + "dist", + "build", + "output", + ".egg-info", + "venv", + ".venv", + "env", + ".env", + } + + try: + for root, dirs, files in os.walk(self.code_directory): + # Filter out excluded directories + dirs[:] = [ + d + for d in dirs + if d not in exclude_patterns and not d.startswith(".") + ] + + for file in files: + # Skip hidden files and excluded patterns + if file.startswith("."): + continue + + # Check if file has a code extension + has_code_ext = any( + file.lower().endswith(ext) for ext in code_extensions + ) + if not has_code_ext: + continue + + # Get full path and convert to relative path + full_path = os.path.join(root, file) + relative_path = os.path.relpath(full_path, self.code_directory) + + # Normalize path separators + relative_path = relative_path.replace(os.sep, "/") + + code_files.append(relative_path) + + # Sort for consistency + code_files = sorted(code_files) + + if code_files: + self.logger.info(f"๐Ÿ“„ Found {len(code_files)} code files in directory") + self.logger.info(f"๐Ÿ“„ Sample files: {code_files[:3]}...") + + return code_files + + except Exception as e: + self.logger.error(f"Failed to extract files from directory: {e}") + return [] + + def _extract_all_files_from_plan(self) -> List[str]: + """ + Extract all file paths from the file_structure section in initial plan + Handles multiple formats: tree structure, YAML, and simple lists + + Returns: + List of all file paths that should be implemented + """ + try: + lines = self.initial_plan.split("\n") + files = [] + + # Method 1: Try to extract from tree structure in file_structure section + files.extend(self._extract_from_tree_structure(lines)) + + # Method 2: If no files found, try to extract from simple list format + if not files: + files.extend(self._extract_from_simple_list(lines)) + + # Method 3: If still no files, try to extract from anywhere in the plan + if not files: + files.extend(self._extract_from_plan_content(lines)) + + # Clean and validate file paths + cleaned_files = self._clean_and_validate_files(files) + + # Log the extracted files + self.logger.info( + f"๐Ÿ“ Extracted {len(cleaned_files)} files from initial plan" + ) + if cleaned_files: + self.logger.info(f"๐Ÿ“ Sample files: {cleaned_files[:3]}...") + + return cleaned_files + + except Exception as e: + self.logger.error(f"Failed to extract files from initial plan: {e}") + return [] + + def _extract_from_tree_structure(self, lines: List[str]) -> List[str]: + """ + Extract files from tree structure format - Advanced algorithm with multi-strategy approach + + Strategy: + 1. Precise indentation-based depth calculation + 2. Smart directory vs file detection using multiple heuristics + 3. Robust path stack management with depth tracking + 4. Fallback to regex pattern matching if tree parsing fails + """ + files = [] + in_file_structure = False + + # Enhanced path tracking: store (depth, name) pairs + path_stack = [] # [(depth, dir_name), ...] + root_dir = None + + # Track the base indentation of tree structure + base_indent = None + + for line_num, line in enumerate(lines): + # === Section Boundary Detection === + if "file_structure:" in line or "file_structure |" in line: + in_file_structure = True + continue + + # End of file_structure section (next YAML key without indentation) + if ( + in_file_structure + and line.strip() + and not line.startswith(" ") + and ":" in line + ): + break + + if not in_file_structure: + continue + + if not line.strip(): + continue + + # Skip YAML comments and keys that are clearly not files + stripped = line.strip() + if stripped.startswith("#") or ( + stripped.endswith(":") and "/" not in stripped + ): + continue + + # === Root Directory Detection === + # Pattern: "project-name/" at minimal indentation, no tree chars + if stripped.endswith("/") and not any( + c in line for c in ["โ”œ", "โ””", "โ”‚", "โ”€"] + ): + indent = len(line) - len(line.lstrip()) + if indent <= 4: # Root level + root_dir = stripped.rstrip("/") + path_stack = [] + base_indent = None + self.logger.debug(f"๐ŸŒณ Detected root directory: {root_dir}") + continue + + # === Tree Structure Line Detection === + has_tree_chars = any(c in line for c in ["โ”œ", "โ””", "โ”‚", "โ”€"]) + if not has_tree_chars: + continue + + # === Calculate Precise Depth === + # Method: Count the actual tree structure symbols to determine hierarchy + indent = len(line) - len(line.lstrip()) + + # Set base indent on first tree line + if base_indent is None: + base_indent = indent + + # Count tree depth indicators + # Each "โ”‚ " or " " block represents one level + # "โ”œโ”€โ”€ " or "โ””โ”€โ”€ " marks the current item + tree_prefix = line[ + : line.find("โ”œ") + if "โ”œ" in line + else line.find("โ””") + if "โ””" in line + else len(line) + ] + + # Count depth by analyzing tree prefix structure + # Pattern: " โ”‚ โ”‚ โ”œโ”€โ”€ filename" -> depth 3 + # Pattern: " โ”œโ”€โ”€ filename" -> depth 1 + # Pattern: " โ”‚ โ”œโ”€โ”€ filename" -> depth 2 + + depth = 0 + i = 0 + while i < len(tree_prefix): + # Look for pipe or tree junction + if i + 4 <= len(tree_prefix): + chunk = tree_prefix[i : i + 4] + if "โ”‚" in chunk or all(c == " " for c in chunk): + depth += 1 + i += 4 + else: + i += 1 + else: + break + + # Fallback: use relative indentation + if depth == 0: + depth = max(1, (indent - base_indent) // 4 + 1) + + # === Clean and Extract Item Name === + item_name = line + # Remove all tree characters + for pattern in ["โ”œโ”€โ”€", "โ””โ”€โ”€", "โ”‚", "โ”œ", "โ””", "โ”€"]: + item_name = item_name.replace(pattern, "") + item_name = item_name.strip() + + # Remove inline comments + if "#" in item_name: + item_name = item_name.split("#")[0].strip() + + if not item_name or ":" in item_name: + continue + + # === Smart Directory vs File Detection === + is_directory = self._is_directory(item_name) + + # === Update Path Stack === + # Remove items deeper than current depth + path_stack = [(d, n) for d, n in path_stack if d < depth] + + if is_directory: + dir_name = item_name.rstrip("/") + path_stack.append((depth, dir_name)) + self.logger.debug(f" {' ' * depth}๐Ÿ“ {dir_name} (depth={depth})") + else: + # Construct full file path + path_parts = [root_dir] if root_dir else [] + path_parts.extend([name for _, name in path_stack]) + path_parts.append(item_name) + + full_path = "/".join(path_parts) + files.append(full_path) + self.logger.debug(f" {' ' * depth}๐Ÿ“„ {full_path}") + + return files + + def _is_directory(self, name: str) -> bool: + """ + Advanced directory detection using multiple heuristics + + Returns True if the name represents a directory, False if it's a file + """ + # Rule 1: Explicit directory marker + if name.endswith("/"): + return True + + # Rule 2: Has file extension -> definitely a file + basename = name.split("/")[-1] + if "." in basename: + # Check if it's a known file extension + known_extensions = [ + ".py", + ".js", + ".ts", + ".jsx", + ".tsx", + ".vue", + ".html", + ".css", + ".scss", + ".sass", + ".json", + ".yaml", + ".yml", + ".xml", + ".toml", + ".md", + ".txt", + ".rst", + ".sh", + ".bat", + ".ps1", + ".c", + ".cpp", + ".h", + ".hpp", + ".java", + ".go", + ".rs", + ".sql", + ".db", + ".env", + ".gitignore", + ".dockerignore", + ".lock", + ".sum", + ".mod", + ] + if any(basename.lower().endswith(ext) for ext in known_extensions): + return False + + # Has extension but not recognized -> might be config file, treat as file + if basename.count(".") == 1: + return False + + # Rule 3: Known special files without extensions + special_files = [ + "README", + "LICENSE", + "CHANGELOG", + "CONTRIBUTING", + "Makefile", + "Dockerfile", + "Vagrantfile", + "requirements.txt", + "setup.py", + "setup.cfg", + "package.json", + "package-lock.json", + "Cargo.toml", + "go.mod", + ] + if basename in special_files or basename.upper() in special_files: + return False + + # Rule 4: Common directory names (even without trailing /) + common_dirs = [ + "src", + "lib", + "app", + "core", + "api", + "web", + "client", + "server", + "config", + "configs", + "settings", + "data", + "datasets", + "models", + "model", + "utils", + "helpers", + "common", + "shared", + "tests", + "test", + "testing", + "__tests__", + "docs", + "documentation", + "scripts", + "bin", + "tools", + "assets", + "static", + "public", + "resources", + "components", + "views", + "pages", + "routes", + "services", + "controllers", + "handlers", + "middleware", + "middlewares", + "types", + "interfaces", + "schemas", + "experiments", + "notebooks", + "dist", + "build", + "output", + "node_modules", + "vendor", + "packages", + "__pycache__", + ".git", + ".vscode", + "training", + "evaluation", + "inference", + ] + if basename.lower() in common_dirs: + return True + + # Rule 5: Plural forms often indicate directories + if basename.endswith("s") and len(basename) > 3: + singular = basename[:-1] + if singular in common_dirs: + return True + + # Rule 6: Python package indicators + if basename == "__init__.py": + return False # This is a file + + # Default: if no extension and not a known file, likely a directory + return "." not in basename + + def _extract_from_simple_list(self, lines: List[str]) -> List[str]: + """Extract files from simple list format (- filename)""" + files = [] + + for line in lines: + line = line.strip() + if line.startswith("- ") and not line.startswith('- "'): + # Remove leading "- " and clean up + filename = line[2:].strip() + + # Remove quotes if present + if filename.startswith('"') and filename.endswith('"'): + filename = filename[1:-1] + + # Check if it looks like a file (has extension) + if "." in filename and "/" in filename: + files.append(filename) + + return files + + def _extract_from_plan_content(self, lines: List[str]) -> List[str]: + """ + Advanced fallback extraction: Extract files from anywhere in the plan content + Uses multiple regex patterns and intelligent filtering + """ + files = [] + import re + + # === Pattern 1: Standard file paths === + # Matches: path/to/file.py, src/model/apt_layer.py + pattern1 = r"([a-zA-Z0-9_\-]+(?:/[a-zA-Z0-9_\-]+)+\.[a-zA-Z0-9]+)" + + # === Pattern 2: Quoted file paths === + # Matches: "path/to/file.py", 'src/utils.py' + pattern2 = r'["\']([a-zA-Z0-9_\-]+(?:/[a-zA-Z0-9_\-]+)+\.[a-zA-Z0-9]+)["\']' + + # === Pattern 3: File paths with special characters === + # Matches: data/data_loader.py, __init__.py paths + pattern3 = r"([a-zA-Z0-9_\-]+(?:/[a-zA-Z0-9_\-]+)*/__init__\.py)" + pattern4 = r"([a-zA-Z0-9_\-]+(?:/[a-zA-Z0-9_\-]+)+\.(?:py|js|ts|jsx|tsx|html|css|md|txt|json|yaml|yml|xml|sql|sh|bat))" + + # === Pattern 5: Backtick-wrapped paths (in code blocks) === + pattern5 = r"`([a-zA-Z0-9_\-]+(?:/[a-zA-Z0-9_\-]+)+\.[a-zA-Z0-9]+)`" + + all_patterns = [pattern1, pattern2, pattern3, pattern4, pattern5] + + # Collect all potential matches + potential_files = set() + + for line in lines: + # Skip comment-only lines + stripped = line.strip() + if stripped.startswith("#") and not ("/" in stripped and "." in stripped): + continue + + # Apply all patterns + for pattern in all_patterns: + matches = re.findall(pattern, line) + potential_files.update(matches) + + # === Filter and validate matches === + code_extensions = { + ".py", + ".js", + ".ts", + ".jsx", + ".tsx", + ".vue", + ".html", + ".css", + ".scss", + ".sass", + ".less", + ".json", + ".yaml", + ".yml", + ".toml", + ".xml", + ".ini", + ".cfg", + ".md", + ".rst", + ".txt", + ".sh", + ".bash", + ".zsh", + ".bat", + ".ps1", + ".cmd", + ".c", + ".cpp", + ".h", + ".hpp", + ".cc", + ".cxx", + ".java", + ".kt", + ".scala", + ".go", + ".rs", + ".php", + ".rb", + ".pl", + ".lua", + ".r", + ".sql", + ".db", + ".dockerfile", + ".env", + ".gitignore", + ".lock", + ".sum", + ".mod", + } + + for file_path in potential_files: + # Must have path separator + if "/" not in file_path: + continue + + # Must have valid extension + has_valid_ext = any( + file_path.lower().endswith(ext) for ext in code_extensions + ) + if not has_valid_ext: + continue + + # Filter out obvious non-files + if any( + bad in file_path.lower() + for bad in [ + "http://", + "https://", + ".png", + ".jpg", + ".jpeg", + ".gif", + ".svg", + ".ico", + ] + ): + continue + + # Must not be too short (avoid false positives) + if len(file_path) < 5: + continue + + # Path components should be reasonable + parts = file_path.split("/") + if any(len(part) == 0 for part in parts): + continue + + files.append(file_path) + + # Sort for consistency + files = sorted(list(set(files))) + + return files + + def _clean_and_validate_files(self, files: List[str]) -> List[str]: + """ + Clean and validate extracted file paths - advanced filtering and deduplication + + Features: + 1. Remove duplicates while preserving order + 2. Normalize paths (handle ../, ./, double slashes) + 3. Filter out non-code files + 4. Smart deduplication (recognize same file with different path prefixes) + """ + cleaned_files = [] + seen_normalized = set() + + # Define code file extensions we want to track + code_extensions = { + ".py", + ".js", + ".ts", + ".jsx", + ".tsx", + ".vue", + ".html", + ".css", + ".scss", + ".sass", + ".less", + ".json", + ".yaml", + ".yml", + ".toml", + ".xml", + ".ini", + ".cfg", + ".md", + ".rst", + ".txt", + ".sh", + ".bash", + ".zsh", + ".bat", + ".ps1", + ".cmd", + ".c", + ".cpp", + ".h", + ".hpp", + ".cc", + ".cxx", + ".java", + ".kt", + ".scala", + ".go", + ".rs", + ".php", + ".rb", + ".pl", + ".lua", + ".r", + ".sql", + ".db", + ".dockerfile", + ".env", + ".gitignore", + ".lock", + ".sum", + ".mod", + } + + for file_path in files: + # === Step 1: Basic Cleaning === + cleaned_path = file_path.strip().strip('"').strip("'").strip("`") + + if not cleaned_path: + continue + + # Remove leading/trailing slashes + cleaned_path = cleaned_path.strip("/") + + # === Step 2: Path Normalization === + # Remove double slashes + while "//" in cleaned_path: + cleaned_path = cleaned_path.replace("//", "/") + + # Handle relative paths (remove ./ prefix) + if cleaned_path.startswith("./"): + cleaned_path = cleaned_path[2:] + + # === Step 3: Validate File Structure === + # Must have filename (not just directory) + if not cleaned_path or "/" not in cleaned_path: + # Single file without path - only accept if it has extension + if "." not in cleaned_path: + continue + + # Extract basename + basename = cleaned_path.split("/")[-1] + + # Skip directories (no file extension in basename) + if "." not in basename: + continue + + # === Step 4: Extension Validation === + # Only include files with code extensions + has_code_extension = any( + cleaned_path.lower().endswith(ext) for ext in code_extensions + ) + if not has_code_extension: + continue + + # === Step 5: Filter Invalid Patterns === + # Skip files that look like YAML keys or config entries + if ":" in cleaned_path and not any( + cleaned_path.endswith(ext) for ext in [".yaml", ".yml"] + ): + continue + + # Skip paths with invalid characters + if any( + char in cleaned_path for char in ['"', "'", "|", "<", ">", "*", "?"] + ): + continue + + # Skip obvious build/temp artifacts + if any( + part in cleaned_path + for part in [ + "__pycache__", + ".pyc", + "node_modules", + ".git/", + "dist/build", + ] + ): + continue + + # === Step 6: Smart Deduplication === + # Normalize for comparison (lowercase, remove common prefixes) + normalized_for_comparison = cleaned_path.lower() + + # Check if we've already seen this file (exact match) + if normalized_for_comparison in seen_normalized: + continue + + # Check for duplicate with different path (e.g., "src/model/apt_layer.py" vs "model/apt_layer.py") + # Keep the longer (more specific) path + is_duplicate = False + paths_to_remove = [] + + for existing_normalized in seen_normalized: + # If current path is suffix of existing, it's a shorter version - skip it + if existing_normalized.endswith("/" + normalized_for_comparison): + is_duplicate = True + break + + # If existing path is suffix of current, current is longer - replace existing + if normalized_for_comparison.endswith("/" + existing_normalized): + paths_to_remove.append(existing_normalized) + + if is_duplicate: + continue + + # Remove shorter versions + for path_to_remove in paths_to_remove: + seen_normalized.discard(path_to_remove) + # Also remove from cleaned_files list + cleaned_files = [ + f for f in cleaned_files if f.lower() != path_to_remove + ] + + # === Step 7: Add to Results === + seen_normalized.add(normalized_for_comparison) + cleaned_files.append(cleaned_path) + + return sorted(cleaned_files) + + def record_file_implementation( + self, file_path: str, implementation_content: str = "" + ): + """ + Record a newly implemented file (simplified version) + NEW LOGIC: File implementation is tracked via write_file tool detection + + Args: + file_path: Path of the implemented file + implementation_content: Content of the implemented file + """ + # Add file to implemented files list if not already present + if file_path not in self.implemented_files: + self.implemented_files.append(file_path) + + self.logger.info(f"๐Ÿ“ File implementation recorded: {file_path}") + + async def create_code_implementation_summary( + self, + client, + client_type: str, + file_path: str, + implementation_content: str, + files_implemented: int, + ) -> str: + """ + Create LLM-based code implementation summary after writing a file + Uses LLM to analyze and summarize the implemented code + + Args: + client: LLM client instance + client_type: Type of LLM client ("anthropic" or "openai") + file_path: Path of the implemented file + implementation_content: Content of the implemented file + files_implemented: Number of files implemented so far + + Returns: + LLM-generated formatted code implementation summary + """ + try: + # Record the file implementation first + self.record_file_implementation(file_path, implementation_content) + + # Create prompt for LLM summary + summary_prompt = self._create_code_summary_prompt( + file_path, implementation_content, files_implemented + ) + summary_messages = [{"role": "user", "content": summary_prompt}] + + # Get LLM-generated summary + llm_response = await self._call_llm_for_summary( + client, client_type, summary_messages + ) + llm_summary = llm_response.get("content", "") + + # Extract different sections from LLM summary + sections = self._extract_summary_sections(llm_summary) + + # Store Next Steps in temporary variable (not saved to file) + self.current_next_steps = sections.get("next_steps", "") + if self.current_next_steps: + self.logger.info("๐Ÿ“ Next Steps stored temporarily (not saved to file)") + + # Format summary with only Implementation Progress and Dependencies for file saving + file_summary_content = "" + if sections.get("core_purpose"): + file_summary_content += sections["core_purpose"] + "\n\n" + if sections.get("public_interface"): + file_summary_content += sections["public_interface"] + "\n\n" + if sections.get("internal_dependencies"): + file_summary_content += sections["internal_dependencies"] + "\n\n" + if sections.get("external_dependencies"): + file_summary_content += sections["external_dependencies"] + "\n\n" + if sections.get("implementation_notes"): + file_summary_content += sections["implementation_notes"] + "\n\n" + + # Create the formatted summary for file saving (without Next Steps) + formatted_summary = self._format_code_implementation_summary( + file_path, file_summary_content.strip(), files_implemented + ) + + # Save to implement_code_summary.md (append mode) - only Implementation Progress and Dependencies + await self._save_code_summary_to_file(formatted_summary, file_path) + + self.logger.info(f"Created and saved code summary for: {file_path}") + return formatted_summary + + except Exception as e: + self.logger.error( + f"Failed to create LLM-based code implementation summary: {e}" + ) + # Fallback to simple summary + return self._create_fallback_code_summary( + file_path, implementation_content, files_implemented + ) + + def _create_code_summary_prompt( + self, file_path: str, implementation_content: str, files_implemented: int + ) -> str: + """ + Create prompt for LLM to generate code implementation summary + + Args: + file_path: Path of the implemented file + implementation_content: Content of the implemented file + files_implemented: Number of files implemented so far + + Returns: + Prompt for LLM summarization + """ + current_round = self.current_round + + # Get formatted file lists + file_lists = self.get_formatted_files_lists() + implemented_files_list = file_lists["implemented"] + unimplemented_files_list = file_lists["unimplemented"] + + prompt = f"""You are an expert code implementation summarizer. Analyze the implemented code file and create a structured summary. + +**๐Ÿšจ CRITICAL: The files listed below are ALREADY IMPLEMENTED - DO NOT suggest them in Next Steps! ๐Ÿšจ** + +**All Previously Implemented Files:** +{implemented_files_list} + +**Remaining Unimplemented Files (choose ONLY from these for Next Steps):** +{unimplemented_files_list} + +**Current Implementation Context:** +- **File Implemented**: {file_path} +- **Current Round**: {current_round} +- **Total Files Implemented**: {files_implemented} + + +**Initial Plan Reference:** +{self.initial_plan[:]} + +**Implemented Code Content:** +``` +{implementation_content[:]} +``` + +**Required Summary Format:** + +**Core Purpose** (provide a general overview of the file's main responsibility): +- {{1-2 sentence description of file's main responsibility}} + +**Public Interface** (what other files can use, if any): +- Class {{ClassName}}: {{purpose}} | Key methods: {{method_names}} | Constructor params: {{params}} +- Function {{function_name}}({{params}}): {{purpose}} -> {{return_type}}: {{purpose}} +- Constants/Types: {{name}}: {{value/description}} + +**Internal Dependencies** (what this file imports/requires, if any): +- From {{module/file}}: {{specific_imports}} +- External packages: {{package_name}} - {{usage_context}} + +**External Dependencies** (what depends on this file, if any): +- Expected to be imported by: {{likely_consumer_files}} +- Key exports used elsewhere: {{main_interfaces}} + +**Implementation Notes**: (if any) +- Architecture decisions: {{key_choices_made}} +- Cross-File Relationships: {{how_files_work_together}} + +**Next Steps**: List the code file (ONLY ONE) that will be implemented in the next round (MUST choose from "Remaining Unimplemented Files" above) + Format: Code will be implemented: {{file_path}} + **NEVER suggest any file from the "All Previously Implemented Files" list!** + +**Instructions:** +- Be precise and concise +- Focus on function interfaces that other files will need +- Extract actual function signatures from the code +- **CRITICAL: For Next Steps, ONLY choose ONE file from the "Remaining Unimplemented Files" list above** +- **NEVER suggest implementing a file that is already in the implemented files list** +- Choose the next file based on logical dependencies and implementation order +- Use the exact format specified above + +**Summary:**""" + + return prompt + + # TODO: The prompt is not good, need to be improved + # **Implementation Progress**: List the code file completed in current round and core implementation ideas + # Format: {{file_path}}: {{core implementation ideas}} + + # **Dependencies**: According to the File Structure and initial plan, list functions that may be called by other files + # Format: {{file_path}}: Function {{function_name}}: core ideas--{{ideas}}; Required parameters--{{params}}; Return parameters--{{returns}} + # Required packages: {{packages}} + + def _extract_summary_sections(self, llm_summary: str) -> Dict[str, str]: + """ + Extract different sections from LLM-generated summary + + Args: + llm_summary: Raw LLM-generated summary text + + Returns: + Dictionary with extracted sections: core_purpose, public_interface, internal_dependencies, + external_dependencies, implementation_notes, next_steps + """ + sections = { + "core_purpose": "", + "public_interface": "", + "internal_dependencies": "", + "external_dependencies": "", + "implementation_notes": "", + "next_steps": "", + } + + try: + lines = llm_summary.split("\n") + current_section = None + current_content = [] + + for line in lines: + line_lower = line.lower().strip() + + # Check for section headers + if "core purpose" in line_lower: + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + current_section = "core_purpose" + current_content = [line] # Include the header + elif "public interface" in line_lower: + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + current_section = "public_interface" + current_content = [line] # Include the header + elif "internal dependencies" in line_lower: + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + current_section = "internal_dependencies" + current_content = [line] # Include the header + elif "external dependencies" in line_lower: + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + current_section = "external_dependencies" + current_content = [line] # Include the header + elif "implementation notes" in line_lower: + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + current_section = "implementation_notes" + current_content = [line] # Include the header + elif "next steps" in line_lower: + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + current_section = "next_steps" + current_content = [line] # Include the header + else: + # Add content to current section + if current_section: + current_content.append(line) + + # Don't forget the last section + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + + self.logger.info(f"๐Ÿ“‹ Extracted sections: {list(sections.keys())}") + + except Exception as e: + self.logger.error(f"Failed to extract summary sections: {e}") + # Fallback: put everything in core_purpose + sections["core_purpose"] = llm_summary + + return sections + + def _format_code_implementation_summary( + self, file_path: str, llm_summary: str, files_implemented: int + ) -> str: + """ + Format the LLM-generated summary into the final structure + + Args: + file_path: Path of the implemented file + llm_summary: LLM-generated summary content + files_implemented: Number of files implemented so far + + Returns: + Formatted summary + """ + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + # # Create formatted list of implemented files + # implemented_files_list = ( + # "\n".join([f"- {file}" for file in self.implemented_files]) + # if self.implemented_files + # else "- None yet" + # ) + + # formatted_summary = f"""# Code Implementation Summary + # **All Previously Implemented Files:** + # {implemented_files_list} + # **Generated**: {timestamp} + # **File Implemented**: {file_path} + # **Total Files Implemented**: {files_implemented} + + # {llm_summary} + + # --- + # *Auto-generated by Memory Agent* + # """ + formatted_summary = f"""# Code Implementation Summary +**Generated**: {timestamp} +**File Implemented**: {file_path} + +{llm_summary} + +--- +*Auto-generated by Memory Agent* +""" + return formatted_summary + + def _create_fallback_code_summary( + self, file_path: str, implementation_content: str, files_implemented: int + ) -> str: + """ + Create fallback summary when LLM is unavailable + + Args: + file_path: Path of the implemented file + implementation_content: Content of the implemented file + files_implemented: Number of files implemented so far + + Returns: + Fallback summary + """ + # Create formatted list of implemented files + implemented_files_list = ( + "\n".join([f"- {file}" for file in self.implemented_files]) + if self.implemented_files + else "- None yet" + ) + + summary = f"""# Code Implementation Summary +**All Previously Implemented Files:** +{implemented_files_list} +**Generated**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} +**File Implemented**: {file_path} +**Total Files Implemented**: {files_implemented} +**Summary failed to generate.** + +--- +*Auto-generated by Concise Memory Agent (Fallback Mode)* +""" + return summary + + async def _save_code_summary_to_file(self, new_summary: str, file_path: str): + """ + Append code implementation summary to implement_code_summary.md + Accumulates all implementations with clear separators + + Args: + new_summary: New summary content to append + file_path: Path of the file for which the summary was generated + """ + try: + # Create directory if it doesn't exist + os.makedirs(os.path.dirname(self.code_summary_path), exist_ok=True) + + # Check if file exists to determine if we need header + file_exists = os.path.exists(self.code_summary_path) + + # Open in append mode to accumulate all implementations + with open(self.code_summary_path, "a", encoding="utf-8") as f: + if not file_exists: + # Write header for new file + f.write("# Code Implementation Progress Summary\n") + f.write("*Accumulated implementation progress for all files*\n\n") + + # Add clear separator between implementations + f.write("\n" + "=" * 80 + "\n") + f.write( + f"## IMPLEMENTATION File {file_path}; ROUND {self.current_round} \n" + ) + f.write("=" * 80 + "\n\n") + + # Write the new summary + f.write(new_summary) + f.write("\n\n") + + self.logger.info( + f"Appended LLM-based code implementation summary to: {self.code_summary_path}" + ) + + except Exception as e: + self.logger.error(f"Failed to save code implementation summary: {e}") + + async def _call_llm_for_summary( + self, client, client_type: str, summary_messages: List[Dict] + ) -> Dict[str, Any]: + """ + Call LLM for code implementation summary generation ONLY + + This method is used only for creating code implementation summaries, + NOT for conversation summarization which has been removed. + """ + if client_type == "anthropic": + response = await client.messages.create( + model=self.default_models["anthropic"], + system="You are an expert code implementation summarizer. Create structured summaries of implemented code files that preserve essential information about functions, dependencies, and implementation approaches.", + messages=summary_messages, + max_tokens=5000, + temperature=0.2, + ) + + content = "" + if response and hasattr(response, "content") and response.content: + for block in response.content: + if block.type == "text": + content += block.text + else: + self.logger.warning("Anthropic response is empty or malformed") + + return {"content": content} + + elif client_type == "openai": + openai_messages = [ + { + "role": "system", + "content": "You are an expert code implementation summarizer. Create structured summaries of implemented code files that preserve essential information about functions, dependencies, and implementation approaches.", + } + ] + openai_messages.extend(summary_messages) + + # Try max_tokens and temperature first, fallback to max_completion_tokens without temperature if unsupported + try: + response = await client.chat.completions.create( + model=self.default_models["openai"], + messages=openai_messages, + max_tokens=5000, + temperature=0.2, + ) + except Exception as e: + if "max_tokens" in str(e) and "max_completion_tokens" in str(e): + # Retry with max_completion_tokens and no temperature for models that require it + response = await client.chat.completions.create( + model=self.default_models["openai"], + messages=openai_messages, + max_completion_tokens=5000, + ) + else: + raise + + # Safely extract content from response + if response and hasattr(response, "choices") and response.choices: + return {"content": response.choices[0].message.content or ""} + else: + self.logger.warning("OpenAI response is empty or malformed") + return {"content": ""} + + elif client_type == "google": + from google.genai import types + + # Convert messages to Gemini format + system_instruction = "You are an expert code implementation summarizer. Create structured summaries of implemented code files that preserve essential information about functions, dependencies, and implementation approaches." + + gemini_messages = [] + for msg in summary_messages: + role = msg.get("role", "user") + content = msg.get("content", "") + + # Convert role names: "assistant" -> "model" + if role == "assistant": + role = "model" + elif role not in ["user", "model"]: + role = "user" + + gemini_messages.append( + types.Content(role=role, parts=[types.Part.from_text(text=content)]) + ) + + config = types.GenerateContentConfig( + max_output_tokens=5000, + temperature=0.2, + system_instruction=system_instruction, + ) + + response = await client.aio.models.generate_content( + model=self.default_models.get("google", "gemini-2.0-flash"), + contents=gemini_messages, + config=config, + ) + + # Extract content from Gemini response + content = "" + if response and hasattr(response, "candidates") and response.candidates: + candidate = response.candidates[0] + if hasattr(candidate, "content") and candidate.content: + if hasattr(candidate.content, "parts") and candidate.content.parts: + for part in candidate.content.parts: + if hasattr(part, "text") and part.text: + content += part.text + + if not content: + self.logger.warning("Google response is empty or malformed") + + return {"content": content} + + else: + raise ValueError(f"Unsupported client type: {client_type}") + + def start_new_round(self, iteration: Optional[int] = None): + """Start a new dialogue round and reset tool results + + Args: + iteration: Optional iteration number from workflow to sync with current_round + """ + if iteration is not None: + # Sync with workflow iteration + self.current_round = iteration + # self.logger.info(f"๐Ÿ”„ Synced round with workflow iteration {iteration}") + else: + # Default behavior: increment round counter + self.current_round += 1 + self.logger.info(f"๐Ÿ”„ Started new round {self.current_round}") + + self.current_round_tool_results = [] # Clear previous round results + # Note: Don't reset last_write_file_detected and should_clear_memory_next here + # These flags persist across rounds until memory optimization is applied + # self.logger.info(f"๐Ÿ”„ Round {self.current_round} - Tool results cleared, memory flags preserved") + + def record_tool_result( + self, tool_name: str, tool_input: Dict[str, Any], tool_result: Any + ): + """ + Record tool result for current round and detect write_file calls + + Args: + tool_name: Name of the tool called + tool_input: Input parameters for the tool + tool_result: Result returned by the tool + """ + # Detect write_file calls to trigger memory clearing + if tool_name == "write_file": + self.last_write_file_detected = True + self.should_clear_memory_next = True + + # self.logger.info(f"๐Ÿ”„ WRITE_FILE DETECTED: {file_path} - Memory will be cleared in next round") + + # Only record specific tools that provide essential information + essential_tools = [ + # "read_code_mem", # Read code summary from implement_code_summary.md + # "read_file", # Read file contents + "write_file", # Write file contents (important for tracking implementations) + # "execute_python", # Execute Python code (for testing/validation) + "execute_bash", # Execute bash commands (for build/execution) + # "search_code", # Search code patterns + "search_reference_code", # Search reference code (if available) + # "get_file_structure", # Get file structure (for understanding project layout) + ] + + if tool_name in essential_tools: + tool_record = { + "tool_name": tool_name, + "tool_input": tool_input, + "tool_result": tool_result, + "timestamp": time.time(), + } + self.current_round_tool_results.append(tool_record) + # self.logger.info(f"๐Ÿ“Š Essential tool result recorded: {tool_name} ({len(self.current_round_tool_results)} total)") + + def should_use_concise_mode(self) -> bool: + """ + Check if concise memory mode should be used + + Returns: + True if first file has been generated and concise mode should be active + """ + return self.last_write_file_detected + + def create_concise_messages( + self, + system_prompt: str, + messages: List[Dict[str, Any]], + files_implemented: int, + ) -> List[Dict[str, Any]]: + """ + Create concise message list for LLM input + NEW LOGIC: Always clear after write_file, keep system_prompt + initial_plan + current round tools + + Args: + system_prompt: Current system prompt + messages: Original message list + files_implemented: Number of files implemented so far + + Returns: + Concise message list containing only essential information + """ + if not self.last_write_file_detected: + # Before any write_file, use normal flow + self.logger.info( + "๐Ÿ”„ Using normal conversation flow (before any write_file)" + ) + return messages + + # After write_file detection, use concise approach with clean slate + self.logger.info( + f"๐ŸŽฏ Using CONCISE memory mode - Clear slate after write_file, Round {self.current_round}" + ) + + concise_messages = [] + + # Get formatted file lists + file_lists = self.get_formatted_files_lists() + implemented_files_list = file_lists["implemented"] + unimplemented_files_list = file_lists["unimplemented"] + + # Debug output for unimplemented files (clean format without dashes) + unimplemented_files = self.get_unimplemented_files() + print("โœ… Unimplemented Files:") + for file_path in unimplemented_files: + print(f"{file_path}") + if self.current_next_steps.strip(): + print(f"\n๐Ÿ“‹ {self.current_next_steps}") + + # 1. Add initial plan message (always preserved) + initial_plan_message = { + "role": "user", + "content": f"""**Task: Implement code based on the following reproduction plan** + +**Code Reproduction Plan:** +{self.initial_plan} + +**Working Directory:** Current workspace + +**All Previously Implemented Files:** +{implemented_files_list} + +**Current Status:** {files_implemented} files implemented + +**Remaining Files to Implement:** +{unimplemented_files_list} + +**IMPORTANT:** If the remaining files list shows "All files implemented!", you MUST reply with "All files implemented" to complete the task. Do NOT continue calling tools. + +**Objective:** {"Reply 'All files implemented' to finish" if not unimplemented_files else "Continue implementation by analyzing dependencies and implementing the next required file according to the plan's priority order."}""", + } + + # Append Next Steps information if available + # if self.current_next_steps.strip(): + # initial_plan_message["content"] += ( + # f"\n\n**Next Steps (from previous analysis):**\n{self.current_next_steps}" + # ) + + concise_messages.append(initial_plan_message) + + # 2. Add Knowledge Base + knowledge_base_message = { + "role": "user", + "content": f"""**Below is the Knowledge Base of the LATEST implemented code file:** +{self._read_code_knowledge_base()} + +**Development Cycle - START HERE:** + +**FIRST - Check completion status:** +- If "Remaining Files to Implement" above shows "All files implemented!", reply "All files implemented" immediately + +**For NEW file implementation (if remaining files exist):** +1. `search_code_references` โ†’ OPTIONALLY search reference patterns for inspiration (use for reference only, original paper specs take priority) +2. Write_file can be used to implement the new component + +**Remember:** Stop and declare completion when all files are done!""", + } + if self.current_next_steps.strip(): + knowledge_base_message["content"] += ( + f"\n\n**Next Steps (from previous analysis):**\n{self.current_next_steps}" + ) + concise_messages.append(knowledge_base_message) + + # 3. Add current tool results (essential information for next file generation) + if self.current_round_tool_results: + tool_results_content = self._format_tool_results() + + # # Append Next Steps information if available + # if self.current_next_steps.strip(): + # tool_results_content += f"\n\n**Next Steps (from previous analysis):**\n{self.current_next_steps}" + + tool_results_message = { + "role": "user", + "content": f"""**Current Tool Results:** +{tool_results_content}""", + } + concise_messages.append(tool_results_message) + else: + # If no tool results yet, add guidance for next steps + guidance_content = f"""**Current Round:** {self.current_round} + +**Development Cycle - START HERE:** + +**For NEW file implementation:** +1. `search_code_references` โ†’ OPTIONALLY search reference patterns for inspiration (use for reference only, original paper specs take priority) +2. Write_file can be used to implement the new component""" + + # # Append Next Steps information if available (even when no tool results) + # if self.current_next_steps.strip(): + # guidance_content += f"\n\n**Next Steps (from previous analysis):**\n{self.current_next_steps}" + + guidance_message = { + "role": "user", + "content": guidance_content, + } + concise_messages.append(guidance_message) + # **Available Essential Tools:** read_code_mem, write_file, execute_python, execute_bash + # **Remember:** Start with read_code_mem when implementing NEW files to understand existing code. When all files are implemented, focus on testing and completion. Implement according to the original paper's specifications - any reference code is for inspiration only. + # self.logger.info(f"โœ… Concise messages created: {len(concise_messages)} messages (original: {len(messages)})") + return concise_messages + + def _read_code_knowledge_base(self) -> Optional[str]: + """ + Read the implement_code_summary.md file as code knowledge base + Returns all content from the file + + Returns: + Full content of the file if it exists, None otherwise + """ + try: + if os.path.exists(self.code_summary_path): + with open(self.code_summary_path, "r", encoding="utf-8") as f: + content = f.read().strip() + + if content: + # Return all content instead of just the latest entry + return content + else: + return None + else: + return None + + except Exception as e: + self.logger.error(f"Failed to read code knowledge base: {e}") + return None + + def _extract_latest_implementation_entry(self, content: str) -> Optional[str]: + """ + Extract the latest/final implementation entry from the implement_code_summary.md content + Uses a simpler approach to find the last implementation section + + Args: + content: Full content of implement_code_summary.md + + Returns: + Latest implementation entry content, or None if not found + """ + try: + import re + + # Pattern to match the start of implementation sections + section_pattern = ( + r"={80}\s*\n## IMPLEMENTATION File .+?; ROUND \d+\s*\n={80}" + ) + + # Find all implementation section starts + matches = list(re.finditer(section_pattern, content)) + + if not matches: + # No implementation sections found + lines = content.split("\n") + fallback_content = ( + "\n".join(lines[:10]) + "\n... (truncated for brevity)" + if len(lines) > 10 + else content + ) + self.logger.info( + "๐Ÿ“– No implementation sections found, using fallback content" + ) + return fallback_content + + # Get the start position of the last implementation section + last_match = matches[-1] + start_pos = last_match.start() + + # Take everything from the last section start to the end of content + latest_entry = content[start_pos:].strip() + + # self.logger.info(f"๐Ÿ“– Extracted latest implementation entry from knowledge base") + # print(f"DEBUG: Extracted content length: {len(latest_entry)}") + # print(f"DEBUG: First 200 chars: {latest_entry[:]}") + + return latest_entry + + except Exception as e: + self.logger.error(f"Failed to extract latest implementation entry: {e}") + # Return last 1000 characters as fallback + return content[-500:] if len(content) > 500 else content + + def _format_tool_results(self) -> str: + """ + Format current round tool results for LLM input + + Returns: + Formatted string of tool results + """ + if not self.current_round_tool_results: + return "No tool results in current round." + + formatted_results = [] + + for result in self.current_round_tool_results: + tool_name = result["tool_name"] + tool_input = result["tool_input"] + tool_result = result["tool_result"] + + # Format based on tool type + if tool_name == "read_code_mem": + file_path = tool_input.get("file_path", "unknown") + formatted_results.append(f""" +**read_code_mem Result for {file_path}:** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "read_file": + file_path = tool_input.get("file_path", "unknown") + formatted_results.append(f""" +**read_file Result for {file_path}:** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "write_file": + file_path = tool_input.get("file_path", "unknown") + formatted_results.append(f""" +**write_file Result for {file_path}:** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "execute_python": + code_snippet = ( + tool_input.get("code", "")[:50] + "..." + if len(tool_input.get("code", "")) > 50 + else tool_input.get("code", "") + ) + formatted_results.append(f""" +**execute_python Result (code: {code_snippet}):** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "execute_bash": + command = tool_input.get("command", "unknown") + formatted_results.append(f""" +**execute_bash Result (command: {command}):** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "search_code": + pattern = tool_input.get("pattern", "unknown") + file_pattern = tool_input.get("file_pattern", "") + formatted_results.append(f""" +**search_code Result (pattern: {pattern}, files: {file_pattern}):** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "search_reference_code": + target_file = tool_input.get("target_file", "unknown") + keywords = tool_input.get("keywords", "") + formatted_results.append(f""" +**search_reference_code Result for {target_file} (keywords: {keywords}):** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "get_file_structure": + directory = tool_input.get( + "directory_path", tool_input.get("path", "current") + ) + formatted_results.append(f""" +**get_file_structure Result for {directory}:** +{self._format_tool_result_content(tool_result)} +""") + + return "\n".join(formatted_results) + + def _format_tool_result_content(self, tool_result: Any) -> str: + """ + Format tool result content for display + + Args: + tool_result: Tool result to format + + Returns: + Formatted string representation + """ + if isinstance(tool_result, str): + # Try to parse as JSON for better formatting + try: + result_data = json.loads(tool_result) + if isinstance(result_data, dict): + # Format key information + if result_data.get("status") == "summary_found": + return ( + f"Summary found:\n{result_data.get('summary_content', '')}" + ) + elif result_data.get("status") == "no_summary": + return "No summary available" + else: + return json.dumps(result_data, indent=2) + else: + return str(result_data) + except json.JSONDecodeError: + return tool_result + else: + return str(tool_result) + + def get_memory_statistics(self, files_implemented: int = 0) -> Dict[str, Any]: + """Get memory agent statistics""" + unimplemented_files = self.get_unimplemented_files() + return { + "last_write_file_detected": self.last_write_file_detected, + "should_clear_memory_next": self.should_clear_memory_next, + "current_round": self.current_round, + "concise_mode_active": self.should_use_concise_mode(), + "current_round_tool_results": len(self.current_round_tool_results), + "essential_tools_recorded": [ + r["tool_name"] for r in self.current_round_tool_results + ], + "implemented_files_tracked": files_implemented, + "implemented_files_list": self.implemented_files.copy(), + "phases_parsed": len(self.phase_structure), + "next_steps_available": bool(self.current_next_steps.strip()), + "next_steps_length": len(self.current_next_steps.strip()) + if self.current_next_steps + else 0, + # File tracking statistics + "total_files_in_plan": len(self.all_files_list), + "files_implemented_count": len(self.implemented_files), + "files_remaining_count": len(unimplemented_files), + "all_files_list": self.all_files_list.copy(), + "unimplemented_files_list": unimplemented_files, + "implementation_progress_percent": ( + len(self.implemented_files) / len(self.all_files_list) * 100 + ) + if self.all_files_list + else 0, + } + + def get_implemented_files(self) -> List[str]: + """Get list of all implemented files""" + return self.implemented_files.copy() + + def get_all_files_list(self) -> List[str]: + """Get list of all files that should be implemented according to the plan""" + return self.all_files_list.copy() + + def refresh_files_list_from_directory(self) -> bool: + """ + Refresh the files list by extracting from the generated directory + Useful when the directory structure has been updated after initialization + + Returns: + True if successfully refreshed from directory, False if fell back to plan + """ + if os.path.exists(self.code_directory): + files_from_dir = self._extract_files_from_generated_directory() + if files_from_dir: + old_count = len(self.all_files_list) + self.all_files_list = files_from_dir + new_count = len(self.all_files_list) + self.logger.info( + f"๐Ÿ”„ Files list refreshed from directory: {old_count} โ†’ {new_count} files" + ) + return True + + self.logger.warning("Cannot refresh from directory, keeping current list") + return False + + def get_unimplemented_files(self) -> List[str]: + """ + Get list of files that haven't been implemented yet + Uses fuzzy path matching to handle partial paths + + Returns: + List of file paths that still need to be implemented + """ + + # def is_implemented(plan_file: str) -> bool: + # """Check if a file from plan is implemented (with fuzzy matching)""" + # # Normalize paths for comparison + # plan_file_normalized = plan_file.replace("\\", "/").strip("/") + # plan_filename = plan_file_normalized.split("/")[-1] # Extract filename + + # for impl_file in self.implemented_files: + # impl_file_normalized = impl_file.replace("\\", "/").strip("/") + # impl_filename = impl_file_normalized.split("/")[-1] # Extract filename + + # # Strategy 1: Exact path match + # if plan_file_normalized == impl_file_normalized: + # return True + + # # Strategy 2: One path ends with the other (partial path match) + # if plan_file_normalized.endswith( + # impl_file_normalized + # ) or impl_file_normalized.endswith(plan_file_normalized): + # # Ensure match is at a path boundary (not middle of directory name) + # if ( + # plan_file_normalized.endswith("/" + impl_file_normalized) + # or impl_file_normalized.endswith("/" + plan_file_normalized) + # ): + # return True + + # # Strategy 3: Same filename (fallback for different directory structures) + # # Only match if filenames are identical and reasonably unique (length > 5) + # if (plan_filename == impl_filename and len(plan_filename) > 5): + # return True + + # return False + def is_implemented(plan_file: str) -> bool: + """Check if a file from plan is implemented (with fuzzy matching)""" + # Normalize paths for comparison + plan_file_normalized = plan_file.replace("\\", "/").strip("/") + + for impl_file in self.implemented_files: + impl_file_normalized = impl_file.replace("\\", "/").strip("/") + + # Check if plan_file ends with impl_file (partial path match) + # or impl_file ends with plan_file (reverse partial match) + if plan_file_normalized.endswith( + impl_file_normalized + ) or impl_file_normalized.endswith(plan_file_normalized): + # Ensure match is at a path boundary (not middle of directory name) + if ( + plan_file_normalized.endswith("/" + impl_file_normalized) + or plan_file_normalized == impl_file_normalized + or impl_file_normalized.endswith("/" + plan_file_normalized) + ): + return True + return False + + # unimplemented = [f for f in self.all_files_list if not is_implemented(f)] + # return unimplemented + + unimplemented = [f for f in self.all_files_list if not is_implemented(f)] + return unimplemented + + def get_formatted_files_lists(self) -> Dict[str, str]: + """ + Get formatted strings for implemented and unimplemented files + + Returns: + Dictionary with 'implemented' and 'unimplemented' formatted lists + """ + implemented_list = ( + "\n".join([f"- {file}" for file in self.implemented_files]) + if self.implemented_files + else "- None yet" + ) + + unimplemented_files = self.get_unimplemented_files() + unimplemented_list = ( + "\n".join([f"- {file}" for file in unimplemented_files]) + if unimplemented_files + else "- All files implemented!" + ) + + return {"implemented": implemented_list, "unimplemented": unimplemented_list} + + def get_current_next_steps(self) -> str: + """Get the current Next Steps information""" + return self.current_next_steps + + def clear_next_steps(self): + """Clear the stored Next Steps information""" + if self.current_next_steps.strip(): + self.logger.info("๐Ÿงน Next Steps information cleared") + self.current_next_steps = "" + + def set_next_steps(self, next_steps: str): + """Manually set Next Steps information""" + self.current_next_steps = next_steps + self.logger.info( + f"๐Ÿ“ Next Steps manually set ({len(next_steps.strip())} chars)" + ) + + def should_trigger_memory_optimization( + self, messages: List[Dict[str, Any]], files_implemented: int = 0 + ) -> bool: + """ + Check if memory optimization should be triggered + NEW LOGIC: Trigger after write_file has been detected + + Args: + messages: Current message list + files_implemented: Number of files implemented so far + + Returns: + True if concise mode should be applied + """ + # Trigger if we detected write_file and should clear memory + if self.should_clear_memory_next: + # self.logger.info(f"๐ŸŽฏ Triggering CONCISE memory optimization (write_file detected, files: {files_implemented})") + return True + + # No optimization before any write_file + return False + + def apply_memory_optimization( + self, system_prompt: str, messages: List[Dict[str, Any]], files_implemented: int + ) -> List[Dict[str, Any]]: + """ + Apply memory optimization using concise approach + NEW LOGIC: Clear all history after write_file, keep only system_prompt + initial_plan + current tools + + Args: + system_prompt: Current system prompt + messages: Original message list + files_implemented: Number of files implemented so far + + Returns: + Optimized message list + """ + if not self.should_clear_memory_next: + # Before any write_file, return original messages + return messages + + # Apply concise memory optimization after write_file detection + # self.logger.info(f"๐Ÿงน CLEARING MEMORY after write_file - creating clean slate") + optimized_messages = self.create_concise_messages( + system_prompt, messages, files_implemented + ) + + # Clear the flag after applying optimization + self.should_clear_memory_next = False + + compression_ratio = ( + ((len(messages) - len(optimized_messages)) / len(messages) * 100) + if messages + else 0 + ) + print( + f"๐ŸŽฏ CONCISE optimization applied: {len(messages)} โ†’ {len(optimized_messages)} messages ({compression_ratio:.1f}% compression)" + ) + + return optimized_messages + + def clear_current_round_tool_results(self): + """Clear current round tool results (called when starting new round)""" + self.current_round_tool_results = [] + self.logger.info("๐Ÿงน Current round tool results cleared") + + def debug_concise_state(self, files_implemented: int = 0): + """Debug method to show current concise memory state""" + stats = self.get_memory_statistics(files_implemented) + + print("=" * 60) + print("๐ŸŽฏ CONCISE MEMORY AGENT STATE (Write-File-Based)") + print("=" * 60) + print(f"Last write_file detected: {stats['last_write_file_detected']}") + print(f"Should clear memory next: {stats['should_clear_memory_next']}") + print(f"Files implemented: {stats['implemented_files_tracked']}") + print(f"Current round: {stats['current_round']}") + print(f"Concise mode active: {stats['concise_mode_active']}") + print(f"Current round tool results: {stats['current_round_tool_results']}") + print(f"Essential tools recorded: {stats['essential_tools_recorded']}") + print(f"Implemented files tracked: {len(self.implemented_files)}") + print(f"Implemented files list: {self.implemented_files}") + print(f"Code summary file exists: {os.path.exists(self.code_summary_path)}") + print(f"Next Steps available: {stats['next_steps_available']}") + print(f"Next Steps length: {stats['next_steps_length']} chars") + if self.current_next_steps.strip(): + print(f"Next Steps preview: {self.current_next_steps[:100]}...") + print("") + print("๐Ÿ“‹ FILE TRACKING:") + print(f" Total files in plan: {stats['total_files_in_plan']}") + print(f" Files implemented: {stats['files_implemented_count']}") + print(f" Files remaining: {stats['files_remaining_count']}") + print(f" Progress: {stats['implementation_progress_percent']:.1f}%") + if stats["unimplemented_files_list"]: + print(f" Next possible files: {stats['unimplemented_files_list'][:3]}...") + print("") + print( + "๐Ÿ“Š NEW LOGIC: write_file โ†’ clear memory โ†’ accumulate tools โ†’ next write_file" + ) + print("๐Ÿ“Š NEXT STEPS: Stored separately from file, included in tool results") + print( + "๐Ÿ“Š FILE TRACKING: All files extracted from plan, unimplemented files guide LLM decisions" + ) + print("๐Ÿ“Š Essential Tools Tracked:") + essential_tools = [ + "read_code_mem", + "read_file", + "write_file", + "execute_python", + "execute_bash", + "search_code", + "search_reference_code", + "get_file_structure", + ] + for tool in essential_tools: + tool_count = sum( + 1 for r in self.current_round_tool_results if r["tool_name"] == tool + ) + print(f" - {tool}: {tool_count} calls") + print("=" * 60) diff --git a/DeepCode/workflows/agents/memory_agent_concise_multi.py b/DeepCode/workflows/agents/memory_agent_concise_multi.py new file mode 100644 index 00000000..ee82dfef --- /dev/null +++ b/DeepCode/workflows/agents/memory_agent_concise_multi.py @@ -0,0 +1,1708 @@ +""" +Concise Memory Agent for Code Implementation Workflow - Multi-File Only Support + +This memory agent implements a focused approach with ONLY multi-file capabilities: +1. Before first batch: Normal conversation flow +2. After first batch: Keep only system_prompt + initial_plan + current round tool results +3. Clean slate for each new code batch generation +4. MULTI-FILE ONLY: Support for summarizing multiple files simultaneously (max 5) + +Key Features: +- Preserves system prompt and initial plan always +- After first batch generation, discards previous conversation history +- Keeps only current round tool results from essential tools: + * read_multiple_files, write_multiple_files + * execute_python, execute_bash + * search_code, search_reference_code, get_file_structure +- Provides clean, focused input for next write_multiple_files operation +- MULTI-FILE ONLY: No single file support +- FILE TRACKING: Gets ALL file information from workflow, no internal tracking +""" + +import json +import logging +import os +import time +from datetime import datetime +from typing import Dict, Any, List, Optional + + +class ConciseMemoryAgent: + """ + Concise Memory Agent - Focused Information Retention with MULTI-FILE ONLY Support + + Core Philosophy: + - Preserve essential context (system prompt + initial plan) + - After first batch generation, use clean slate approach + - Keep only current round tool results from multi-file MCP tools + - Remove conversational clutter and previous tool calls + - MULTI-FILE ONLY: Support for multiple file implementations in single operation + - FILE TRACKING: Receives ALL file information from workflow (no internal tracking) + + Essential Tools Tracked: + - Multi-File Operations: read_multiple_files, write_multiple_files + - Code Analysis: search_code, search_reference_code, get_file_structure + - Execution: execute_python, execute_bash + """ + + def __init__( + self, + initial_plan_content: str, + logger: Optional[logging.Logger] = None, + target_directory: Optional[str] = None, + default_models: Optional[Dict[str, str]] = None, + max_files_per_batch: int = 3, + ): + """ + Initialize Concise Memory Agent with MULTI-FILE ONLY support + + Args: + initial_plan_content: Content of initial_plan.txt + logger: Logger instance + target_directory: Target directory for saving summaries + default_models: Default models configuration from workflow + max_files_per_batch: Maximum number of files to implement simultaneously (default: 3) + """ + self.logger = logger or self._create_default_logger() + self.initial_plan = initial_plan_content + self.max_files_per_batch = max_files_per_batch + + # Store default models configuration + self.default_models = default_models or { + "anthropic": "claude-sonnet-4-20250514", + "openai": "o3-mini", + "google": "gemini-2.0-flash", + } + + # Memory state tracking - new logic: trigger after each write_multiple_files + self.last_write_multiple_files_detected = ( + False # Track if write_multiple_files was called in current iteration + ) + self.should_clear_memory_next = False # Flag to clear memory in next round + self.current_round = 0 + + # self.phase_structure = self._parse_phase_structure() + + # Memory configuration + if target_directory: + self.save_path = target_directory + else: + self.save_path = "./deepcode_lab/papers/1/" + + # Code summary file path + self.code_summary_path = os.path.join( + self.save_path, "implement_code_summary.md" + ) + + # Current round tool results storage + self.current_round_tool_results = [] + + self.logger.info( + f"Concise Memory Agent initialized with target directory: {self.save_path}" + ) + self.logger.info(f"Code summary will be saved to: {self.code_summary_path}") + self.logger.info(f"Max files per batch: {self.max_files_per_batch}") + self.logger.info( + "๐Ÿ“ MULTI-FILE LOGIC: Memory clearing triggered after each write_multiple_files call" + ) + self.logger.info( + "๐Ÿ†• MULTI-FILE ONLY: No single file support - batch operations only" + ) + self.logger.info( + "๐Ÿ“Š FILE TRACKING: ALL file information received from workflow (no internal tracking)" + ) + + def _create_default_logger(self) -> logging.Logger: + """Create default logger""" + logger = logging.getLogger(f"{__name__}.ConciseMemoryAgent") + logger.setLevel(logging.INFO) + return logger + + async def create_multi_code_implementation_summary( + self, + client, + client_type: str, + file_implementations: Dict[str, str], + files_implemented: int, + implemented_files: List[str], # Receive from workflow + ) -> str: + """ + Create LLM-based code implementation summary for multiple files + ONLY AVAILABLE METHOD: Handles multiple files simultaneously with separate summaries for each + + Args: + client: LLM client instance + client_type: Type of LLM client ("anthropic" or "openai") + file_implementations: Dictionary mapping file_path to implementation_content + files_implemented: Number of files implemented so far + implemented_files: List of all implemented files (from workflow) + + Returns: + LLM-generated formatted code implementation summaries for all files + """ + try: + # Validate input + if not file_implementations: + raise ValueError("No file implementations provided") + + if len(file_implementations) > self.max_files_per_batch: + raise ValueError( + f"Too many files provided ({len(file_implementations)}), max is {self.max_files_per_batch}" + ) + + # Create prompt for LLM summary of multiple files + summary_prompt = self._create_multi_code_summary_prompt( + file_implementations, files_implemented, implemented_files + ) + summary_messages = [{"role": "user", "content": summary_prompt}] + + # Get LLM-generated summary + llm_response = await self._call_llm_for_summary( + client, client_type, summary_messages + ) + llm_summary = llm_response.get("content", "") + + # Extract sections for each file and next steps + multi_sections = self._extract_multi_summary_sections( + llm_summary, file_implementations.keys() + ) + + # Format and save summary for each file (WITHOUT Next Steps) + all_formatted_summaries = [] + + for file_path in file_implementations.keys(): + file_sections = multi_sections.get("files", {}).get(file_path, {}) + + # Format summary with ONLY Implementation Progress and Dependencies for file saving + file_summary_content = "" + if file_sections.get("core_purpose"): + file_summary_content += file_sections["core_purpose"] + "\n\n" + if file_sections.get("public_interface"): + file_summary_content += file_sections["public_interface"] + "\n\n" + if file_sections.get("internal_dependencies"): + file_summary_content += ( + file_sections["internal_dependencies"] + "\n\n" + ) + if file_sections.get("external_dependencies"): + file_summary_content += ( + file_sections["external_dependencies"] + "\n\n" + ) + if file_sections.get("implementation_notes"): + file_summary_content += ( + file_sections["implementation_notes"] + "\n\n" + ) + + # Create the formatted summary for file saving (WITHOUT Next Steps) + formatted_summary = self._format_code_implementation_summary( + file_path, file_summary_content.strip(), files_implemented + ) + + all_formatted_summaries.append(formatted_summary) + + # Save to implement_code_summary.md (append mode) - ONLY Implementation Progress and Dependencies + await self._save_code_summary_to_file(formatted_summary, file_path) + + # Combine all summaries for return + combined_summary = "\n".join(all_formatted_summaries) + + self.logger.info( + f"Created and saved multi-file code summaries for {len(file_implementations)} files" + ) + + return combined_summary + + except Exception as e: + self.logger.error( + f"Failed to create LLM-based multi-file code implementation summary: {e}" + ) + # Fallback to simple summary for each file + return self._create_fallback_multi_code_summary( + file_implementations, files_implemented + ) + + def _create_multi_code_summary_prompt( + self, + file_implementations: Dict[str, str], + files_implemented: int, + implemented_files: List[str], + ) -> str: + """ + Create prompt for LLM to generate multi-file code implementation summary + + Args: + file_implementations: Dictionary mapping file_path to implementation_content + files_implemented: Number of files implemented so far + implemented_files: List of all implemented files (from workflow) + + Returns: + Prompt for LLM multi-file summarization + """ + + # Format file lists using workflow data + implemented_files_list = ( + "\n".join([f"- {file}" for file in implemented_files]) + if implemented_files + else "- None yet" + ) + + # Note: We don't have unimplemented files list anymore - workflow will provide when needed + + # Format file implementations for the prompt + implementation_sections = [] + for file_path, content in file_implementations.items(): + implementation_sections.append(f""" + **File: {file_path}** + {content} + """) + + files_list = list(file_implementations.keys()) + files_count = len(files_list) + + prompt = f"""You are an expert code implementation summarizer. Analyze the {files_count} implemented code files and create structured summaries for each. + +**All Previously Implemented Files:** +{implemented_files_list} + +**Current Implementation Context:** +- **Files Implemented**: {', '.join(files_list)} +- **Total Files Implemented**: {files_implemented} +- **Files in This Batch**: {files_count} + +**Initial Plan Reference:** +{self.initial_plan[:]} + +**Implemented Code Content:** +{''.join(implementation_sections)} + +**Required Summary Format:** + +**FOR EACH FILE, provide separate sections:** + +**File: {{file_path}}** +**Core Purpose** (provide a general overview of the file's main responsibility): +- {{1-2 sentence description of file's main responsibility}} + +**Public Interface** (what other files can use, if any): +- Class {{ClassName}}: {{purpose}} | Key methods: {{method_names}} | Constructor params: {{params}} +- Function {{function_name}}({{params}}): {{purpose}} -> {{return_type}}: {{purpose}} +- Constants/Types: {{name}}: {{value/description}} + +**Internal Dependencies** (what this file imports/requires, if any): +- From {{module/file}}: {{specific_imports}} +- External packages: {{package_name}} - {{usage_context}} + +**External Dependencies** (what depends on this file, if any): +- Expected to be imported by: {{likely_consumer_files}} +- Key exports used elsewhere: {{main_interfaces}} + +**Implementation Notes**: (if any) +- Architecture decisions: {{key_choices_made}} +- Cross-File Relationships: {{how_files_work_together}} + +[Repeat for all {files_count} files...] + +**Instructions:** +- Provide separate Implementation Progress and Dependencies sections for each of the {files_count} files +- Be precise and concise for each file +- Focus on function interfaces that other files will need +- Extract actual function signatures from the code +- Use the exact format specified above + +**Summary:**""" + + return prompt + + def _extract_multi_summary_sections( + self, llm_summary: str, file_paths: List[str] + ) -> Dict[str, Any]: + """ + Extract different sections from LLM-generated multi-file summary + """ + result = { + "files": {}, + } + + try: + # Convert dict_keys to list if needed + if hasattr(file_paths, "keys"): + file_paths = list(file_paths) + elif not isinstance(file_paths, list): + file_paths = list(file_paths) + + lines = llm_summary.split("\n") + current_file = None + current_section = None + current_content = [] + file_sections = {} + + for i, line in enumerate(lines): + line_lower = line.lower().strip() + original_line = line.strip() + + # Skip empty lines + if not original_line: + if current_section: + current_content.append(line) + continue + + # File header detection + if ( + "**file:" in line_lower or "file:" in line_lower + ) and "**" in original_line: + # Save previous section + if current_file and current_section and current_content: + if current_file not in file_sections: + file_sections[current_file] = {} + file_sections[current_file][current_section] = "\n".join( + current_content + ).strip() + + # Extract file path + file_header = original_line.lower() + if "**file:" in file_header: + file_header = original_line[ + original_line.lower().find("file:") + 5 : + ] + if "**" in file_header: + file_header = file_header[: file_header.find("**")] + else: + file_header = original_line[ + original_line.lower().find("file:") + 5 : + ] + + file_header = file_header.strip() + current_file = None + + # File matching + for file_path in file_paths: + file_name = file_path.split("/")[-1] + if ( + file_path in file_header + or file_header in file_path + or file_name in file_header + or file_header in file_name + ): + current_file = file_path + break + + current_section = None + current_content = [] + continue + + # Section detection within files + if current_file: + section_matched = False + + if "core purpose" in line_lower and "**" in original_line: + if current_section and current_content: + if current_file not in file_sections: + file_sections[current_file] = {} + file_sections[current_file][current_section] = "\n".join( + current_content + ).strip() + current_section = "core_purpose" + current_content = [] + section_matched = True + elif "public interface" in line_lower and "**" in original_line: + if current_section and current_content: + if current_file not in file_sections: + file_sections[current_file] = {} + file_sections[current_file][current_section] = "\n".join( + current_content + ).strip() + current_section = "public_interface" + current_content = [] + section_matched = True + elif ( + "internal dependencies" in line_lower and "**" in original_line + ): + if current_section and current_content: + if current_file not in file_sections: + file_sections[current_file] = {} + file_sections[current_file][current_section] = "\n".join( + current_content + ).strip() + current_section = "internal_dependencies" + current_content = [] + section_matched = True + elif ( + "external dependencies" in line_lower and "**" in original_line + ): + if current_section and current_content: + if current_file not in file_sections: + file_sections[current_file] = {} + file_sections[current_file][current_section] = "\n".join( + current_content + ).strip() + current_section = "external_dependencies" + current_content = [] + section_matched = True + elif "implementation notes" in line_lower and "**" in original_line: + if current_section and current_content: + if current_file not in file_sections: + file_sections[current_file] = {} + file_sections[current_file][current_section] = "\n".join( + current_content + ).strip() + current_section = "implementation_notes" + current_content = [] + section_matched = True + + # If no section header matched, add to current content + if not section_matched and current_section: + current_content.append(line) + + # Save the final section + if current_file and current_section and current_content: + if current_file not in file_sections: + file_sections[current_file] = {} + file_sections[current_file][current_section] = "\n".join( + current_content + ).strip() + + # Build final result + for file_path in file_paths: + sections = file_sections.get(file_path, {}) + result["files"][file_path] = {} + if "core_purpose" in sections: + result["files"][file_path]["core_purpose"] = ( + "**Core Purpose**:\n" + sections["core_purpose"] + ) + if "public_interface" in sections: + result["files"][file_path]["public_interface"] = ( + "**Public Interface**:\n" + sections["public_interface"] + ) + if "implementation_notes" in sections: + result["files"][file_path]["implementation_notes"] = ( + "**Implementation Notes**:\n" + sections["implementation_notes"] + ) + if "internal_dependencies" in sections: + result["files"][file_path]["internal_dependencies"] = ( + "**Internal Dependencies**:\n" + + sections["internal_dependencies"] + ) + if "external_dependencies" in sections: + result["files"][file_path]["external_dependencies"] = ( + "**External Dependencies**:\n" + + sections["external_dependencies"] + ) + + self.logger.info( + f"๐Ÿ“‹ Extracted multi-file sections for {len(result['files'])} files" + ) + + except Exception as e: + self.logger.error(f"Failed to extract multi-file summary sections: {e}") + self.logger.error(f"๐Ÿ“‹ file_paths type: {type(file_paths)}") + self.logger.error(f"๐Ÿ“‹ file_paths value: {file_paths}") + self.logger.error(f"๐Ÿ“‹ file_paths length: {len(file_paths)}") + for file_path in file_paths: + result["files"][file_path] = { + "core_purpose": f"**Core Purpose**: {file_path} completed.", + "public_interface": "**Public Interface**: Public interface need manual review.", + "internal_dependencies": "**Internal Dependencies**: Internal dependencies need manual review.", + "external_dependencies": "**External Dependencies**: External dependencies need manual review.", + "implementation_notes": "**Implementation Notes**: Implementation notes need manual review.", + } + + return result + + def _format_code_implementation_summary( + self, file_path: str, llm_summary: str, files_implemented: int + ) -> str: + """ + Format the LLM-generated summary into the final structure + + Args: + file_path: Path of the implemented file + llm_summary: LLM-generated summary content + files_implemented: Number of files implemented so far + + Returns: + Formatted summary + """ + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + formatted_summary = f"""# Code Implementation Summary +**Generated**: {timestamp} +**File Implemented**: {file_path} + +{llm_summary} + +--- +*Auto-generated by Memory Agent* +""" + return formatted_summary + + def _create_fallback_multi_code_summary( + self, file_implementations: Dict[str, str], files_implemented: int + ) -> str: + """ + Create fallback multi-file summary when LLM is unavailable + + Args: + file_implementations: Dictionary mapping file_path to implementation_content + files_implemented: Number of files implemented so far + + Returns: + Fallback multi-file summary + """ + # Create fallback summaries for each file + fallback_summaries = [] + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + for file_path in file_implementations.keys(): + fallback_summary = f"""# Code Implementation Summary +**Generated**: {timestamp} +**File Implemented**: {file_path} +**Multi-file batch summary failed to generate.** + +--- +*Auto-generated by Concise Memory Agent (Multi-File Fallback Mode)* +""" + fallback_summaries.append(fallback_summary) + + return "\n".join(fallback_summaries) + + async def _save_code_summary_to_file(self, new_summary: str, file_path: str): + """ + Append code implementation summary to implement_code_summary.md + Accumulates all implementations with clear separators + + Args: + new_summary: New summary content to append + file_path: Path of the file for which the summary was generated + """ + try: + # Create directory if it doesn't exist + os.makedirs(os.path.dirname(self.code_summary_path), exist_ok=True) + + # Check if file exists to determine if we need header + file_exists = os.path.exists(self.code_summary_path) + + # Open in append mode to accumulate all implementations + with open(self.code_summary_path, "a", encoding="utf-8") as f: + if not file_exists: + # Write header for new file + f.write("# Code Implementation Progress Summary\n") + f.write("*Accumulated implementation progress for all files*\n\n") + + # Add clear separator between implementations + f.write("\n" + "=" * 80 + "\n") + f.write(f"## IMPLEMENTATION File {file_path}\n") + f.write("=" * 80 + "\n\n") + + # Write the new summary + f.write(new_summary) + f.write("\n\n") + + self.logger.info( + f"Appended LLM-based code implementation summary to: {self.code_summary_path}" + ) + + except Exception as e: + self.logger.error(f"Failed to save code implementation summary: {e}") + + async def _call_llm_for_summary( + self, client, client_type: str, summary_messages: List[Dict] + ) -> Dict[str, Any]: + """ + Call LLM for code implementation summary generation ONLY + + This method is used only for creating code implementation summaries, + NOT for conversation summarization which has been removed. + """ + if client_type == "anthropic": + response = await client.messages.create( + model=self.default_models["anthropic"], + system="You are an expert code implementation summarizer. Create structured summaries of implemented code files that preserve essential information about functions, dependencies, and implementation approaches.", + messages=summary_messages, + max_tokens=8000, # Increased for multi-file support + temperature=0.2, + ) + + content = "" + for block in response.content: + if block.type == "text": + content += block.text + + return {"content": content} + + elif client_type == "openai": + openai_messages = [ + { + "role": "system", + "content": "You are an expert code implementation summarizer. Create structured summaries of implemented code files that preserve essential information about functions, dependencies, and implementation approaches.", + } + ] + openai_messages.extend(summary_messages) + + # Try max_tokens and temperature first, fallback to max_completion_tokens without temperature if unsupported + try: + response = await client.chat.completions.create( + model=self.default_models["openai"], + messages=openai_messages, + max_tokens=8000, # Increased for multi-file support + temperature=0.2, + ) + except Exception as e: + if "max_tokens" in str(e) and "max_completion_tokens" in str(e): + # Retry with max_completion_tokens and no temperature for models that require it + response = await client.chat.completions.create( + model=self.default_models["openai"], + messages=openai_messages, + max_completion_tokens=8000, # Increased for multi-file support + ) + else: + raise + + return {"content": response.choices[0].message.content or ""} + + elif client_type == "google": + from google.genai import types + + # Convert messages to Gemini format + system_instruction = "You are an expert code implementation summarizer. Create structured summaries of implemented code files that preserve essential information about functions, dependencies, and implementation approaches." + + gemini_messages = [] + for msg in summary_messages: + role = msg.get("role", "user") + content = msg.get("content", "") + + # Convert role names: "assistant" -> "model" + if role == "assistant": + role = "model" + elif role not in ["user", "model"]: + role = "user" + + gemini_messages.append( + types.Content(role=role, parts=[types.Part.from_text(text=content)]) + ) + + config = types.GenerateContentConfig( + max_output_tokens=8000, # Increased for multi-file support + temperature=0.2, + system_instruction=system_instruction, + ) + + response = await client.aio.models.generate_content( + model=self.default_models.get("google", "gemini-2.0-flash"), + contents=gemini_messages, + config=config, + ) + + # Extract content from Gemini response + content = "" + if response and hasattr(response, "candidates") and response.candidates: + candidate = response.candidates[0] + if hasattr(candidate, "content") and candidate.content: + if hasattr(candidate.content, "parts") and candidate.content.parts: + for part in candidate.content.parts: + if hasattr(part, "text") and part.text: + content += part.text + + if not content: + self.logger.warning("Google response is empty or malformed") + + return {"content": content} + + else: + raise ValueError(f"Unsupported client type: {client_type}") + + def start_new_round(self, iteration: Optional[int] = None): + """Start a new dialogue round and reset tool results + + Args: + iteration: Optional iteration number from workflow to sync with current_round + """ + if iteration is not None: + # Sync with workflow iteration + self.current_round = iteration + else: + # Default behavior: increment round counter + self.current_round += 1 + self.logger.info(f"๐Ÿ”„ Started new round {self.current_round}") + + self.current_round_tool_results = [] # Clear previous round results + + def record_tool_result( + self, tool_name: str, tool_input: Dict[str, Any], tool_result: Any + ): + """ + Record tool result for current round and detect write_multiple_files calls + + Args: + tool_name: Name of the tool called + tool_input: Input parameters for the tool + tool_result: Result returned by the tool + """ + # Detect write_multiple_files calls to trigger memory clearing + if tool_name == "write_multiple_files": + self.last_write_multiple_files_detected = True + self.should_clear_memory_next = True + + # Only record specific tools that provide essential information + essential_tools = [ + "read_multiple_files", # Read multiple file contents + "write_multiple_files", # Write multiple file contents (important for tracking implementations) + "execute_python", # Execute Python code (for testing/validation) + "execute_bash", # Execute bash commands (for build/execution) + "search_code", # Search code patterns + "search_reference_code", # Search reference code (if available) + "get_file_structure", # Get file structure (for understanding project layout) + ] + + if tool_name in essential_tools: + tool_record = { + "tool_name": tool_name, + "tool_input": tool_input, + "tool_result": tool_result, + "timestamp": time.time(), + } + self.current_round_tool_results.append(tool_record) + + def should_use_concise_mode(self) -> bool: + """ + Check if concise memory mode should be used + + Returns: + True if first batch has been generated and concise mode should be active + """ + return self.last_write_multiple_files_detected + + def create_concise_messages_revise( + self, + system_prompt: str, + messages: List[Dict[str, Any]], + files_implemented: int, + task_description: str, + file_batch: List[str], + is_first_batch: bool = True, + implemented_files: List[str] = None, # Receive from workflow + all_files: List[str] = None, # NEW: Receive all files from workflow + ) -> List[Dict[str, Any]]: + """ + Create concise message list for LLM input specifically for revision execution + ALIGNED with _execute_multi_file_batch_revision in code_evaluation_workflow + + Args: + system_prompt: Current system prompt + messages: Original message list + files_implemented: Number of files implemented so far + task_description: Description of the current task + file_batch: Files to implement in this batch + is_first_batch: Whether this is the first batch (use file_batch) or subsequent + implemented_files: List of all implemented files (from workflow) + all_files: List of all files that should be implemented (from workflow) + + Returns: + Concise message list containing only essential information for revision + """ + # Use empty lists if not provided + if implemented_files is None: + implemented_files = [] + if all_files is None: + all_files = [] + + self.logger.info( + "๐ŸŽฏ Using CONCISE memory mode for revision - Clear slate after write_multiple_files" + ) + + concise_messages = [] + + # Format file lists using workflow data + implemented_files_list = ( + "\n".join([f"- {file}" for file in implemented_files]) + if implemented_files + else "- None yet" + ) + + # Calculate unimplemented files from workflow data + + # Read initial plan and memory content + initial_plan_content = self.initial_plan + memory_content = ( + self._read_code_knowledge_base() + or "No previous implementation memory available" + ) + + files_to_implement = file_batch + file_list = "\n".join([f"- {file_path}" for file_path in files_to_implement]) + + # Create revision-specific task message + task_message = f"""Task: {task_description} + + Files to implement in this batch ({len(files_to_implement)} files): + {file_list} + + MANDATORY JSON FORMAT REQUIREMENTS: + 1. Use write_multiple_files tool + 2. Parameter name: "file_implementations" + 3. Value must be a VALID JSON string with ESCAPED newlines + 4. Use \\n for newlines, \\t for tabs, \\" for quotes + 5. NO literal newlines in the JSON string + + CORRECT JSON FORMAT EXAMPLE: + {{ + "file1.py": "# Comment\\nclass MyClass:\\n def __init__(self):\\n pass\\n", + "file2.py": "import os\\n\\ndef main():\\n print('Hello')\\n" + }} + + Initial Implementation Plan Context: + {initial_plan_content} + + Previous Implementation Memory: + {memory_content} + + **All Previously Implemented Files:** + {implemented_files_list} + + **Current Status:** {files_implemented} files implemented + + IMPLEMENTATION REQUIREMENTS: + - Create functional code for each file + - Use proper Python syntax and imports + - Include docstrings and comments + - Follow the existing patterns from memory + + Files to implement: {files_to_implement} + + Call write_multiple_files NOW with PROPERLY ESCAPED JSON containing all {len(files_to_implement)} files.""" + + concise_messages.append({"role": "user", "content": task_message}) + + # Debug output for files to implement + print("โœ… Files to implement:") + for file_path in files_to_implement: + print(f"{file_path}") + + return concise_messages + + def _calculate_message_statistics( + self, messages: List[Dict[str, Any]], label: str + ) -> Dict[str, Any]: + """ + Calculate statistics for a message list + + Args: + messages: List of messages to analyze + label: Label for logging + + Returns: + Dictionary with statistics + """ + total_chars = 0 + total_words = 0 + + for msg in messages: + content = msg.get("content", "") + total_chars += len(content) + total_words += len(content.split()) + + # Estimate tokens (rough approximation: ~4 characters per token) + estimated_tokens = total_chars // 4 + + stats = { + "message_count": len(messages), + "total_characters": total_chars, + "total_words": total_words, + "estimated_tokens": estimated_tokens, + "summary": f"{len(messages)} msgs, {total_chars:,} chars, ~{estimated_tokens:,} tokens", + } + + return stats + + def _calculate_memory_savings( + self, original_stats: Dict[str, Any], optimized_stats: Dict[str, Any] + ) -> Dict[str, Any]: + """ + Calculate memory savings between original and optimized messages + + Args: + original_stats: Statistics for original messages + optimized_stats: Statistics for optimized messages + + Returns: + Dictionary with savings calculations + """ + messages_saved = ( + original_stats["message_count"] - optimized_stats["message_count"] + ) + chars_saved = ( + original_stats["total_characters"] - optimized_stats["total_characters"] + ) + tokens_saved_estimate = ( + original_stats["estimated_tokens"] - optimized_stats["estimated_tokens"] + ) + + # Calculate percentages (avoid division by zero) + messages_saved_percent = ( + messages_saved / max(original_stats["message_count"], 1) + ) * 100 + chars_saved_percent = ( + chars_saved / max(original_stats["total_characters"], 1) + ) * 100 + tokens_saved_percent = ( + tokens_saved_estimate / max(original_stats["estimated_tokens"], 1) + ) * 100 + + return { + "messages_saved": messages_saved, + "chars_saved": chars_saved, + "tokens_saved_estimate": tokens_saved_estimate, + "messages_saved_percent": messages_saved_percent, + "chars_saved_percent": chars_saved_percent, + "tokens_saved_percent": tokens_saved_percent, + } + + def _read_code_knowledge_base(self) -> Optional[str]: + """ + Read the implement_code_summary.md file as code knowledge base + Returns only the final/latest implementation entry, not all historical entries + + Returns: + Content of the latest implementation entry if it exists, None otherwise + """ + try: + if os.path.exists(self.code_summary_path): + with open(self.code_summary_path, "r", encoding="utf-8") as f: + content = f.read().strip() + return content + else: + return None + + except Exception as e: + self.logger.error(f"Failed to read code knowledge base: {e}") + return None + + def _extract_latest_implementation_entry(self, content: str) -> Optional[str]: + """ + Extract the latest/final implementation entry from the implement_code_summary.md content + Uses a simpler approach to find the last implementation section + + Args: + content: Full content of implement_code_summary.md + + Returns: + Latest implementation entry content, or None if not found + """ + try: + import re + + # Pattern to match the start of implementation sections + section_pattern = r"={80}\s*\n## IMPLEMENTATION File .+?" + + # Find all implementation section starts + matches = list(re.finditer(section_pattern, content)) + + if not matches: + # No implementation sections found + lines = content.split("\n") + fallback_content = ( + "\n".join(lines[:10]) + "\n... (truncated for brevity)" + if len(lines) > 10 + else content + ) + self.logger.info( + "๐Ÿ“– No implementation sections found, using fallback content" + ) + return fallback_content + + # Get the start position of the last implementation section + last_match = matches[-1] + start_pos = last_match.start() + + # Take everything from the last section start to the end of content + latest_entry = content[start_pos:].strip() + + return latest_entry + + except Exception as e: + self.logger.error(f"Failed to extract latest implementation entry: {e}") + # Return last 1000 characters as fallback + return content[-500:] if len(content) > 500 else content + + def _format_tool_results(self) -> str: + """ + Format current round tool results for LLM input + + Returns: + Formatted string of tool results + """ + if not self.current_round_tool_results: + return "No tool results in current round." + + formatted_results = [] + + for result in self.current_round_tool_results: + tool_name = result["tool_name"] + tool_input = result["tool_input"] + tool_result = result["tool_result"] + + # Format based on tool type + if tool_name == "read_multiple_files": + file_requests = tool_input.get("file_requests", "unknown") + formatted_results.append(f""" +**read_multiple_files Result for {file_requests}:** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "write_multiple_files": + formatted_results.append(f""" +**write_multiple_files Result for batch:** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "execute_python": + code_snippet = ( + tool_input.get("code", "")[:50] + "..." + if len(tool_input.get("code", "")) > 50 + else tool_input.get("code", "") + ) + formatted_results.append(f""" +**execute_python Result (code: {code_snippet}):** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "execute_bash": + command = tool_input.get("command", "unknown") + formatted_results.append(f""" +**execute_bash Result (command: {command}):** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "search_code": + pattern = tool_input.get("pattern", "unknown") + file_pattern = tool_input.get("file_pattern", "") + formatted_results.append(f""" +**search_code Result (pattern: {pattern}, files: {file_pattern}):** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "search_reference_code": + target_file = tool_input.get("target_file", "unknown") + keywords = tool_input.get("keywords", "") + formatted_results.append(f""" +**search_reference_code Result for {target_file} (keywords: {keywords}):** +{self._format_tool_result_content(tool_result)} +""") + elif tool_name == "get_file_structure": + directory = tool_input.get( + "directory_path", tool_input.get("path", "current") + ) + formatted_results.append(f""" +**get_file_structure Result for {directory}:** +{self._format_tool_result_content(tool_result)} +""") + + return "\n".join(formatted_results) + + def _format_tool_result_content(self, tool_result: Any) -> str: + """ + Format tool result content for display + + Args: + tool_result: Tool result to format + + Returns: + Formatted string representation + """ + if isinstance(tool_result, str): + # Try to parse as JSON for better formatting + try: + result_data = json.loads(tool_result) + if isinstance(result_data, dict): + # Format key information + if result_data.get("status") == "success": + return json.dumps(result_data, indent=2) + else: + return json.dumps(result_data, indent=2) + else: + return str(result_data) + except json.JSONDecodeError: + return tool_result + else: + return str(tool_result) + + def get_memory_statistics( + self, all_files: List[str] = None, implemented_files: List[str] = None + ) -> Dict[str, Any]: + """ + Get memory agent statistics for multi-file operations + + Args: + all_files: List of all files that should be implemented (from workflow) + implemented_files: List of all implemented files (from workflow) + """ + if all_files is None: + all_files = [] + if implemented_files is None: + implemented_files = [] + + # Calculate unimplemented files from workflow data + unimplemented_files = [f for f in all_files if f not in implemented_files] + + return { + "last_write_multiple_files_detected": self.last_write_multiple_files_detected, + "should_clear_memory_next": self.should_clear_memory_next, + "current_round": self.current_round, + "concise_mode_active": self.should_use_concise_mode(), + "current_round_tool_results": len(self.current_round_tool_results), + "essential_tools_recorded": [ + r["tool_name"] for r in self.current_round_tool_results + ], + # File tracking statistics (from workflow) + "total_files_in_plan": len(all_files), + "files_implemented_count": len(implemented_files), + "files_remaining_count": len(unimplemented_files), + "all_files_list": all_files.copy(), + "implemented_files_list": implemented_files.copy(), + "unimplemented_files_list": unimplemented_files, + "implementation_progress_percent": ( + len(implemented_files) / len(all_files) * 100 + ) + if all_files + else 0, + # Multi-file support statistics + "max_files_per_batch": self.max_files_per_batch, + "multi_file_support": True, + "single_file_support": False, # Explicitly disabled + } + + def record_multi_file_implementation(self, file_implementations: Dict[str, str]): + """ + Record multi-file implementation (for compatibility with workflow) + NOTE: This method doesn't track files internally - workflow manages file tracking + + Args: + file_implementations: Dictionary mapping file_path to implementation_content + """ + self.logger.info( + f"๐Ÿ“ Recorded multi-file implementation batch: {len(file_implementations)} files" + ) + # Note: We don't track files internally anymore - workflow handles this + + # ===== ENHANCED MEMORY SYNCHRONIZATION METHODS (Phase 4+) ===== + + async def synchronize_revised_file_memory( + self, + client, + client_type: str, + revised_file_path: str, + diff_content: str, + new_content: str, + revision_type: str = "targeted_fix", + ) -> str: + """ + Synchronize memory for a single revised file with diff information + + Args: + client: LLM client instance + client_type: Type of LLM client ("anthropic" or "openai") + revised_file_path: Path of the revised file + diff_content: Unified diff showing changes made + new_content: Complete new content of the file + revision_type: Type of revision ("targeted_fix", "comprehensive_revision", etc.) + + Returns: + Updated memory summary for the revised file + """ + try: + self.logger.info( + f"๐Ÿ”„ Synchronizing memory for revised file: {revised_file_path}" + ) + + # Create revision-specific summary prompt + revision_prompt = self._create_file_revision_summary_prompt( + revised_file_path, diff_content, new_content, revision_type + ) + + summary_messages = [{"role": "user", "content": revision_prompt}] + + # Get LLM-generated revision summary + llm_response = await self._call_llm_for_summary( + client, client_type, summary_messages + ) + llm_summary = llm_response.get("content", "") + + # Extract summary sections + revision_sections = self._extract_revision_summary_sections(llm_summary) + + # Format revision summary + formatted_summary = self._format_file_revision_summary( + revised_file_path, revision_sections, diff_content, revision_type + ) + + # Save the revision summary (replace old summary) + await self._save_revised_file_summary(formatted_summary, revised_file_path) + + self.logger.info( + f"โœ… Memory synchronized for revised file: {revised_file_path}" + ) + + return formatted_summary + + except Exception as e: + self.logger.error( + f"Failed to synchronize memory for revised file {revised_file_path}: {e}" + ) + + # Fallback to simple revision summary + return self._create_fallback_revision_summary( + revised_file_path, revision_type + ) + + async def synchronize_multiple_revised_files( + self, client, client_type: str, revision_results: List[Dict[str, Any]] + ) -> Dict[str, str]: + """ + Synchronize memory for multiple revised files based on revision results + + Args: + client: LLM client instance + client_type: Type of LLM client + revision_results: List of revision results with file paths, diffs, and new content + + Returns: + Dictionary mapping file paths to updated memory summaries + """ + try: + self.logger.info( + f"๐Ÿ”„ Synchronizing memory for {len(revision_results)} revised files" + ) + + synchronized_summaries = {} + + for revision_result in revision_results: + file_path = revision_result.get("file_path", "") + diff_content = revision_result.get("diff", "") + new_content = revision_result.get("new_content", "") + revision_type = revision_result.get("revision_type", "targeted_fix") + + if file_path and revision_result.get("success", False): + summary = await self.synchronize_revised_file_memory( + client, + client_type, + file_path, + diff_content, + new_content, + revision_type, + ) + synchronized_summaries[file_path] = summary + else: + self.logger.warning( + f"โš ๏ธ Skipping memory sync for failed revision: {file_path}" + ) + + self.logger.info( + f"โœ… Memory synchronized for {len(synchronized_summaries)} successfully revised files" + ) + + return synchronized_summaries + + except Exception as e: + self.logger.error( + f"Failed to synchronize memory for multiple revised files: {e}" + ) + return {} + + def _create_file_revision_summary_prompt( + self, file_path: str, diff_content: str, new_content: str, revision_type: str + ) -> str: + """ + Create prompt for LLM to generate file revision summary + + Args: + file_path: Path of the revised file + diff_content: Unified diff showing changes + new_content: Complete new content of the file + revision_type: Type of revision performed + + Returns: + Prompt for LLM revision summarization + """ + # Truncate content if too long for prompt + content_preview = ( + new_content[:2000] + "..." if len(new_content) > 2000 else new_content + ) + diff_preview = ( + diff_content[:1000] + "..." if len(diff_content) > 1000 else diff_content + ) + + prompt = f"""You are an expert code revision summarizer. A file has been REVISED with targeted changes. Create a structured summary of the revision. + +**File Revised**: {file_path} +**Revision Type**: {revision_type} + +**Changes Made (Diff):** +```diff +{diff_preview} +``` + +**Updated File Content:** +```python +{content_preview} +``` + +**Required Summary Format:** + +**Revision Summary**: +- Brief description of what was changed and why + +**Changes Made**: +- Specific modifications applied (line-level changes) +- Functions/classes affected +- New functionality added or bugs fixed + +**Impact Assessment**: +- How the changes affect the file's behavior +- Dependencies that might be affected +- Integration points that need attention + +**Quality Improvements**: +- Code quality enhancements made +- Error handling improvements +- Performance or maintainability gains + +**Post-Revision Status**: +- Current functionality of the file +- Key interfaces and exports +- Dependencies and imports + +**Instructions:** +- Focus on the CHANGES made, not just the final state +- Highlight the specific improvements and fixes applied +- Be concise but comprehensive about the revision impact +- Use the exact format specified above + +**Summary:**""" + + return prompt + + def _extract_revision_summary_sections(self, llm_summary: str) -> Dict[str, str]: + """ + Extract different sections from LLM-generated revision summary + + Args: + llm_summary: Raw LLM response containing revision summary + + Returns: + Dictionary with extracted sections + """ + sections = { + "revision_summary": "", + "changes_made": "", + "impact_assessment": "", + "quality_improvements": "", + "post_revision_status": "", + } + + try: + lines = llm_summary.split("\n") + current_section = None + current_content = [] + + for line in lines: + line_lower = line.lower().strip() + original_line = line.strip() + + # Skip empty lines + if not original_line: + if current_section: + current_content.append(line) + continue + + # Section detection + section_matched = False + + if "revision summary" in line_lower and "**" in original_line: + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + current_section = "revision_summary" + current_content = [] + section_matched = True + elif "changes made" in line_lower and "**" in original_line: + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + current_section = "changes_made" + current_content = [] + section_matched = True + elif "impact assessment" in line_lower and "**" in original_line: + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + current_section = "impact_assessment" + current_content = [] + section_matched = True + elif "quality improvements" in line_lower and "**" in original_line: + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + current_section = "quality_improvements" + current_content = [] + section_matched = True + elif "post-revision status" in line_lower and "**" in original_line: + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + current_section = "post_revision_status" + current_content = [] + section_matched = True + + # If no section header matched, add to current content + if not section_matched and current_section: + current_content.append(line) + + # Save the final section + if current_section and current_content: + sections[current_section] = "\n".join(current_content).strip() + + self.logger.info( + f"๐Ÿ“‹ Extracted {len([s for s in sections.values() if s])} revision summary sections" + ) + + except Exception as e: + self.logger.error(f"Failed to extract revision summary sections: {e}") + # Provide fallback content + sections["revision_summary"] = "File revision completed" + sections["changes_made"] = ( + "Targeted changes applied based on error analysis" + ) + sections["impact_assessment"] = ( + "Changes should improve code functionality and reduce errors" + ) + sections["quality_improvements"] = ( + "Code quality enhanced through targeted fixes" + ) + sections["post_revision_status"] = "File functionality updated and improved" + + return sections + + def _format_file_revision_summary( + self, + file_path: str, + revision_sections: Dict[str, str], + diff_content: str, + revision_type: str, + ) -> str: + """ + Format the revision summary into the final structure + + Args: + file_path: Path of the revised file + revision_sections: Extracted sections from LLM summary + diff_content: Unified diff content + revision_type: Type of revision performed + + Returns: + Formatted revision summary + """ + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + # Format sections with fallbacks + revision_summary = revision_sections.get( + "revision_summary", "File revision completed" + ) + changes_made = revision_sections.get("changes_made", "Targeted changes applied") + impact_assessment = revision_sections.get( + "impact_assessment", "Changes should improve functionality" + ) + quality_improvements = revision_sections.get( + "quality_improvements", "Code quality enhanced" + ) + post_revision_status = revision_sections.get( + "post_revision_status", "File updated successfully" + ) + + formatted_summary = f"""# File Revision Summary (UPDATED) +**Generated**: {timestamp} +**File Revised**: {file_path} +**Revision Type**: {revision_type} + +## Revision Summary +{revision_summary} + +## Changes Made +{changes_made} + +## Impact Assessment +{impact_assessment} + +## Quality Improvements +{quality_improvements} + +## Post-Revision Status +{post_revision_status} + +## Technical Details +**Diff Applied:** +```diff +{diff_content[:500]}{"..." if len(diff_content) > 500 else ""} +``` + +--- +*Auto-generated by Enhanced Memory Agent (Revision Mode)* +""" + return formatted_summary + + def _create_fallback_revision_summary( + self, file_path: str, revision_type: str + ) -> str: + """ + Create fallback revision summary when LLM is unavailable + + Args: + file_path: Path of the revised file + revision_type: Type of revision performed + + Returns: + Fallback revision summary + """ + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + fallback_summary = f"""# File Revision Summary (UPDATED) +**Generated**: {timestamp} +**File Revised**: {file_path} +**Revision Type**: {revision_type} + +## Revision Summary +File has been revised with targeted changes. LLM summary generation failed. + +## Changes Made +- Targeted modifications applied based on error analysis +- Specific line-level changes implemented +- Code functionality updated + +## Impact Assessment +- File behavior should be improved +- Error conditions addressed +- Integration points maintained + +## Quality Improvements +- Code quality enhanced through precise fixes +- Error handling improved +- Maintainability increased + +## Post-Revision Status +- File successfully updated +- Functionality preserved and enhanced +- Ready for integration testing + +--- +*Auto-generated by Enhanced Memory Agent (Revision Fallback Mode)* +""" + return fallback_summary + + async def _save_revised_file_summary(self, revision_summary: str, file_path: str): + """ + Save or update the revision summary for a file (replaces old summary) + + Args: + revision_summary: New revision summary content + file_path: Path of the file for which the summary was generated + """ + try: + # For revised files, we replace the existing summary rather than append + # Read existing content to find and replace the specific file's summary + file_exists = os.path.exists(self.code_summary_path) + + if file_exists: + with open(self.code_summary_path, "r", encoding="utf-8") as f: + existing_content = f.read() + + # Look for existing summary for this file and replace it + import re + + # Pattern to match existing implementation section for this file + file_pattern = re.escape(file_path) + section_pattern = rf"={80}\s*\n## IMPLEMENTATION File {file_pattern}\n={80}.*?(?=\n={80}|\Z)" + + # Check if this file already has a summary + if re.search(section_pattern, existing_content, re.DOTALL): + # Replace existing summary + new_section = f"\n{'=' * 80}\n## IMPLEMENTATION File {file_path} (REVISED)\n{'=' * 80}\n\n{revision_summary}\n\n" + updated_content = re.sub( + section_pattern, + new_section.strip(), + existing_content, + flags=re.DOTALL, + ) + + with open(self.code_summary_path, "w", encoding="utf-8") as f: + f.write(updated_content) + + self.logger.info( + f"Updated existing summary for revised file: {file_path}" + ) + else: + # Append new summary for this file + with open(self.code_summary_path, "a", encoding="utf-8") as f: + f.write("\n" + "=" * 80 + "\n") + f.write(f"## IMPLEMENTATION File {file_path} (REVISED)\n") + f.write("=" * 80 + "\n\n") + f.write(revision_summary) + f.write("\n\n") + + self.logger.info( + f"Appended new summary for revised file: {file_path}" + ) + else: + # Create new file with header + os.makedirs(os.path.dirname(self.code_summary_path), exist_ok=True) + + with open(self.code_summary_path, "w", encoding="utf-8") as f: + f.write("# Code Implementation Progress Summary\n") + f.write("*Accumulated implementation progress for all files*\n\n") + f.write("\n" + "=" * 80 + "\n") + f.write(f"## IMPLEMENTATION File {file_path} (REVISED)\n") + f.write("=" * 80 + "\n\n") + f.write(revision_summary) + f.write("\n\n") + + self.logger.info( + f"Created new summary file with revised file: {file_path}" + ) + + except Exception as e: + self.logger.error( + f"Failed to save revised file summary for {file_path}: {e}" + ) + + def get_revision_memory_statistics( + self, revised_files: List[str] + ) -> Dict[str, Any]: + """ + Get memory statistics for revised files + + Args: + revised_files: List of file paths that have been revised + + Returns: + Dictionary with revision memory statistics + """ + try: + total_revisions = len(revised_files) + + # Count how many files have updated summaries + summaries_updated = 0 + if os.path.exists(self.code_summary_path): + with open(self.code_summary_path, "r", encoding="utf-8") as f: + content = f.read() + + for file_path in revised_files: + if f"File {file_path} (REVISED)" in content: + summaries_updated += 1 + + return { + "total_revised_files": total_revisions, + "summaries_updated": summaries_updated, + "memory_sync_rate": (summaries_updated / total_revisions * 100) + if total_revisions > 0 + else 0, + "revised_files_list": revised_files.copy(), + "memory_summary_path": self.code_summary_path, + "revision_memory_mode": "active", + } + + except Exception as e: + self.logger.error(f"Failed to get revision memory statistics: {e}") + return { + "total_revised_files": len(revised_files), + "summaries_updated": 0, + "memory_sync_rate": 0, + "revised_files_list": revised_files.copy(), + "memory_summary_path": self.code_summary_path, + "revision_memory_mode": "error", + } diff --git a/DeepCode/workflows/agents/requirement_analysis_agent.py b/DeepCode/workflows/agents/requirement_analysis_agent.py new file mode 100644 index 00000000..79bab67c --- /dev/null +++ b/DeepCode/workflows/agents/requirement_analysis_agent.py @@ -0,0 +1,410 @@ +""" +User Requirement Analysis Agent + +Responsible for analyzing user initial requirements, generating guiding questions, +and summarizing detailed requirement documents based on user responses. +This Agent seamlessly integrates with existing chat workflows to provide more precise requirement understanding. +""" + +import json +import logging +from typing import Dict, List, Optional + +from mcp_agent.agents.agent import Agent +from utils.llm_utils import get_preferred_llm_class + + +class RequirementAnalysisAgent: + """ + User Requirement Analysis Agent + + Core Functions: + 1. Generate 5-8 guiding questions based on user initial requirements + 2. Collect user responses and analyze requirement completeness + 3. Generate detailed requirement documents for subsequent workflows + 4. Support skipping questions to directly enter implementation process + + Design Philosophy:รŸ + - Intelligent question generation covering functionality, technology, performance, UI, deployment dimensions + - Flexible user interaction supporting partial answers or complete skipping + - Structured requirement output for easy understanding by code generation agents + """ + + def __init__(self, logger: Optional[logging.Logger] = None): + """ + Initialize requirement analysis agent + Args: + logger: Logger instance + """ + self.logger = logger or self._create_default_logger() + self.mcp_agent = None + self.llm = None + + def _create_default_logger(self) -> logging.Logger: + """Create default logger""" + logger = logging.getLogger(f"{__name__}.RequirementAnalysisAgent") + logger.setLevel(logging.INFO) + return logger + + async def __aenter__(self): + """Async context manager entry""" + await self.initialize() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Async context manager exit""" + await self.cleanup() + + async def initialize(self): + """Initialize MCP Agent connection and LLM""" + try: + self.mcp_agent = Agent( + name="RequirementAnalysisAgent", + instruction="""You are a professional requirement analysis expert, skilled at guiding users to provide more detailed project requirements through precise questions. + +Your core capabilities: +1. **Intelligent Question Generation**: Based on user initial descriptions, generate 5-8 key questions covering functional requirements, technology selection, performance requirements, user interface, deployment environment, etc. +2. **Requirement Understanding Analysis**: Deep analysis of user's real intentions and implicit requirements +3. **Structured Requirement Output**: Integrate scattered requirement information into clear technical specification documents + +Question Generation Principles: +- Questions should be specific and clear, avoiding overly broad scope +- Cover key decision points for technical implementation +- Consider project feasibility and complexity +- Help users think about important details they might have missed + +Requirement Summary Principles: +- Maintain user's original intent unchanged +- Supplement key information for technical implementation +- Provide clear functional module division +- Give reasonable technical architecture suggestions""", + server_names=[], # No MCP servers needed, only use LLM + ) + + # Initialize agent context + await self.mcp_agent.__aenter__() + + # Attach LLM + self.llm = await self.mcp_agent.attach_llm(get_preferred_llm_class()) + + self.logger.info("RequirementAnalysisAgent initialized successfully") + + except Exception as e: + self.logger.error(f"RequirementAnalysisAgent initialization failed: {e}") + raise + + async def cleanup(self): + """Clean up resources""" + if self.mcp_agent: + try: + await self.mcp_agent.__aexit__(None, None, None) + except Exception as e: + self.logger.warning(f"Error during resource cleanup: {e}") + + async def generate_guiding_questions(self, user_input: str) -> List[Dict[str, str]]: + """ + Generate guiding questions based on user initial requirements + + Args: + user_input: User's initial requirement description + + Returns: + List[Dict]: Question list, each question contains category, question, importance and other fields + """ + try: + self.logger.info("Starting to generate AI precise guiding questions") + + # Build more precise prompt + prompt = f"""Based on user's project requirements, generate precise guiding questions to help refine requirements. + +User Requirements: {user_input} + +Please analyze user requirements and generate 1-3 most critical targeted questions focusing on the most important aspects for this specific project + +Return format (pure JSON array, no other text): +[ + {{ + "category": "Functional Requirements", + "question": "Specific question content", + "importance": "High", + "hint": "Question hint" + }} +] + +Requirements: Questions should be specific and practical, avoiding general discussions.""" + + from mcp_agent.workflows.llm.augmented_llm import RequestParams + + params = RequestParams( + max_tokens=3000, + temperature=0.5, # Lower temperature for more stable JSON output + ) + + self.logger.info( + f"Calling LLM to generate precise questions, input length: {len(user_input)}" + ) + + result = await self.llm.generate_str(message=prompt, request_params=params) + + self.logger.info( + f"LLM returned result length: {len(result) if result else 0}" + ) + + if not result or not result.strip(): + self.logger.error("LLM returned empty result") + raise ValueError("LLM returned empty result") + + self.logger.info(f"LLM returned result: {result[:500]}...") + + # Clean result and extract JSON part + result_cleaned = result.strip() + + # Try to find JSON array + import re + + json_pattern = r"\[\s*\{.*?\}\s*\]" + json_match = re.search(json_pattern, result_cleaned, re.DOTALL) + + if json_match: + json_str = json_match.group() + self.logger.info(f"Extracted JSON: {json_str[:200]}...") + else: + # If complete JSON not found, try direct parsing + json_str = result_cleaned + + # Parse JSON result + try: + questions = json.loads(json_str) + if isinstance(questions, list) and len(questions) > 0: + self.logger.info( + f"โœ… Successfully generated {len(questions)} AI precise guiding questions" + ) + return questions + else: + raise ValueError("Returned result is not a valid question list") + + except json.JSONDecodeError as e: + self.logger.error(f"JSON parsing failed: {e}") + self.logger.error(f"Original result: {result}") + + # Try more lenient JSON extraction + lines = result.split("\n") + json_lines = [] + in_json = False + + for line in lines: + if "[" in line: + in_json = True + if in_json: + json_lines.append(line) + if "]" in line and in_json: + break + + if json_lines: + try: + json_attempt = "\n".join(json_lines) + questions = json.loads(json_attempt) + if isinstance(questions, list) and len(questions) > 0: + self.logger.info( + f"โœ… Generated {len(questions)} questions through lenient parsing" + ) + return questions + except Exception: + pass + + # If JSON parsing fails, raise an error + self.logger.error("JSON parsing completely failed") + raise ValueError("Failed to parse AI generated questions") + + except Exception as e: + self.logger.error(f"Failed to generate guiding questions: {e}") + # Re-raise the exception instead of falling back to default questions + raise + + async def summarize_detailed_requirements( + self, initial_input: str, answers: Dict[str, str] + ) -> str: + """ + Generate detailed requirement document based on initial input and user answers + + Args: + initial_input: User's initial requirement description + answers: User's answer dictionary {question_id: answer} + + Returns: + str: Detailed requirement document + """ + try: + self.logger.info("Starting to generate AI detailed requirement summary") + + # Build answer content + answers_text = "" + if answers: + for question_id, answer in answers.items(): + if answer and answer.strip(): + answers_text += f"โ€ข {answer}\n" + + if not answers_text: + answers_text = "User chose to skip questions, generating based on initial requirements" + + prompt = f"""Based on user requirements and responses, generate a concise project requirement document. + +Initial Requirements: {initial_input} + +Additional Information: +{answers_text} + +Please generate a focused requirement document including: + +## Project Overview +Brief description of project's core goals and value proposition + +## Functional Requirements +Detailed list of required features and functional modules: +- Core functionalities +- User interactions and workflows +- Data processing requirements +- Integration needs + +## Technical Architecture +Recommended technical design including: +- Technology stack and frameworks +- System architecture design +- Database and data storage solutions +- API design considerations +- Security requirements + +## Performance & Scalability +- Expected user scale and performance requirements +- Scalability considerations and constraints + +Requirements: Focus on what needs to be built and how to build it technically. Be concise but comprehensive - avoid unnecessary implementation details.""" + + from mcp_agent.workflows.llm.augmented_llm import RequestParams + + params = RequestParams(max_tokens=4000, temperature=0.3) + + self.logger.info( + f"Calling LLM to generate requirement summary, initial requirement length: {len(initial_input)}" + ) + + result = await self.llm.generate_str(message=prompt, request_params=params) + + if not result or not result.strip(): + self.logger.error("LLM returned empty requirement summary") + raise ValueError("LLM returned empty requirement summary") + + self.logger.info( + f"โœ… Requirement summary generation completed, length: {len(result)}" + ) + return result.strip() + + except Exception as e: + self.logger.error(f"Requirement summary failed: {e}") + # Return basic requirement document + return f"""## Project Overview +Based on user requirements: {initial_input} + +## Functional Requirements +Core functionality needed: {initial_input} + +## Technical Architecture +- Select appropriate technology stack based on project requirements +- Adopt modular architecture design +- Consider database and data storage solutions +- Implement necessary security measures + +## Performance & Scalability +- Design for expected user scale +- Consider scalability and performance requirements + +Note: Due to technical issues, this is a simplified requirement document. Manual supplementation of detailed information is recommended.""" + + async def modify_requirements( + self, current_requirements: str, modification_feedback: str + ) -> str: + """ + Modify existing requirement document based on user feedback + + Args: + current_requirements: Current requirement document content + modification_feedback: User's modification requests and feedback + + Returns: + str: Modified requirement document + """ + try: + self.logger.info("Starting to modify requirements based on user feedback") + + # Build modification prompt + prompt = f"""Based on the current requirement document and user's modification requests, generate an updated requirement document. + +Current Requirements Document: +{current_requirements} + +User's Modification Requests: +{modification_feedback} + +CRITICAL REQUIREMENT: You MUST generate a complete, well-structured requirement document regardless of how complete or incomplete the user's modification requests are. Even if the user only provides minimal or unclear feedback, you must still produce a comprehensive requirement document following the exact format below. + +Generate an updated requirement document that incorporates any reasonable interpretation of the user's requested changes while maintaining the EXACT structure and format: + +## Project Overview +Brief description of project's core goals and value proposition + +## Functional Requirements +Detailed list of required features and functional modules: +- Core functionalities +- User interactions and workflows +- Data processing requirements +- Integration needs + +## Technical Architecture +Recommended technical design including: +- Technology stack and frameworks +- System architecture design +- Database and data storage solutions +- API design considerations +- Security requirements + +## Performance & Scalability +- Expected user scale and performance requirements +- Scalability considerations and constraints + +MANDATORY REQUIREMENTS: +1. ALWAYS return a complete document with ALL sections above, regardless of user input completeness +2. If user feedback is unclear or incomplete, make reasonable assumptions based on the current requirements +3. Incorporate any clear user requests while filling in missing details intelligently +4. Maintain consistency and coherence throughout the document +5. Ensure all technical suggestions are feasible and practical +6. NEVER return an incomplete or partial document - always provide full sections +7. Keep the same professional structure and format in all cases""" + + from mcp_agent.workflows.llm.augmented_llm import RequestParams + + params = RequestParams(max_tokens=4000, temperature=0.3) + + self.logger.info( + f"Calling LLM to modify requirements, feedback length: {len(modification_feedback)}" + ) + + result = await self.llm.generate_str(message=prompt, request_params=params) + + if not result or not result.strip(): + self.logger.error("LLM returned empty modified requirements") + raise ValueError("LLM returned empty modified requirements") + + self.logger.info( + f"โœ… Requirements modification completed, length: {len(result)}" + ) + return result.strip() + + except Exception as e: + self.logger.error(f"Requirements modification failed: {e}") + # Return current requirements with a note about the modification attempt + return f"""{current_requirements} + +--- +**Note:** Automatic modification failed due to technical issues. The original requirements are shown above. Please manually incorporate the following requested changes: + +{modification_feedback}""" diff --git a/DeepCode/workflows/code_implementation_workflow.py b/DeepCode/workflows/code_implementation_workflow.py new file mode 100644 index 00000000..c2a7b29f --- /dev/null +++ b/DeepCode/workflows/code_implementation_workflow.py @@ -0,0 +1,1479 @@ +""" +Paper Code Implementation Workflow - MCP-compliant Iterative Development + +Features: +1. File Tree Creation +2. Code Implementation - Based on aisi-basic-agent iterative development + +MCP Architecture: +- MCP Server: tools/code_implementation_server.py +- MCP Client: Called through mcp_agent framework +- Configuration: mcp_agent.config.yaml +""" + +import asyncio +import json +import logging +import os +import sys +import time +import yaml +from pathlib import Path +from typing import Dict, Any, Optional, List + +# MCP Agent imports +from mcp_agent.agents.agent import Agent + +# Local imports +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from prompts.code_prompts import STRUCTURE_GENERATOR_PROMPT +from prompts.code_prompts import ( + GENERAL_CODE_IMPLEMENTATION_SYSTEM_PROMPT, +) +from workflows.agents import CodeImplementationAgent +from workflows.agents.memory_agent_concise import ConciseMemoryAgent +from config.mcp_tool_definitions import get_mcp_tools +from utils.llm_utils import get_preferred_llm_class, get_default_models +# DialogueLogger removed - no longer needed + + +class CodeImplementationWorkflow: + """ + Paper Code Implementation Workflow Manager + + Uses standard MCP architecture: + 1. Connect to code-implementation server via MCP client + 2. Use MCP protocol for tool calls + 3. Support workspace management and operation history tracking + """ + + # ==================== 1. Class Initialization and Configuration (Infrastructure Layer) ==================== + + def __init__(self, config_path: str = "mcp_agent.secrets.yaml"): + """Initialize workflow with configuration""" + self.config_path = config_path + self.api_config = self._load_api_config() + self.default_models = get_default_models("mcp_agent.config.yaml") + self.logger = self._create_logger() + self.mcp_agent = None + self.enable_read_tools = ( + True # Default value, will be overridden by run_workflow parameter + ) + + def _load_api_config(self) -> Dict[str, Any]: + """Load API configuration from YAML file""" + try: + with open(self.config_path, "r", encoding="utf-8") as f: + return yaml.safe_load(f) + except Exception as e: + raise Exception(f"Failed to load API config: {e}") + + def _create_logger(self) -> logging.Logger: + """Create and configure logger""" + logger = logging.getLogger(__name__) + # Don't add handlers to child loggers - let them propagate to root + logger.setLevel(logging.INFO) + return logger + + def _read_plan_file(self, plan_file_path: str) -> str: + """Read implementation plan file""" + plan_path = Path(plan_file_path) + if not plan_path.exists(): + raise FileNotFoundError( + f"Implementation plan file not found: {plan_file_path}" + ) + + with open(plan_path, "r", encoding="utf-8") as f: + return f.read() + + def _check_file_tree_exists(self, target_directory: str) -> bool: + """Check if file tree structure already exists""" + code_directory = os.path.join(target_directory, "generate_code") + return os.path.exists(code_directory) and len(os.listdir(code_directory)) > 0 + + # ==================== 2. Public Interface Methods (External API Layer) ==================== + + async def run_workflow( + self, + plan_file_path: str, + target_directory: Optional[str] = None, + pure_code_mode: bool = False, + enable_read_tools: bool = True, + ): + """Run complete workflow - Main public interface""" + # Set the read tools configuration + self.enable_read_tools = enable_read_tools + + try: + plan_content = self._read_plan_file(plan_file_path) + + if target_directory is None: + target_directory = str(Path(plan_file_path).parent) + + # Calculate code directory for workspace alignment + code_directory = os.path.join(target_directory, "generate_code") + + self.logger.info("=" * 80) + self.logger.info("๐Ÿš€ STARTING CODE IMPLEMENTATION WORKFLOW") + self.logger.info("=" * 80) + self.logger.info(f"๐Ÿ“„ Plan file: {plan_file_path}") + self.logger.info(f"๐Ÿ“‚ Plan file parent: {target_directory}") + self.logger.info(f"๐ŸŽฏ Code directory (MCP workspace): {code_directory}") + self.logger.info( + f"โš™๏ธ Read tools: {'ENABLED' if self.enable_read_tools else 'DISABLED'}" + ) + self.logger.info("=" * 80) + + results = {} + + # Check if file tree exists + if self._check_file_tree_exists(target_directory): + self.logger.info("File tree exists, skipping creation") + results["file_tree"] = "Already exists, skipped creation" + else: + self.logger.info("Creating file tree...") + results["file_tree"] = await self.create_file_structure( + plan_content, target_directory + ) + + # Code implementation + if pure_code_mode: + self.logger.info("Starting pure code implementation...") + results["code_implementation"] = await self.implement_code_pure( + plan_content, target_directory, code_directory + ) + else: + pass + + self.logger.info("Workflow execution successful") + + return { + "status": "success", + "plan_file": plan_file_path, + "target_directory": target_directory, + "code_directory": os.path.join(target_directory, "generate_code"), + "results": results, + "mcp_architecture": "standard", + } + + except Exception as e: + self.logger.error(f"Workflow execution failed: {e}") + + return {"status": "error", "message": str(e), "plan_file": plan_file_path} + finally: + await self._cleanup_mcp_agent() + + async def create_file_structure( + self, plan_content: str, target_directory: str + ) -> str: + """Create file tree structure based on implementation plan""" + self.logger.info("Starting file tree creation...") + + structure_agent = Agent( + name="StructureGeneratorAgent", + instruction=STRUCTURE_GENERATOR_PROMPT, + server_names=["command-executor"], + ) + + async with structure_agent: + creator = await structure_agent.attach_llm( + get_preferred_llm_class(self.config_path) + ) + + message = f"""Analyze the following implementation plan and generate shell commands to create the file tree structure. + +Target Directory: {target_directory}/generate_code/ + +Implementation Plan: +{plan_content} + +Tasks: +1. Find the file tree structure in the implementation plan +2. Generate shell commands (mkdir -p, touch) to create that structure +3. Use the execute_commands tool to run the commands and create the file structure + +Requirements: +- Use mkdir -p to create directories +- Use touch to create files +- Include __init__.py file for Python packages +- Use relative paths to the target directory +- Execute commands to actually create the file structure""" + + result = await creator.generate_str(message=message) + self.logger.info(f"LLM response: {result[:200]}...") # Log first 200 chars + + # Verify directory was created, if not create it manually + code_dir = os.path.join(target_directory, "generate_code") + if not os.path.exists(code_dir): + self.logger.warning( + "LLM did not create directory, creating manually..." + ) + os.makedirs(code_dir, exist_ok=True) + self.logger.info(f"โœ… Manually created directory: {code_dir}") + else: + self.logger.info(f"โœ… Directory exists: {code_dir}") + + return result + + async def implement_code_pure( + self, plan_content: str, target_directory: str, code_directory: str = None + ) -> str: + """Pure code implementation - focus on code writing without testing""" + self.logger.info("Starting pure code implementation (no testing)...") + + # Use provided code_directory or calculate it (for backwards compatibility) + if code_directory is None: + code_directory = os.path.join(target_directory, "generate_code") + + self.logger.info(f"๐ŸŽฏ Using code directory (MCP workspace): {code_directory}") + + if not os.path.exists(code_directory): + self.logger.warning( + f"Code directory does not exist, creating it: {code_directory}" + ) + os.makedirs(code_directory, exist_ok=True) + self.logger.info(f"โœ… Code directory created: {code_directory}") + + try: + client, client_type = await self._initialize_llm_client() + await self._initialize_mcp_agent(code_directory) + + tools = self._prepare_mcp_tool_definitions() + system_message = GENERAL_CODE_IMPLEMENTATION_SYSTEM_PROMPT + messages = [] + + # implementation_message = f"""**TASK: Implement Research Paper Reproduction Code** + + # You are implementing a complete, working codebase that reproduces the core algorithms, experiments, and methods described in a research paper. Your goal is to create functional code that can replicate the paper's key results and contributions. + + # **What you need to do:** + # - Analyze the paper content and reproduction plan to understand requirements + # - Implement all core algorithms mentioned in the main body of the paper + # - Create the necessary components following the planned architecture + # - Test each component to ensure functionality + # - Integrate components into a cohesive, executable system + # - Focus on reproducing main contributions rather than appendix-only experiments + + # **RESOURCES:** + # - **Paper & Reproduction Plan**: `{target_directory}/` (contains .md paper files and initial_plan.txt with detailed implementation guidance) + # - **Reference Code Indexes**: `{target_directory}/indexes/` (JSON files with implementation patterns from related codebases) + # - **Implementation Directory**: `{code_directory}/` (your working directory for all code files) + + # **CURRENT OBJECTIVE:** + # Start by reading the reproduction plan (`{target_directory}/initial_plan.txt`) to understand the implementation strategy, then examine the paper content to identify the first priority component to implement. Use the search_code tool to find relevant reference implementations from the indexes directory (`{target_directory}/indexes/*.json`) before coding. + + # --- + # **START:** Review the plan above and begin implementation.""" + implementation_message = f"""**Task: Implement code based on the following reproduction plan** + +**Code Reproduction Plan:** +{plan_content} + +**Working Directory:** {code_directory} + +**Current Objective:** Begin implementation by analyzing the plan structure, examining the current project layout, and implementing the first foundation file according to the plan's priority order.""" + + messages.append({"role": "user", "content": implementation_message}) + + result = await self._pure_code_implementation_loop( + client, + client_type, + system_message, + messages, + tools, + plan_content, + target_directory, + ) + + return result + + finally: + await self._cleanup_mcp_agent() + + # ==================== 3. Core Business Logic (Implementation Layer) ==================== + + async def _pure_code_implementation_loop( + self, + client, + client_type, + system_message, + messages, + tools, + plan_content, + target_directory, + ): + """Pure code implementation loop with memory optimization and phase consistency""" + max_iterations = 800 + iteration = 0 + start_time = time.time() + max_time = 7200 # 120 minutes (2 hours) + + # Initialize specialized agents + code_agent = CodeImplementationAgent( + self.mcp_agent, self.logger, self.enable_read_tools + ) + + # Pass code_directory to memory agent for file extraction + code_directory = os.path.join(target_directory, "generate_code") + memory_agent = ConciseMemoryAgent( + plan_content, + self.logger, + target_directory, + self.default_models, + code_directory, + ) + + # Log read tools configuration + read_tools_status = "ENABLED" if self.enable_read_tools else "DISABLED" + self.logger.info( + f"๐Ÿ”ง Read tools (read_file, read_code_mem): {read_tools_status}" + ) + if not self.enable_read_tools: + self.logger.info( + "๐Ÿšซ No read mode: read_file and read_code_mem tools will be skipped" + ) + + # Connect code agent with memory agent for summary generation + # Note: Concise memory agent doesn't need LLM client for summary generation + code_agent.set_memory_agent(memory_agent, client, client_type) + + # Initialize memory agent with iteration 0 + memory_agent.start_new_round(iteration=0) + + while iteration < max_iterations: + iteration += 1 + elapsed_time = time.time() - start_time + + if elapsed_time > max_time: + self.logger.warning(f"Time limit reached: {elapsed_time:.2f}s") + break + + # # Test simplified memory approach if we have files implemented + # if iteration == 5 and code_agent.get_files_implemented_count() > 0: + # self.logger.info("๐Ÿงช Testing simplified memory approach...") + # test_results = await memory_agent.test_simplified_memory_approach() + # self.logger.info(f"Memory test results: {test_results}") + + # self.logger.info(f"Pure code implementation iteration {iteration}: generating code") + + messages = self._validate_messages(messages) + current_system_message = code_agent.get_system_prompt() + + # Round logging removed + + # Call LLM + response = await self._call_llm_with_tools( + client, client_type, current_system_message, messages, tools + ) + + response_content = response.get("content", "").strip() + if not response_content: + response_content = "Continue implementing code files..." + + messages.append({"role": "assistant", "content": response_content}) + + # Handle tool calls + if response.get("tool_calls"): + tool_results = await code_agent.execute_tool_calls( + response["tool_calls"] + ) + + # Record essential tool results in concise memory agent + for tool_call, tool_result in zip(response["tool_calls"], tool_results): + memory_agent.record_tool_result( + tool_name=tool_call["name"], + tool_input=tool_call["input"], + tool_result=tool_result.get("result"), + ) + + # NEW LOGIC: Check if write_file was called and trigger memory optimization immediately + + # Determine guidance based on results + has_error = self._check_tool_results_for_errors(tool_results) + files_count = code_agent.get_files_implemented_count() + + if has_error: + guidance = self._generate_error_guidance() + else: + guidance = self._generate_success_guidance(files_count) + + compiled_response = self._compile_user_response(tool_results, guidance) + messages.append({"role": "user", "content": compiled_response}) + + # NEW LOGIC: Apply memory optimization immediately after write_file detection + if memory_agent.should_trigger_memory_optimization( + messages, code_agent.get_files_implemented_count() + ): + # Memory optimization triggered + + # Apply concise memory optimization + files_implemented_count = code_agent.get_files_implemented_count() + current_system_message = code_agent.get_system_prompt() + messages = memory_agent.apply_memory_optimization( + current_system_message, messages, files_implemented_count + ) + + # Memory optimization completed + + else: + files_count = code_agent.get_files_implemented_count() + no_tools_guidance = self._generate_no_tools_guidance(files_count) + messages.append({"role": "user", "content": no_tools_guidance}) + + # # Check for analysis loop and provide corrective guidance + # if code_agent.is_in_analysis_loop(): + # analysis_loop_guidance = code_agent.get_analysis_loop_guidance() + # messages.append({"role": "user", "content": analysis_loop_guidance}) + # self.logger.warning( + # "Analysis loop detected and corrective guidance provided" + # ) + + # Record file implementations in memory agent (for the current round) + for file_info in code_agent.get_implementation_summary()["completed_files"]: + memory_agent.record_file_implementation(file_info["file"]) + + # REMOVED: Old memory optimization logic - now happens immediately after write_file + # Memory optimization is now triggered immediately after write_file detection + + # Start new round for next iteration, sync with workflow iteration + memory_agent.start_new_round(iteration=iteration) + + # Check completion based on actual unimplemented files list + unimplemented_files = memory_agent.get_unimplemented_files() + if not unimplemented_files: # Empty list means all files implemented + self.logger.info( + "โœ… Code implementation complete - All files implemented" + ) + break + + # Emergency trim if too long + if len(messages) > 50: + self.logger.warning( + "Emergency message trim - applying concise memory optimization" + ) + + current_system_message = code_agent.get_system_prompt() + files_implemented_count = code_agent.get_files_implemented_count() + messages = memory_agent.apply_memory_optimization( + current_system_message, messages, files_implemented_count + ) + + return await self._generate_pure_code_final_report_with_concise_agents( + iteration, time.time() - start_time, code_agent, memory_agent + ) + + # ==================== 4. MCP Agent and LLM Communication Management (Communication Layer) ==================== + + async def _initialize_mcp_agent(self, code_directory: str): + """Initialize MCP agent and connect to code-implementation server""" + try: + self.mcp_agent = Agent( + name="CodeImplementationAgent", + instruction="You are a code implementation assistant, using MCP tools to implement paper code replication.", + server_names=["code-implementation", "code-reference-indexer"], + ) + + await self.mcp_agent.__aenter__() + llm = await self.mcp_agent.attach_llm( + get_preferred_llm_class(self.config_path) + ) + + # Set workspace to the target code directory + workspace_result = await self.mcp_agent.call_tool( + "set_workspace", {"workspace_path": code_directory} + ) + self.logger.info(f"Workspace setup result: {workspace_result}") + + return llm + + except Exception as e: + self.logger.error(f"Failed to initialize MCP agent: {e}") + if self.mcp_agent: + try: + await self.mcp_agent.__aexit__(None, None, None) + except Exception: + pass + self.mcp_agent = None + raise + + async def _cleanup_mcp_agent(self): + """Clean up MCP agent resources""" + if self.mcp_agent: + try: + await self.mcp_agent.__aexit__(None, None, None) + self.logger.info("MCP agent connection closed") + except Exception as e: + self.logger.warning(f"Error closing MCP agent: {e}") + finally: + self.mcp_agent = None + + async def _initialize_llm_client(self): + """Initialize LLM client based on llm_provider preference and API key availability""" + # Get API keys + anthropic_key = self.api_config.get("anthropic", {}).get("api_key", "") + openai_key = self.api_config.get("openai", {}).get("api_key", "") + google_key = self.api_config.get("google", {}).get("api_key", "") + + # Read user preference from main config + preferred_provider = None + try: + import yaml + + config_path = "mcp_agent.config.yaml" + if os.path.exists(config_path): + with open(config_path, "r", encoding="utf-8") as f: + config = yaml.safe_load(f) + preferred_provider = config.get("llm_provider", "").strip().lower() + except Exception as e: + self.logger.warning(f"Could not read llm_provider preference: {e}") + + # Define provider initialization functions + async def init_anthropic(): + if not (anthropic_key and anthropic_key.strip()): + return None + try: + from anthropic import AsyncAnthropic + + client = AsyncAnthropic(api_key=anthropic_key) + await client.messages.create( + model=self.default_models["anthropic"], + max_tokens=20, + messages=[{"role": "user", "content": "test"}], + ) + self.logger.info( + f"Using Anthropic API with model: {self.default_models['anthropic']}" + ) + return client, "anthropic" + except Exception as e: + self.logger.warning(f"Anthropic API unavailable: {e}") + return None + + async def init_google(): + if not (google_key and google_key.strip()): + return None + try: + from google import genai + + client = genai.Client(api_key=google_key) + try: + test_response = await client.aio.models.generate_content( + model=self.default_models.get("google", "gemini-2.0-flash"), + contents="test", + ) + self.logger.info( + "Google API connection successful: " + str(test_response) + ) + except Exception as test_err: + self.logger.warning( + f"Could not test Google API: {test_err}, but will try to use client" + ) + + self.logger.info( + f"Using Google API with model: {self.default_models.get('google', 'gemini-2.0-flash')}" + ) + return client, "google" + except Exception as e: + self.logger.warning(f"Google API unavailable: {e}") + return None + + async def init_openai(): + if not (openai_key and openai_key.strip()): + return None + try: + from openai import AsyncOpenAI + + openai_config = self.api_config.get("openai", {}) + base_url = openai_config.get("base_url") + + if base_url: + client = AsyncOpenAI(api_key=openai_key, base_url=base_url) + else: + client = AsyncOpenAI(api_key=openai_key) + + model_name = self.default_models.get("openai", "o3-mini") + + try: + await client.chat.completions.create( + model=model_name, + max_tokens=20, + messages=[{"role": "user", "content": "test"}], + ) + except Exception as e: + if "max_tokens" in str(e) and "max_completion_tokens" in str(e): + self.logger.info( + f"Model {model_name} requires max_completion_tokens parameter" + ) + await client.chat.completions.create( + model=model_name, + max_completion_tokens=20, + messages=[{"role": "user", "content": "test"}], + ) + else: + raise + self.logger.info(f"Using OpenAI API with model: {model_name}") + if base_url: + self.logger.info(f"Using custom base URL: {base_url}") + return client, "openai" + except Exception as e: + self.logger.warning(f"OpenAI API unavailable: {e}") + return None + + # Map providers to their init functions + provider_init_map = { + "anthropic": init_anthropic, + "google": init_google, + "openai": init_openai, + } + + # Try preferred provider first + if preferred_provider and preferred_provider in provider_init_map: + self.logger.info(f"๐ŸŽฏ Trying preferred provider: {preferred_provider}") + result = await provider_init_map[preferred_provider]() + if result: + return result + else: + self.logger.warning( + f"โš ๏ธ Preferred provider '{preferred_provider}' unavailable, trying alternatives..." + ) + + # Fallback: try providers in order + for provider_name, init_func in provider_init_map.items(): + if provider_name == preferred_provider: + continue # Already tried + result = await init_func() + if result: + return result + + raise ValueError( + "No available LLM API - please check your API keys in configuration" + ) + + async def _call_llm_with_tools( + self, client, client_type, system_message, messages, tools, max_tokens=8192 + ): + """Call LLM with tools""" + try: + if client_type == "anthropic": + return await self._call_anthropic_with_tools( + client, system_message, messages, tools, max_tokens + ) + elif client_type == "openai": + return await self._call_openai_with_tools( + client, system_message, messages, tools, max_tokens + ) + elif client_type == "google": + return await self._call_google_with_tools( + client, system_message, messages, tools, max_tokens + ) + else: + raise ValueError(f"Unsupported client type: {client_type}") + except Exception as e: + self.logger.error(f"LLM call failed: {e}") + raise + + async def _call_anthropic_with_tools( + self, client, system_message, messages, tools, max_tokens + ): + """Call Anthropic API""" + validated_messages = self._validate_messages(messages) + if not validated_messages: + validated_messages = [ + {"role": "user", "content": "Please continue implementing code"} + ] + + try: + response = await client.messages.create( + model=self.default_models["anthropic"], + system=system_message, + messages=validated_messages, + tools=tools, + max_tokens=max_tokens, + temperature=0.2, + ) + except Exception as e: + self.logger.error(f"Anthropic API call failed: {e}") + raise + + content = "" + tool_calls = [] + + for block in response.content: + if block.type == "text": + content += block.text + elif block.type == "tool_use": + tool_calls.append( + {"id": block.id, "name": block.name, "input": block.input} + ) + + return {"content": content, "tool_calls": tool_calls} + + async def _call_google_with_tools( + self, client, system_message, messages, tools, max_tokens + ): + """ + Call Google Gemini API with tools + + Note: Google Gemini uses a completely different API structure. + The client here is expected to be google.genai.Client from google-genai SDK. + + Reference: https://ai.google.dev/gemini-api/docs/function-calling + """ + try: + from google.genai import types + except ImportError: + raise ImportError("google-genai package is required for Google API calls") + + validated_messages = self._validate_messages(messages) + if not validated_messages: + validated_messages = [ + {"role": "user", "content": "Please continue implementing code"} + ] + + # Convert messages to Google Gemini format (types.Content) + # Gemini expects: role="user" or role="model" (not "assistant") + gemini_messages = [] + for msg in validated_messages: + role = msg.get("role", "user") + content = msg.get("content", "") + + # Convert role names: "assistant" -> "model" + if role == "assistant": + role = "model" + elif role not in ["user", "model"]: + # Skip unsupported roles or convert to user + role = "user" + + gemini_messages.append( + types.Content(role=role, parts=[types.Part.from_text(text=content)]) + ) + + # Convert tools to Google Gemini format (types.Tool with FunctionDeclaration) + # Following the EXACT pattern from GoogleAugmentedLLM line 92-103 + # IMPORTANT: Each tool should be wrapped in its own Tool object! + gemini_tools = [] + if tools: + for tool in tools: + # Transform the input_schema to be Gemini-compatible + parameters = self._transform_schema_for_gemini(tool["input_schema"]) + + # Each tool gets its own Tool wrapper (not all in one!) + gemini_tools.append( + types.Tool( + function_declarations=[ + types.FunctionDeclaration( + name=tool["name"], + description=tool["description"], + parameters=parameters, + ) + ] + ) + ) + + # Create config with system instruction and tools + config = types.GenerateContentConfig( + max_output_tokens=max_tokens, + temperature=0.2, + system_instruction=system_message if system_message else None, + tools=gemini_tools if gemini_tools else None, + # Disable automatic function calling - we handle it manually + automatic_function_calling=types.AutomaticFunctionCallingConfig( + disable=True + ), + ) + + try: + # Google Gemini API call using the native SDK + # client is google.genai.Client instance + response = await client.aio.models.generate_content( + model=self.default_models["google"], + contents=gemini_messages, + config=config, + ) + except Exception as e: + self.logger.error(f"Google API call failed: {e}") + raise + + # Parse Gemini response (types.GenerateContentResponse) + # Following the pattern from augmented_llm_google.py lines 145-165 + content = "" + tool_calls = [] + + if response and hasattr(response, "candidates") and response.candidates: + candidate = response.candidates[0] + + if hasattr(candidate, "content") and candidate.content: + if hasattr(candidate.content, "parts") and candidate.content.parts: + for part in candidate.content.parts: + # Handle text content + if hasattr(part, "text") and part.text: + content += part.text + + # Handle function calls + # Check for function_call attribute, matching augmented_llm_google.py line 164 + if hasattr(part, "function_call") and part.function_call: + fc = part.function_call + # Extract function call details + # Note: Gemini function_call has name and args attributes + tool_call = { + "id": getattr( + fc, "id", getattr(fc, "name", "") + ), # Use name as fallback for id + "name": fc.name if hasattr(fc, "name") else "", + "input": dict(fc.args) + if hasattr(fc, "args") and fc.args + else {}, + } + self.logger.debug( + f"Google function_call parsed: {tool_call}" + ) + tool_calls.append(tool_call) + + return {"content": content, "tool_calls": tool_calls} + + def _transform_schema_for_gemini(self, schema: dict) -> dict: + """ + Transform JSON Schema to OpenAPI Schema format compatible with Gemini. + + This is based on the transform_mcp_tool_schema from GoogleAugmentedLLM. + Key transformations: + 1. Convert camelCase to snake_case + 2. Remove unsupported fields (default, additionalProperties) + 3. Handle nullable types via anyOf + """ + if not isinstance(schema, dict): + return schema + + # Fields to exclude + EXCLUDED_PROPERTIES = {"default", "additionalProperties"} + + # camelCase to snake_case mappings + CAMEL_TO_SNAKE = { + "anyOf": "any_of", + "maxLength": "max_length", + "minLength": "min_length", + "minProperties": "min_properties", + "maxProperties": "max_properties", + "maxItems": "max_items", + "minItems": "min_items", + } + + result = {} + + for key, value in schema.items(): + # Skip excluded properties + if key in EXCLUDED_PROPERTIES: + continue + + # Convert camelCase to snake_case + snake_key = CAMEL_TO_SNAKE.get(key, key) + + # Handle nested structures + if key == "properties" and isinstance(value, dict): + result[snake_key] = { + prop_k: self._transform_schema_for_gemini(prop_v) + for prop_k, prop_v in value.items() + } + elif key == "items" and isinstance(value, dict): + result[snake_key] = self._transform_schema_for_gemini(value) + elif key == "anyOf" and isinstance(value, list): + # Handle nullable types (Type | None) + has_null = any( + isinstance(item, dict) and item.get("type") == "null" + for item in value + ) + if has_null: + result["nullable"] = True + + # Get first non-null schema + for item in value: + if isinstance(item, dict) and item.get("type") != "null": + transformed = self._transform_schema_for_gemini(item) + for k, v in transformed.items(): + if k not in result: + result[k] = v + break + else: + result[snake_key] = value + + return result + + def _repair_truncated_json(self, json_str: str, tool_name: str = "") -> dict: + """ + Advanced JSON repair for truncated or malformed JSON from LLM responses. + + Handles: + - Missing closing braces/brackets + - Truncated string values + - Missing required fields + - Trailing commas + """ + import re + + # Step 1: Try basic fixes first + fixed = json_str.strip() + + # Remove trailing commas + fixed = re.sub(r",\s*}", "}", fixed) + fixed = re.sub(r",\s*]", "]", fixed) + + try: + return json.loads(fixed) + except json.JSONDecodeError as e: + print(" ๐Ÿ”ง Attempting advanced JSON repair...") + + # Step 2: Check for truncation issues + if e.msg == "Expecting value": + # Likely truncated - try to close open structures + fixed = self._close_json_structures(fixed) + try: + return json.loads(fixed) + except (json.JSONDecodeError, ValueError, TypeError): + pass + + # Step 3: Try to extract partial valid JSON + if e.msg.startswith("Expecting") and e.pos: + # Truncate at error position and try to close + truncated = fixed[: e.pos] + closed = self._close_json_structures(truncated) + try: + partial = json.loads(closed) + print(" โœ… Extracted partial JSON successfully") + return partial + except (json.JSONDecodeError, ValueError, TypeError): + pass + + # Step 4: Tool-specific defaults for critical tools + if tool_name == "write_file": + # For write_file, try to extract at least file_path + file_path_match = re.search(r'"file_path"\s*:\s*"([^"]*)"', fixed) + if file_path_match: + print(" โš ๏ธ write_file JSON truncated, using minimal structure") + return { + "file_path": file_path_match.group(1), + "content": "", # Empty content is better than crashing + } + + # Step 5: Last resort - return error indicator + print(" โŒ JSON repair failed completely") + return None + + def _close_json_structures(self, json_str: str) -> str: + """ + Intelligently close unclosed JSON structures. + Counts braces and brackets to determine what needs closing. + """ + # Count open structures + open_braces = json_str.count("{") - json_str.count("}") + open_brackets = json_str.count("[") - json_str.count("]") + + # Check if we're in the middle of a string + quote_count = json_str.count('"') + in_string = (quote_count % 2) != 0 + + result = json_str + + # Close string if needed + if in_string: + result += '"' + + # Close brackets first (inner structures) + result += "]" * open_brackets + + # Close braces + result += "}" * open_braces + + return result + + async def _call_openai_with_tools( + self, client, system_message, messages, tools, max_tokens + ): + """Call OpenAI API with robust JSON error handling and retry mechanism""" + openai_tools = [] + for tool in tools: + openai_tools.append( + { + "type": "function", + "function": { + "name": tool["name"], + "description": tool["description"], + "parameters": tool["input_schema"], + }, + } + ) + + openai_messages = [{"role": "system", "content": system_message}] + openai_messages.extend(messages) + + # Retry mechanism for API calls + max_retries = 3 + retry_delay = 2 # seconds + + for attempt in range(max_retries): + try: + # Try max_tokens first, fallback to max_completion_tokens if unsupported + try: + response = await client.chat.completions.create( + model=self.default_models["openai"], + messages=openai_messages, + tools=openai_tools if openai_tools else None, + max_tokens=max_tokens, + temperature=0.2, + ) + except Exception as e: + if "max_tokens" in str(e) and "max_completion_tokens" in str(e): + # Retry with max_completion_tokens for models that require it + response = await client.chat.completions.create( + model=self.default_models["openai"], + messages=openai_messages, + tools=openai_tools if openai_tools else None, + max_completion_tokens=max_tokens, + ) + else: + raise + + # Validate response structure + if ( + not response + or not hasattr(response, "choices") + or not response.choices + ): + raise ValueError("Invalid API response: missing choices") + + if not response.choices[0] or not hasattr( + response.choices[0], "message" + ): + raise ValueError("Invalid API response: missing message in choice") + + message = response.choices[0].message + content = message.content or "" + + # Successfully got a valid response + break + + except json.JSONDecodeError as e: + print( + f"\nโŒ JSON Decode Error in API response (attempt {attempt + 1}/{max_retries}):" + ) + print(f" Error: {e}") + print(f" Position: line {e.lineno}, column {e.colno}") + + if attempt < max_retries - 1: + print(f" โณ Retrying in {retry_delay} seconds...") + await asyncio.sleep(retry_delay) + retry_delay *= 2 # Exponential backoff + else: + print(" โŒ All retries exhausted") + raise + + except (ValueError, AttributeError, TypeError) as e: + print(f"\nโŒ API Response Error (attempt {attempt + 1}/{max_retries}):") + print(f" Error type: {type(e).__name__}") + print(f" Error: {e}") + + if attempt < max_retries - 1: + print(f" โณ Retrying in {retry_delay} seconds...") + await asyncio.sleep(retry_delay) + retry_delay *= 2 + else: + print(" โŒ All retries exhausted") + # Return empty response instead of crashing + return { + "content": "API error - unable to get valid response", + "tool_calls": [], + } + + except Exception as e: + print( + f"\nโŒ Unexpected API Error (attempt {attempt + 1}/{max_retries}):" + ) + print(f" Error type: {type(e).__name__}") + print(f" Error: {e}") + + if attempt < max_retries - 1: + print(f" โณ Retrying in {retry_delay} seconds...") + await asyncio.sleep(retry_delay) + retry_delay *= 2 + else: + print(" โŒ All retries exhausted") + raise + + tool_calls = [] + if message.tool_calls: + for tool_call in message.tool_calls: + try: + # Attempt to parse tool call arguments + parsed_input = json.loads(tool_call.function.arguments) + tool_calls.append( + { + "id": tool_call.id, + "name": tool_call.function.name, + "input": parsed_input, + } + ) + except json.JSONDecodeError as e: + # Detailed JSON parsing error logging + print("\nโŒ JSON Parsing Error in tool call:") + print(f" Tool: {tool_call.function.name}") + print(f" Error: {e}") + print(" Raw arguments (first 500 chars):") + print(f" {tool_call.function.arguments[:500]}") + print(f" Error position: line {e.lineno}, column {e.colno}") + print( + f" Problem at: ...{tool_call.function.arguments[max(0, e.pos-50):e.pos+50]}..." + ) + + # Attempt advanced JSON repair + repaired = self._repair_truncated_json( + tool_call.function.arguments, tool_call.function.name + ) + + if repaired: + print(" โœ… JSON repaired successfully") + tool_calls.append( + { + "id": tool_call.id, + "name": tool_call.function.name, + "input": repaired, + } + ) + else: + # Skip this tool call if repair failed + print(" โš ๏ธ Skipping unrepairable tool call") + continue + + return {"content": content, "tool_calls": tool_calls} + + # ==================== 5. Tools and Utility Methods (Utility Layer) ==================== + + def _validate_messages(self, messages: List[Dict]) -> List[Dict]: + """Validate and clean message list""" + valid_messages = [] + for msg in messages: + content = msg.get("content", "").strip() + if content: + valid_messages.append( + {"role": msg.get("role", "user"), "content": content} + ) + else: + self.logger.warning(f"Skipping empty message: {msg}") + return valid_messages + + def _prepare_mcp_tool_definitions(self) -> List[Dict[str, Any]]: + """Prepare tool definitions in Anthropic API standard format""" + return get_mcp_tools("code_implementation") + + def _check_tool_results_for_errors(self, tool_results: List[Dict]) -> bool: + """Check tool results for errors with JSON repair capability""" + for result in tool_results: + try: + if hasattr(result["result"], "content") and result["result"].content: + content_text = result["result"].content[0].text + + # First attempt: try direct JSON parsing + try: + parsed_result = json.loads(content_text) + if parsed_result.get("status") == "error": + return True + except json.JSONDecodeError as e: + # JSON parsing failed - try to repair + print("\nโš ๏ธ JSON parsing failed in tool result check:") + print(f" Error: {e}") + print( + f" Position: line {e.lineno}, column {e.colno}, char {e.pos}" + ) + print(f" Content length: {len(content_text)} chars") + print(f" First 300 chars: {content_text[:300]}") + + # Attempt to repair the JSON + repaired = self._repair_truncated_json(content_text) + if repaired: + print(" โœ… Tool result JSON repaired successfully") + if repaired.get("status") == "error": + return True + else: + # Fallback: check for "error" keyword in text + if "error" in content_text.lower(): + return True + + elif isinstance(result["result"], str): + if "error" in result["result"].lower(): + return True + + except (AttributeError, IndexError) as e: + # Unexpected result structure + print(f"\nโš ๏ธ Unexpected result structure: {type(e).__name__}: {e}") + result_str = str(result["result"]) + if "error" in result_str.lower(): + return True + return False + + # ==================== 6. User Interaction and Feedback (Interaction Layer) ==================== + + def _generate_success_guidance(self, files_count: int) -> str: + """Generate concise success guidance for continuing implementation""" + return f"""โœ… File implementation completed successfully! + +๐Ÿ“Š **Progress Status:** {files_count} files implemented + +๐ŸŽฏ **Next Action:** Check if ALL files from the reproduction plan are implemented. + +โšก **Decision Process:** +1. **If ALL files implemented:** Reply with "All files implemented" to complete the task +2. **If MORE files need implementation:** Continue with dependency-aware workflow: + - **Use `write_file` to implement the new component""" + + def _generate_error_guidance(self) -> str: + """Generate error guidance for handling issues""" + return """โŒ Error detected during file implementation. + +๐Ÿ”ง **Action Required:** +1. Review the error details above +2. Fix the identified issue +3. **Check if ALL files from the reproduction plan are implemented:** + - **If YES:** Respond "**implementation complete**" to end the conversation + - **If NO:** Continue with proper development cycle for next file: + - **Use `write_file` to implement properly +4. Ensure proper error handling in future implementations""" + + def _generate_no_tools_guidance(self, files_count: int) -> str: + """Generate concise guidance when no tools are called""" + return f"""โš ๏ธ No tool calls detected in your response. + +๐Ÿ“Š **Current Progress:** {files_count} files implemented + +๐Ÿšจ **Action Required:** Check completion status NOW: + +โšก **Decision Process:** +1. **If ALL files from plan are implemented:** Reply "All files implemented" to complete +2. **If MORE files need implementation:** Use tools to continue: + - **Use `write_file` to implement the new component + +๐Ÿšจ **Critical:** Don't just explain - either declare completion or use tools!""" + + def _compile_user_response(self, tool_results: List[Dict], guidance: str) -> str: + """Compile tool results and guidance into a single user response""" + response_parts = [] + + if tool_results: + response_parts.append("๐Ÿ”ง **Tool Execution Results:**") + for tool_result in tool_results: + tool_name = tool_result["tool_name"] + result_content = tool_result["result"] + response_parts.append( + f"```\nTool: {tool_name}\nResult: {result_content}\n```" + ) + + if guidance: + response_parts.append("\n" + guidance) + + return "\n\n".join(response_parts) + + # ==================== 7. Reporting and Output (Output Layer) ==================== + + async def _generate_pure_code_final_report_with_concise_agents( + self, + iterations: int, + elapsed_time: float, + code_agent: CodeImplementationAgent, + memory_agent: ConciseMemoryAgent, + ): + """Generate final report using concise agent statistics""" + try: + code_stats = code_agent.get_implementation_statistics() + memory_stats = memory_agent.get_memory_statistics( + code_stats["files_implemented_count"] + ) + + if self.mcp_agent: + history_result = await self.mcp_agent.call_tool( + "get_operation_history", {"last_n": 30} + ) + history_data = ( + json.loads(history_result) + if isinstance(history_result, str) + else history_result + ) + else: + history_data = {"total_operations": 0, "history": []} + + write_operations = 0 + files_created = [] + if "history" in history_data: + for item in history_data["history"]: + if item.get("action") == "write_file": + write_operations += 1 + file_path = item.get("details", {}).get("file_path", "unknown") + files_created.append(file_path) + + report = f""" +# Pure Code Implementation Completion Report (Write-File-Based Memory Mode) + +## Execution Summary +- Implementation iterations: {iterations} +- Total elapsed time: {elapsed_time:.2f} seconds +- Files implemented: {code_stats['total_files_implemented']} +- File write operations: {write_operations} +- Total MCP operations: {history_data.get('total_operations', 0)} + +## Read Tools Configuration +- Read tools enabled: {code_stats['read_tools_status']['read_tools_enabled']} +- Status: {code_stats['read_tools_status']['status']} +- Tools affected: {', '.join(code_stats['read_tools_status']['tools_affected'])} + +## Agent Performance +### Code Implementation Agent +- Files tracked: {code_stats['files_implemented_count']} +- Technical decisions: {code_stats['technical_decisions_count']} +- Constraints tracked: {code_stats['constraints_count']} +- Architecture notes: {code_stats['architecture_notes_count']} +- Dependency analysis performed: {code_stats['dependency_analysis_count']} +- Files read for dependencies: {code_stats['files_read_for_dependencies']} +- Last summary triggered at file count: {code_stats['last_summary_file_count']} + +### Concise Memory Agent (Write-File-Based) +- Last write_file detected: {memory_stats['last_write_file_detected']} +- Should clear memory next: {memory_stats['should_clear_memory_next']} +- Files implemented count: {memory_stats['implemented_files_tracked']} +- Current round: {memory_stats['current_round']} +- Concise mode active: {memory_stats['concise_mode_active']} +- Current round tool results: {memory_stats['current_round_tool_results']} +- Essential tools recorded: {memory_stats['essential_tools_recorded']} + +## Files Created +""" + for file_path in files_created[-20:]: + report += f"- {file_path}\n" + + if len(files_created) > 20: + report += f"... and {len(files_created) - 20} more files\n" + + report += """ +## Architecture Features +โœ… WRITE-FILE-BASED Memory Agent - Clear after each file generation +โœ… After write_file: Clear history โ†’ Keep system prompt + initial plan + tool results +โœ… Tool accumulation: read_code_mem, read_file, search_reference_code until next write_file +โœ… Clean memory cycle: write_file โ†’ clear โ†’ accumulate โ†’ write_file โ†’ clear +โœ… Essential tool recording with write_file detection +โœ… Specialized agent separation for clean code organization +โœ… MCP-compliant tool execution +โœ… Production-grade code with comprehensive type hints +โœ… Intelligent dependency analysis and file reading +โœ… Automated read_file usage for implementation context +โœ… Eliminates conversation clutter between file generations +โœ… Focused memory for efficient next file generation +""" + return report + + except Exception as e: + self.logger.error(f"Failed to generate final report: {e}") + return f"Failed to generate final report: {str(e)}" + + +async def main(): + """Main function for running the workflow""" + # Configure root logger carefully to avoid duplicates + root_logger = logging.getLogger() + if not root_logger.handlers: + handler = logging.StreamHandler() + formatter = logging.Formatter("%(levelname)s:%(name)s:%(message)s") + handler.setFormatter(formatter) + root_logger.addHandler(handler) + root_logger.setLevel(logging.INFO) + + workflow = CodeImplementationWorkflow() + + print("=" * 60) + print("Code Implementation Workflow with UNIFIED Reference Indexer") + print("=" * 60) + print("Select mode:") + print("1. Test Code Reference Indexer Integration") + print("2. Run Full Implementation Workflow") + print("3. Run Implementation with Pure Code Mode") + print("4. Test Read Tools Configuration") + + # mode_choice = input("Enter choice (1-4, default: 3): ").strip() + + # For testing purposes, we'll run the test first + # if mode_choice == "4": + # print("Testing Read Tools Configuration...") + + # # Create a test workflow normally + # test_workflow = CodeImplementationWorkflow() + + # # Create a mock code agent for testing + # print("\n๐Ÿงช Testing with read tools DISABLED:") + # test_agent_disabled = CodeImplementationAgent(None, enable_read_tools=False) + # await test_agent_disabled.test_read_tools_configuration() + + # print("\n๐Ÿงช Testing with read tools ENABLED:") + # test_agent_enabled = CodeImplementationAgent(None, enable_read_tools=True) + # await test_agent_enabled.test_read_tools_configuration() + + # print("โœ… Read tools configuration testing completed!") + # return + + # print("Running Code Reference Indexer Integration Test...") + + test_success = True + if test_success: + print("\n" + "=" * 60) + print("๐ŸŽ‰ UNIFIED Code Reference Indexer Integration Test PASSED!") + print("๐Ÿ”ง Three-step process successfully merged into ONE tool") + print("=" * 60) + + # Ask if user wants to continue with actual workflow + print("\nContinuing with workflow execution...") + + plan_file = "/Users/lizongwei/Desktop/DeepCode_Project/workbase/DeepCode/deepcode_lab/papers/2/initial_plan.txt" + target_directory = "/Users/lizongwei/Desktop/DeepCode_Project/workbase/DeepCode/deepcode_lab/papers/2/" + print("Implementation Mode Selection:") + print("1. Pure Code Implementation Mode (Recommended)") + print("2. Iterative Implementation Mode") + + pure_code_mode = True + mode_name = "Pure Code Implementation Mode with Memory Agent Architecture + Code Reference Indexer" + print(f"Using: {mode_name}") + + # Configure read tools - modify this parameter to enable/disable read tools + enable_read_tools = ( + True # Set to False to disable read_file and read_code_mem tools + ) + read_tools_status = "ENABLED" if enable_read_tools else "DISABLED" + print(f"๐Ÿ”ง Read tools (read_file, read_code_mem): {read_tools_status}") + + # NOTE: To test without read tools, change the line above to: + # enable_read_tools = False + + result = await workflow.run_workflow( + plan_file, + target_directory=target_directory, + pure_code_mode=pure_code_mode, + enable_read_tools=enable_read_tools, + ) + + print("=" * 60) + print("Workflow Execution Results:") + print(f"Status: {result['status']}") + print(f"Mode: {mode_name}") + + if result["status"] == "success": + print(f"Code Directory: {result['code_directory']}") + print(f"MCP Architecture: {result.get('mcp_architecture', 'unknown')}") + print("Execution completed!") + else: + print(f"Error Message: {result['message']}") + + print("=" * 60) + print( + "โœ… Using Standard MCP Architecture with Memory Agent + Code Reference Indexer" + ) + + else: + print("\n" + "=" * 60) + print("โŒ Code Reference Indexer Integration Test FAILED!") + print("Please check the configuration and try again.") + print("=" * 60) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/DeepCode/workflows/code_implementation_workflow_index.py b/DeepCode/workflows/code_implementation_workflow_index.py new file mode 100644 index 00000000..079754d9 --- /dev/null +++ b/DeepCode/workflows/code_implementation_workflow_index.py @@ -0,0 +1,1503 @@ +""" +Paper Code Implementation Workflow - MCP-compliant Iterative Development + +Features: +1. File Tree Creation +2. Code Implementation - Based on aisi-basic-agent iterative development + +MCP Architecture: +- MCP Server: tools/code_implementation_server.py +- MCP Client: Called through mcp_agent framework +- Configuration: mcp_agent.config.yaml +""" + +import asyncio +import json +import logging +import os +import sys +import time +import yaml +from pathlib import Path +from typing import Dict, Any, Optional, List + +# MCP Agent imports +from mcp_agent.agents.agent import Agent + +# Local imports +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from prompts.code_prompts import STRUCTURE_GENERATOR_PROMPT +from prompts.code_prompts import ( + PURE_CODE_IMPLEMENTATION_SYSTEM_PROMPT_INDEX, +) +from workflows.agents import CodeImplementationAgent +from workflows.agents.memory_agent_concise import ConciseMemoryAgent +from config.mcp_tool_definitions_index import get_mcp_tools +from utils.llm_utils import get_preferred_llm_class, get_default_models +# DialogueLogger removed - no longer needed + + +class CodeImplementationWorkflowWithIndex: + """ + Paper Code Implementation Workflow Manager with Code Reference Indexer + + Uses standard MCP architecture with enhanced indexing capabilities: + 1. Connect to code-implementation server via MCP client + 2. Use MCP protocol for tool calls + 3. Support workspace management and operation history tracking + 4. Integrated code reference indexer for enhanced code understanding + """ + + # ==================== 1. Class Initialization and Configuration (Infrastructure Layer) ==================== + + def __init__(self, config_path: str = "mcp_agent.secrets.yaml"): + """Initialize workflow with configuration""" + self.config_path = config_path + self.api_config = self._load_api_config() + self.default_models = get_default_models("mcp_agent.config.yaml") + self.logger = self._create_logger() + self.mcp_agent = None + self.enable_read_tools = ( + True # Default value, will be overridden by run_workflow parameter + ) + + def _load_api_config(self) -> Dict[str, Any]: + """Load API configuration from YAML file""" + try: + with open(self.config_path, "r", encoding="utf-8") as f: + return yaml.safe_load(f) + except Exception as e: + raise Exception(f"Failed to load API config: {e}") + + def _create_logger(self) -> logging.Logger: + """Create and configure logger""" + logger = logging.getLogger(__name__) + # Don't add handlers to child loggers - let them propagate to root + logger.setLevel(logging.INFO) + return logger + + def _read_plan_file(self, plan_file_path: str) -> str: + """Read implementation plan file""" + plan_path = Path(plan_file_path) + if not plan_path.exists(): + raise FileNotFoundError( + f"Implementation plan file not found: {plan_file_path}" + ) + + with open(plan_path, "r", encoding="utf-8") as f: + return f.read() + + def _check_file_tree_exists(self, target_directory: str) -> bool: + """Check if file tree structure already exists""" + code_directory = os.path.join(target_directory, "generate_code") + return os.path.exists(code_directory) and len(os.listdir(code_directory)) > 0 + + # ==================== 2. Public Interface Methods (External API Layer) ==================== + + async def run_workflow( + self, + plan_file_path: str, + target_directory: Optional[str] = None, + pure_code_mode: bool = False, + enable_read_tools: bool = True, + ): + """Run complete workflow - Main public interface""" + # Set the read tools configuration + self.enable_read_tools = enable_read_tools + + try: + plan_content = self._read_plan_file(plan_file_path) + + if target_directory is None: + target_directory = str(Path(plan_file_path).parent) + + # Calculate code directory for workspace alignment + code_directory = os.path.join(target_directory, "generate_code") + + self.logger.info("=" * 80) + self.logger.info("๐Ÿš€ STARTING CODE IMPLEMENTATION WORKFLOW") + self.logger.info("=" * 80) + self.logger.info(f"๐Ÿ“„ Plan file: {plan_file_path}") + self.logger.info(f"๐Ÿ“‚ Plan file parent: {target_directory}") + self.logger.info(f"๐ŸŽฏ Code directory (MCP workspace): {code_directory}") + self.logger.info( + f"โš™๏ธ Read tools: {'ENABLED' if self.enable_read_tools else 'DISABLED'}" + ) + self.logger.info("=" * 80) + + results = {} + + # Check if file tree exists + if self._check_file_tree_exists(target_directory): + self.logger.info("File tree exists, skipping creation") + results["file_tree"] = "Already exists, skipped creation" + else: + self.logger.info("Creating file tree...") + results["file_tree"] = await self.create_file_structure( + plan_content, target_directory + ) + + # Code implementation + if pure_code_mode: + self.logger.info("Starting pure code implementation...") + results["code_implementation"] = await self.implement_code_pure( + plan_content, target_directory, code_directory + ) + else: + pass + + self.logger.info("Workflow execution successful") + + return { + "status": "success", + "plan_file": plan_file_path, + "target_directory": target_directory, + "code_directory": os.path.join(target_directory, "generate_code"), + "results": results, + "mcp_architecture": "standard", + } + + except Exception as e: + self.logger.error(f"Workflow execution failed: {e}") + + return {"status": "error", "message": str(e), "plan_file": plan_file_path} + finally: + await self._cleanup_mcp_agent() + + async def create_file_structure( + self, plan_content: str, target_directory: str + ) -> str: + """Create file tree structure based on implementation plan""" + self.logger.info("Starting file tree creation...") + + structure_agent = Agent( + name="StructureGeneratorAgent", + instruction=STRUCTURE_GENERATOR_PROMPT, + server_names=["command-executor"], + ) + + async with structure_agent: + creator = await structure_agent.attach_llm( + get_preferred_llm_class(self.config_path) + ) + + message = f"""Analyze the following implementation plan and generate shell commands to create the file tree structure. + +Target Directory: {target_directory}/generate_code + +Implementation Plan: +{plan_content} + +Tasks: +1. Find the file tree structure in the implementation plan +2. Generate shell commands (mkdir -p, touch) to create that structure +3. Use the execute_commands tool to run the commands and create the file structure + +Requirements: +- Use mkdir -p to create directories +- Use touch to create files +- Include __init__.py file for Python packages +- Use relative paths to the target directory +- Execute commands to actually create the file structure""" + + result = await creator.generate_str(message=message) + self.logger.info("File tree structure creation completed") + return result + + async def implement_code_pure( + self, plan_content: str, target_directory: str, code_directory: str = None + ) -> str: + """Pure code implementation - focus on code writing without testing""" + self.logger.info("Starting pure code implementation (no testing)...") + + # Use provided code_directory or calculate it (for backwards compatibility) + if code_directory is None: + code_directory = os.path.join(target_directory, "generate_code") + + self.logger.info(f"๐ŸŽฏ Using code directory (MCP workspace): {code_directory}") + + if not os.path.exists(code_directory): + self.logger.warning( + f"Code directory does not exist, creating it: {code_directory}" + ) + os.makedirs(code_directory, exist_ok=True) + self.logger.info(f"โœ… Code directory created: {code_directory}") + + try: + client, client_type = await self._initialize_llm_client() + await self._initialize_mcp_agent(code_directory) + + tools = self._prepare_mcp_tool_definitions() + system_message = PURE_CODE_IMPLEMENTATION_SYSTEM_PROMPT_INDEX + messages = [] + + # implementation_message = f"""**TASK: Implement Research Paper Reproduction Code** + + # You are implementing a complete, working codebase that reproduces the core algorithms, experiments, and methods described in a research paper. Your goal is to create functional code that can replicate the paper's key results and contributions. + + # **What you need to do:** + # - Analyze the paper content and reproduction plan to understand requirements + # - Implement all core algorithms mentioned in the main body of the paper + # - Create the necessary components following the planned architecture + # - Test each component to ensure functionality + # - Integrate components into a cohesive, executable system + # - Focus on reproducing main contributions rather than appendix-only experiments + + # **RESOURCES:** + # - **Paper & Reproduction Plan**: `{target_directory}/` (contains .md paper files and initial_plan.txt with detailed implementation guidance) + # - **Reference Code Indexes**: `{target_directory}/indexes/` (JSON files with implementation patterns from related codebases) + # - **Implementation Directory**: `{code_directory}/` (your working directory for all code files) + + # **CURRENT OBJECTIVE:** + # Start by reading the reproduction plan (`{target_directory}/initial_plan.txt`) to understand the implementation strategy, then examine the paper content to identify the first priority component to implement. Use the search_code tool to find relevant reference implementations from the indexes directory (`{target_directory}/indexes/*.json`) before coding. + + # --- + # **START:** Review the plan above and begin implementation.""" + implementation_message = f"""**Task: Implement code based on the following reproduction plan** + +**Code Reproduction Plan:** +{plan_content} + +**Working Directory:** {code_directory} + +**Current Objective:** Begin implementation by analyzing the plan structure, examining the current project layout, and implementing the first foundation file according to the plan's priority order.""" + + messages.append({"role": "user", "content": implementation_message}) + + result = await self._pure_code_implementation_loop( + client, + client_type, + system_message, + messages, + tools, + plan_content, + target_directory, + ) + + return result + + finally: + await self._cleanup_mcp_agent() + + # ==================== 3. Core Business Logic (Implementation Layer) ==================== + + async def _pure_code_implementation_loop( + self, + client, + client_type, + system_message, + messages, + tools, + plan_content, + target_directory, + ): + """Pure code implementation loop with memory optimization and phase consistency""" + max_iterations = 800 + iteration = 0 + start_time = time.time() + max_time = 7200 # 120 minutes (2 hours) + + # Initialize specialized agents + code_agent = CodeImplementationAgent( + self.mcp_agent, self.logger, self.enable_read_tools + ) + + # Pass code_directory to memory agent for file extraction + code_directory = os.path.join(target_directory, "generate_code") + memory_agent = ConciseMemoryAgent( + plan_content, + self.logger, + target_directory, + self.default_models, + code_directory, + ) + + # Log read tools configuration + read_tools_status = "ENABLED" if self.enable_read_tools else "DISABLED" + self.logger.info( + f"๐Ÿ”ง Read tools (read_file, read_code_mem): {read_tools_status}" + ) + if not self.enable_read_tools: + self.logger.info( + "๐Ÿšซ No read mode: read_file and read_code_mem tools will be skipped" + ) + + # Connect code agent with memory agent for summary generation + # Note: Concise memory agent doesn't need LLM client for summary generation + code_agent.set_memory_agent(memory_agent, client, client_type) + + # Initialize memory agent with iteration 0 + memory_agent.start_new_round(iteration=0) + + while iteration < max_iterations: + iteration += 1 + elapsed_time = time.time() - start_time + + if elapsed_time > max_time: + self.logger.warning(f"Time limit reached: {elapsed_time:.2f}s") + break + + # # Test simplified memory approach if we have files implemented + # if iteration == 5 and code_agent.get_files_implemented_count() > 0: + # self.logger.info("๐Ÿงช Testing simplified memory approach...") + # test_results = await memory_agent.test_simplified_memory_approach() + # self.logger.info(f"Memory test results: {test_results}") + + # self.logger.info(f"Pure code implementation iteration {iteration}: generating code") + + messages = self._validate_messages(messages) + current_system_message = code_agent.get_system_prompt() + + # Round logging removed + + # Call LLM + response = await self._call_llm_with_tools( + client, client_type, current_system_message, messages, tools + ) + + response_content = response.get("content", "").strip() + if not response_content: + response_content = "Continue implementing code files..." + + messages.append({"role": "assistant", "content": response_content}) + + # Handle tool calls + if response.get("tool_calls"): + tool_results = await code_agent.execute_tool_calls( + response["tool_calls"] + ) + + # Record essential tool results in concise memory agent + for tool_call, tool_result in zip(response["tool_calls"], tool_results): + memory_agent.record_tool_result( + tool_name=tool_call["name"], + tool_input=tool_call["input"], + tool_result=tool_result.get("result"), + ) + + # NEW LOGIC: Check if write_file was called and trigger memory optimization immediately + + # Determine guidance based on results + has_error = self._check_tool_results_for_errors(tool_results) + files_count = code_agent.get_files_implemented_count() + + if has_error: + guidance = self._generate_error_guidance() + else: + guidance = self._generate_success_guidance(files_count) + + compiled_response = self._compile_user_response(tool_results, guidance) + messages.append({"role": "user", "content": compiled_response}) + + # NEW LOGIC: Apply memory optimization immediately after write_file detection + if memory_agent.should_trigger_memory_optimization( + messages, code_agent.get_files_implemented_count() + ): + # Memory optimization triggered + + # Apply concise memory optimization + files_implemented_count = code_agent.get_files_implemented_count() + current_system_message = code_agent.get_system_prompt() + messages = memory_agent.apply_memory_optimization( + current_system_message, messages, files_implemented_count + ) + + # Memory optimization completed + + else: + files_count = code_agent.get_files_implemented_count() + no_tools_guidance = self._generate_no_tools_guidance(files_count) + messages.append({"role": "user", "content": no_tools_guidance}) + + # Check for analysis loop and provide corrective guidance + # if code_agent.is_in_analysis_loop(): + # analysis_loop_guidance = code_agent.get_analysis_loop_guidance() + # messages.append({"role": "user", "content": analysis_loop_guidance}) + # self.logger.warning( + # "Analysis loop detected and corrective guidance provided" + # ) + + # Record file implementations in memory agent (for the current round) + for file_info in code_agent.get_implementation_summary()["completed_files"]: + memory_agent.record_file_implementation(file_info["file"]) + + # REMOVED: Old memory optimization logic - now happens immediately after write_file + # Memory optimization is now triggered immediately after write_file detection + + # Start new round for next iteration, sync with workflow iteration + memory_agent.start_new_round(iteration=iteration) + + # Check completion based on actual unimplemented files list + unimplemented_files = memory_agent.get_unimplemented_files() + if not unimplemented_files: # Empty list means all files implemented + self.logger.info( + "โœ… Code implementation complete - All files implemented" + ) + break + + # Emergency trim if too long + if len(messages) > 50: + self.logger.warning( + "Emergency message trim - applying concise memory optimization" + ) + + current_system_message = code_agent.get_system_prompt() + files_implemented_count = code_agent.get_files_implemented_count() + messages = memory_agent.apply_memory_optimization( + current_system_message, messages, files_implemented_count + ) + + return await self._generate_pure_code_final_report_with_concise_agents( + iteration, time.time() - start_time, code_agent, memory_agent + ) + + # ==================== 4. MCP Agent and LLM Communication Management (Communication Layer) ==================== + + async def _initialize_mcp_agent(self, code_directory: str): + """Initialize MCP agent and connect to code-implementation server""" + try: + self.mcp_agent = Agent( + name="CodeImplementationAgent", + instruction="You are a code implementation assistant, using MCP tools to implement paper code replication.", + server_names=["code-implementation", "code-reference-indexer"], + ) + + await self.mcp_agent.__aenter__() + llm = await self.mcp_agent.attach_llm( + get_preferred_llm_class(self.config_path) + ) + + # Set workspace to the target code directory + workspace_result = await self.mcp_agent.call_tool( + "set_workspace", {"workspace_path": code_directory} + ) + self.logger.info(f"Workspace setup result: {workspace_result}") + + return llm + + except Exception as e: + self.logger.error(f"Failed to initialize MCP agent: {e}") + if self.mcp_agent: + try: + await self.mcp_agent.__aexit__(None, None, None) + except Exception: + pass + self.mcp_agent = None + raise + + async def _cleanup_mcp_agent(self): + """Clean up MCP agent resources""" + if self.mcp_agent: + try: + await self.mcp_agent.__aexit__(None, None, None) + self.logger.info("MCP agent connection closed") + except Exception as e: + self.logger.warning(f"Error closing MCP agent: {e}") + finally: + self.mcp_agent = None + + async def _initialize_llm_client(self): + """Initialize LLM client based on llm_provider preference and API key availability""" + # Get API keys + anthropic_key = self.api_config.get("anthropic", {}).get("api_key", "") + openai_key = self.api_config.get("openai", {}).get("api_key", "") + google_key = self.api_config.get("google", {}).get("api_key", "") + + # Read user preference from main config + preferred_provider = None + try: + import yaml + + config_path = "mcp_agent.config.yaml" + if os.path.exists(config_path): + with open(config_path, "r", encoding="utf-8") as f: + config = yaml.safe_load(f) + preferred_provider = config.get("llm_provider", "").strip().lower() + except Exception as e: + self.logger.warning(f"Could not read llm_provider preference: {e}") + + # Define provider initialization functions + async def init_anthropic(): + if not (anthropic_key and anthropic_key.strip()): + return None + try: + from anthropic import AsyncAnthropic + + client = AsyncAnthropic(api_key=anthropic_key) + await client.messages.create( + model=self.default_models["anthropic"], + max_tokens=20, + messages=[{"role": "user", "content": "test"}], + ) + self.logger.info( + f"Using Anthropic API with model: {self.default_models['anthropic']}" + ) + return client, "anthropic" + except Exception as e: + self.logger.warning(f"Anthropic API unavailable: {e}") + return None + + async def init_google(): + if not (google_key and google_key.strip()): + return None + try: + from google import genai + + client = genai.Client(api_key=google_key) + try: + test_response = await client.aio.models.generate_content( + model=self.default_models.get("google", "gemini-2.0-flash"), + contents="test", + ) + + self.logger.info( + "Google API connection successful: " + str(test_response) + ) + except Exception as test_err: + self.logger.warning( + f"Could not test Google API: {test_err}, but will try to use client" + ) + + self.logger.info( + f"Using Google API with model: {self.default_models.get('google', 'gemini-2.0-flash')}" + ) + return client, "google" + except Exception as e: + self.logger.warning(f"Google API unavailable: {e}") + return None + + async def init_openai(): + if not (openai_key and openai_key.strip()): + return None + try: + from openai import AsyncOpenAI + + openai_config = self.api_config.get("openai", {}) + base_url = openai_config.get("base_url") + + if base_url: + client = AsyncOpenAI(api_key=openai_key, base_url=base_url) + else: + client = AsyncOpenAI(api_key=openai_key) + + model_name = self.default_models.get("openai", "o3-mini") + + try: + await client.chat.completions.create( + model=model_name, + max_tokens=20, + messages=[{"role": "user", "content": "test"}], + ) + except Exception as e: + if "max_tokens" in str(e) and "max_completion_tokens" in str(e): + self.logger.info( + f"Model {model_name} requires max_completion_tokens parameter" + ) + await client.chat.completions.create( + model=model_name, + max_completion_tokens=20, + messages=[{"role": "user", "content": "test"}], + ) + else: + raise + self.logger.info(f"Using OpenAI API with model: {model_name}") + if base_url: + self.logger.info(f"Using custom base URL: {base_url}") + return client, "openai" + except Exception as e: + self.logger.warning(f"OpenAI API unavailable: {e}") + return None + + # Map providers to their init functions + provider_init_map = { + "anthropic": init_anthropic, + "google": init_google, + "openai": init_openai, + } + + # Try preferred provider first + if preferred_provider and preferred_provider in provider_init_map: + self.logger.info(f"๐ŸŽฏ Trying preferred provider: {preferred_provider}") + result = await provider_init_map[preferred_provider]() + if result: + return result + else: + self.logger.warning( + f"โš ๏ธ Preferred provider '{preferred_provider}' unavailable, trying alternatives..." + ) + + # Fallback: try providers in order + for provider_name, init_func in provider_init_map.items(): + if provider_name == preferred_provider: + continue # Already tried + result = await init_func() + if result: + return result + + raise ValueError( + "No available LLM API - please check your API keys in configuration" + ) + + async def _call_llm_with_tools( + self, client, client_type, system_message, messages, tools, max_tokens=8192 + ): + """Call LLM with tools""" + try: + if client_type == "anthropic": + return await self._call_anthropic_with_tools( + client, system_message, messages, tools, max_tokens + ) + elif client_type == "openai": + return await self._call_openai_with_tools( + client, system_message, messages, tools, max_tokens + ) + elif client_type == "google": + return await self._call_google_with_tools( + client, system_message, messages, tools, max_tokens + ) + else: + raise ValueError(f"Unsupported client type: {client_type}") + except Exception as e: + self.logger.error(f"LLM call failed: {e}") + raise + + async def _call_anthropic_with_tools( + self, client, system_message, messages, tools, max_tokens + ): + """Call Anthropic API""" + validated_messages = self._validate_messages(messages) + if not validated_messages: + validated_messages = [ + {"role": "user", "content": "Please continue implementing code"} + ] + + try: + response = await client.messages.create( + model=self.default_models["anthropic"], + system=system_message, + messages=validated_messages, + tools=tools, + max_tokens=max_tokens, + temperature=0.2, + ) + except Exception as e: + self.logger.error(f"Anthropic API call failed: {e}") + raise + + content = "" + tool_calls = [] + + for block in response.content: + if block.type == "text": + content += block.text + elif block.type == "tool_use": + tool_calls.append( + {"id": block.id, "name": block.name, "input": block.input} + ) + + return {"content": content, "tool_calls": tool_calls} + + async def _call_google_with_tools( + self, client, system_message, messages, tools, max_tokens + ): + """ + Call Google Gemini API with tools + + Note: Google Gemini uses a completely different API structure. + The client here is expected to be google.genai.Client from google-genai SDK. + + Reference: https://ai.google.dev/gemini-api/docs/function-calling + """ + try: + from google.genai import types + except ImportError: + raise ImportError("google-genai package is required for Google API calls") + + validated_messages = self._validate_messages(messages) + if not validated_messages: + validated_messages = [ + {"role": "user", "content": "Please continue implementing code"} + ] + + # Convert messages to Google Gemini format (types.Content) + # Gemini expects: role="user" or role="model" (not "assistant") + gemini_messages = [] + for msg in validated_messages: + role = msg.get("role", "user") + content = msg.get("content", "") + + # Convert role names: "assistant" -> "model" + if role == "assistant": + role = "model" + elif role not in ["user", "model"]: + # Skip unsupported roles or convert to user + role = "user" + + gemini_messages.append( + types.Content(role=role, parts=[types.Part.from_text(text=content)]) + ) + + # Convert tools to Google Gemini format (types.Tool with FunctionDeclaration) + # Following the EXACT pattern from GoogleAugmentedLLM line 92-103 + # IMPORTANT: Each tool should be wrapped in its own Tool object! + gemini_tools = [] + if tools: + for tool in tools: + # Transform the input_schema to be Gemini-compatible + parameters = self._transform_schema_for_gemini(tool["input_schema"]) + + # Each tool gets its own Tool wrapper (not all in one!) + gemini_tools.append( + types.Tool( + function_declarations=[ + types.FunctionDeclaration( + name=tool["name"], + description=tool["description"], + parameters=parameters, + ) + ] + ) + ) + + # Create config with system instruction and tools + config = types.GenerateContentConfig( + max_output_tokens=max_tokens, + temperature=0.2, + system_instruction=system_message if system_message else None, + tools=gemini_tools if gemini_tools else None, + # Disable automatic function calling - we handle it manually + automatic_function_calling=types.AutomaticFunctionCallingConfig( + disable=True + ), + ) + + try: + # Google Gemini API call using the native SDK + # client is google.genai.Client instance + response = await client.aio.models.generate_content( + model=self.default_models["google"], + contents=gemini_messages, + config=config, + ) + except Exception as e: + self.logger.error(f"Google API call failed: {e}") + raise + + # Parse Gemini response (types.GenerateContentResponse) + # Following the pattern from augmented_llm_google.py lines 145-165 + content = "" + tool_calls = [] + + if response and hasattr(response, "candidates") and response.candidates: + candidate = response.candidates[0] + + if hasattr(candidate, "content") and candidate.content: + if hasattr(candidate.content, "parts") and candidate.content.parts: + for part in candidate.content.parts: + # Handle text content + if hasattr(part, "text") and part.text: + content += part.text + + # Handle function calls + # Check for function_call attribute, matching augmented_llm_google.py line 164 + if hasattr(part, "function_call") and part.function_call: + fc = part.function_call + # Extract function call details + # Note: Gemini function_call has name and args attributes + tool_call = { + "id": getattr( + fc, "id", getattr(fc, "name", "") + ), # Use name as fallback for id + "name": fc.name if hasattr(fc, "name") else "", + "input": dict(fc.args) + if hasattr(fc, "args") and fc.args + else {}, + } + self.logger.debug( + f"Google function_call parsed: {tool_call}" + ) + tool_calls.append(tool_call) + + return {"content": content, "tool_calls": tool_calls} + + def _transform_schema_for_gemini(self, schema: dict) -> dict: + """ + Transform JSON Schema to OpenAPI Schema format compatible with Gemini. + + This is based on the transform_mcp_tool_schema from GoogleAugmentedLLM. + Key transformations: + 1. Convert camelCase to snake_case + 2. Remove unsupported fields (default, additionalProperties) + 3. Handle nullable types via anyOf + """ + if not isinstance(schema, dict): + return schema + + # Fields to exclude + EXCLUDED_PROPERTIES = {"default", "additionalProperties"} + + # camelCase to snake_case mappings + CAMEL_TO_SNAKE = { + "anyOf": "any_of", + "maxLength": "max_length", + "minLength": "min_length", + "minProperties": "min_properties", + "maxProperties": "max_properties", + "maxItems": "max_items", + "minItems": "min_items", + } + + result = {} + + for key, value in schema.items(): + # Skip excluded properties + if key in EXCLUDED_PROPERTIES: + continue + + # Convert camelCase to snake_case + snake_key = CAMEL_TO_SNAKE.get(key, key) + + # Handle nested structures + if key == "properties" and isinstance(value, dict): + result[snake_key] = { + prop_k: self._transform_schema_for_gemini(prop_v) + for prop_k, prop_v in value.items() + } + elif key == "items" and isinstance(value, dict): + result[snake_key] = self._transform_schema_for_gemini(value) + elif key == "anyOf" and isinstance(value, list): + # Handle nullable types (Type | None) + has_null = any( + isinstance(item, dict) and item.get("type") == "null" + for item in value + ) + if has_null: + result["nullable"] = True + + # Get first non-null schema + for item in value: + if isinstance(item, dict) and item.get("type") != "null": + transformed = self._transform_schema_for_gemini(item) + for k, v in transformed.items(): + if k not in result: + result[k] = v + break + else: + result[snake_key] = value + + return result + + def _repair_truncated_json(self, json_str: str, tool_name: str = "") -> dict: + """ + Advanced JSON repair for truncated or malformed JSON from LLM responses. + + Handles: + - Missing closing braces/brackets + - Truncated string values + - Missing required fields + - Trailing commas + """ + import re + + # Step 1: Try basic fixes first + fixed = json_str.strip() + + # Remove trailing commas + fixed = re.sub(r",\s*}", "}", fixed) + fixed = re.sub(r",\s*]", "]", fixed) + + try: + return json.loads(fixed) + except json.JSONDecodeError as e: + print(" ๐Ÿ”ง Attempting advanced JSON repair...") + + # Step 2: Check for truncation issues + if e.msg == "Expecting value": + # Likely truncated - try to close open structures + fixed = self._close_json_structures(fixed) + try: + return json.loads(fixed) + except (json.JSONDecodeError, ValueError, TypeError): + pass + + # Step 3: Try to extract partial valid JSON + if e.msg.startswith("Expecting") and e.pos: + # Truncate at error position and try to close + truncated = fixed[: e.pos] + closed = self._close_json_structures(truncated) + try: + partial = json.loads(closed) + print(" โœ… Extracted partial JSON successfully") + return partial + except (json.JSONDecodeError, ValueError, TypeError): + pass + + # Step 4: Tool-specific defaults for critical tools + if tool_name == "write_file": + # For write_file, try to extract at least file_path + file_path_match = re.search(r'"file_path"\s*:\s*"([^"]*)"', fixed) + if file_path_match: + print(" โš ๏ธ write_file JSON truncated, using minimal structure") + return { + "file_path": file_path_match.group(1), + "content": "", # Empty content is better than crashing + } + + # Step 5: Last resort - return error indicator + print(" โŒ JSON repair failed completely") + return None + + def _close_json_structures(self, json_str: str) -> str: + """ + Intelligently close unclosed JSON structures. + Counts braces and brackets to determine what needs closing. + """ + # Count open structures + open_braces = json_str.count("{") - json_str.count("}") + open_brackets = json_str.count("[") - json_str.count("]") + + # Check if we're in the middle of a string + quote_count = json_str.count('"') + in_string = (quote_count % 2) != 0 + + result = json_str + + # Close string if needed + if in_string: + result += '"' + + # Close brackets first (inner structures) + result += "]" * open_brackets + + # Close braces + result += "}" * open_braces + + return result + + async def _call_openai_with_tools( + self, client, system_message, messages, tools, max_tokens + ): + """Call OpenAI API with robust JSON error handling and retry mechanism""" + openai_tools = [] + for tool in tools: + openai_tools.append( + { + "type": "function", + "function": { + "name": tool["name"], + "description": tool["description"], + "parameters": tool["input_schema"], + }, + } + ) + + openai_messages = [{"role": "system", "content": system_message}] + openai_messages.extend(messages) + + # Retry mechanism for API calls + max_retries = 3 + retry_delay = 2 # seconds + + for attempt in range(max_retries): + try: + # Try max_tokens first, fallback to max_completion_tokens if unsupported + try: + response = await client.chat.completions.create( + model=self.default_models["openai"], + messages=openai_messages, + tools=openai_tools if openai_tools else None, + max_tokens=max_tokens, + temperature=0.2, + ) + except Exception as e: + if "max_tokens" in str(e) and "max_completion_tokens" in str(e): + # Retry with max_completion_tokens for models that require it + response = await client.chat.completions.create( + model=self.default_models["openai"], + messages=openai_messages, + tools=openai_tools if openai_tools else None, + max_completion_tokens=max_tokens, + ) + else: + raise + + # Validate response structure + if ( + not response + or not hasattr(response, "choices") + or not response.choices + ): + raise ValueError("Invalid API response: missing choices") + + if not response.choices[0] or not hasattr( + response.choices[0], "message" + ): + raise ValueError("Invalid API response: missing message in choice") + + message = response.choices[0].message + content = message.content or "" + + # Successfully got a valid response + break + + except json.JSONDecodeError as e: + print( + f"\nโŒ JSON Decode Error in API response (attempt {attempt + 1}/{max_retries}):" + ) + print(f" Error: {e}") + print(f" Position: line {e.lineno}, column {e.colno}") + + if attempt < max_retries - 1: + print(f" โณ Retrying in {retry_delay} seconds...") + await asyncio.sleep(retry_delay) + retry_delay *= 2 # Exponential backoff + else: + print(" โŒ All retries exhausted") + raise + + except (ValueError, AttributeError, TypeError) as e: + print(f"\nโŒ API Response Error (attempt {attempt + 1}/{max_retries}):") + print(f" Error type: {type(e).__name__}") + print(f" Error: {e}") + + if attempt < max_retries - 1: + print(f" โณ Retrying in {retry_delay} seconds...") + await asyncio.sleep(retry_delay) + retry_delay *= 2 + else: + print(" โŒ All retries exhausted") + # Return empty response instead of crashing + return { + "content": "API error - unable to get valid response", + "tool_calls": [], + } + + except Exception as e: + print( + f"\nโŒ Unexpected API Error (attempt {attempt + 1}/{max_retries}):" + ) + print(f" Error type: {type(e).__name__}") + print(f" Error: {e}") + + if attempt < max_retries - 1: + print(f" โณ Retrying in {retry_delay} seconds...") + await asyncio.sleep(retry_delay) + retry_delay *= 2 + else: + print(" โŒ All retries exhausted") + raise + + tool_calls = [] + if message.tool_calls: + for tool_call in message.tool_calls: + try: + # Attempt to parse tool call arguments + parsed_input = json.loads(tool_call.function.arguments) + tool_calls.append( + { + "id": tool_call.id, + "name": tool_call.function.name, + "input": parsed_input, + } + ) + except json.JSONDecodeError as e: + # Detailed JSON parsing error logging + print("\nโŒ JSON Parsing Error in tool call:") + print(f" Tool: {tool_call.function.name}") + print(f" Error: {e}") + print(" Raw arguments (first 500 chars):") + print(f" {tool_call.function.arguments[:500]}") + print(f" Error position: line {e.lineno}, column {e.colno}") + print( + f" Problem at: ...{tool_call.function.arguments[max(0, e.pos-50):e.pos+50]}..." + ) + + # Attempt advanced JSON repair + repaired = self._repair_truncated_json( + tool_call.function.arguments, tool_call.function.name + ) + + if repaired: + print(" โœ… JSON repaired successfully") + tool_calls.append( + { + "id": tool_call.id, + "name": tool_call.function.name, + "input": repaired, + } + ) + else: + # Skip this tool call if repair failed + print(" โš ๏ธ Skipping unrepairable tool call") + continue + + return {"content": content, "tool_calls": tool_calls} + + # ==================== 5. Tools and Utility Methods (Utility Layer) ==================== + + def _validate_messages(self, messages: List[Dict]) -> List[Dict]: + """Validate and clean message list""" + valid_messages = [] + for msg in messages: + content = msg.get("content", "").strip() + if content: + valid_messages.append( + {"role": msg.get("role", "user"), "content": content} + ) + else: + self.logger.warning(f"Skipping empty message: {msg}") + return valid_messages + + def _prepare_mcp_tool_definitions(self) -> List[Dict[str, Any]]: + """Prepare tool definitions in Anthropic API standard format with filtering""" + # Get all available tools + all_tools = get_mcp_tools("code_implementation") + + # Define essential tools for code implementation + essential_tool_names = {"write_file", "search_code_references"} + + # Filter to only essential tools + filtered_tools = [ + tool for tool in all_tools if tool.get("name") in essential_tool_names + ] + + self.logger.info( + f"๐Ÿ”ง Tool filtering: {len(filtered_tools)}/{len(all_tools)} tools enabled" + ) + self.logger.info( + f" Available tools: {[tool.get('name') for tool in filtered_tools]}" + ) + + return filtered_tools + + # return get_mcp_tools("code_implementation") + + def _check_tool_results_for_errors(self, tool_results: List[Dict]) -> bool: + """Check tool results for errors with JSON repair capability""" + for result in tool_results: + try: + if hasattr(result["result"], "content") and result["result"].content: + content_text = result["result"].content[0].text + + # First attempt: try direct JSON parsing + try: + parsed_result = json.loads(content_text) + if parsed_result.get("status") == "error": + return True + except json.JSONDecodeError as e: + # JSON parsing failed - try to repair + print("\nโš ๏ธ JSON parsing failed in tool result check:") + print(f" Error: {e}") + print( + f" Position: line {e.lineno}, column {e.colno}, char {e.pos}" + ) + print(f" Content length: {len(content_text)} chars") + print(f" First 300 chars: {content_text[:300]}") + + # Attempt to repair the JSON + repaired = self._repair_truncated_json(content_text) + if repaired: + print(" โœ… Tool result JSON repaired successfully") + if repaired.get("status") == "error": + return True + else: + # Fallback: check for "error" keyword in text + if "error" in content_text.lower(): + return True + + elif isinstance(result["result"], str): + if "error" in result["result"].lower(): + return True + + except (AttributeError, IndexError) as e: + # Unexpected result structure + print(f"\nโš ๏ธ Unexpected result structure: {type(e).__name__}: {e}") + result_str = str(result["result"]) + if "error" in result_str.lower(): + return True + return False + + # ==================== 6. User Interaction and Feedback (Interaction Layer) ==================== + + def _generate_success_guidance(self, files_count: int) -> str: + """Generate concise success guidance for continuing implementation""" + return f"""โœ… File implementation completed successfully! + +๐Ÿ“Š **Progress Status:** {files_count} files implemented + +๐ŸŽฏ **Next Action:** Check if ALL files from the reproduction plan are implemented. + +โšก **Decision Process:** +1. **If ALL files are implemented:** Use `execute_python` or `execute_bash` to test the complete implementation, then respond "**implementation complete**" to end the conversation +2. **If MORE files need implementation:** Continue with dependency-aware workflow: + - **Start with `read_code_mem`** to understand existing implementations and dependencies + - **Optionally use `search_code_references`** for reference patterns (OPTIONAL - use for inspiration only, original paper specs take priority) + - **Then `write_file`** to implement the new component + - **Finally: Test** if needed + +๐Ÿ’ก **Key Point:** Always verify completion status before continuing with new file creation.""" + + def _generate_error_guidance(self) -> str: + """Generate error guidance for handling issues""" + return """โŒ Error detected during file implementation. + +๐Ÿ”ง **Action Required:** +1. Review the error details above +2. Fix the identified issue +3. **Check if ALL files from the reproduction plan are implemented:** + - **If YES:** Use `execute_python` or `execute_bash` to test the complete implementation, then respond "**implementation complete**" to end the conversation + - **If NO:** Continue with proper development cycle for next file: + - **Start with `read_code_mem`** to understand existing implementations + - **Optionally use `search_code_references`** for reference patterns (OPTIONAL - for inspiration only) + - **Then `write_file`** to implement properly + - **Test** if needed +4. Ensure proper error handling in future implementations + +๐Ÿ’ก **Remember:** Always verify if all planned files are implemented before continuing with new file creation.""" + + def _generate_no_tools_guidance(self, files_count: int) -> str: + """Generate concise guidance when no tools are called""" + return f"""โš ๏ธ No tool calls detected in your response. + +๐Ÿ“Š **Current Progress:** {files_count} files implemented + +๐Ÿšจ **Action Required:** You must use tools. **FIRST check if ALL files from the reproduction plan are implemented:** + +โšก **Decision Process:** +1. **If ALL files are implemented:** Use `execute_python` or `execute_bash` to test the complete implementation, then respond "**implementation complete**" to end the conversation +2. **If MORE files need implementation:** Follow the development cycle: + - **Start with `read_code_mem`** to understand existing implementations + - **Optionally use `search_code_references`** for reference patterns (OPTIONAL - for inspiration only) + - **Then `write_file`** to implement the new component + - **Finally: Test** if needed + +๐Ÿšจ **Critical:** Always verify completion status first, then use appropriate tools - not just explanations!""" + + def _compile_user_response(self, tool_results: List[Dict], guidance: str) -> str: + """Compile tool results and guidance into a single user response""" + response_parts = [] + + if tool_results: + response_parts.append("๐Ÿ”ง **Tool Execution Results:**") + for tool_result in tool_results: + tool_name = tool_result["tool_name"] + result_content = tool_result["result"] + response_parts.append( + f"```\nTool: {tool_name}\nResult: {result_content}\n```" + ) + + if guidance: + response_parts.append("\n" + guidance) + + return "\n\n".join(response_parts) + + # ==================== 7. Reporting and Output (Output Layer) ==================== + + async def _generate_pure_code_final_report_with_concise_agents( + self, + iterations: int, + elapsed_time: float, + code_agent: CodeImplementationAgent, + memory_agent: ConciseMemoryAgent, + ): + """Generate final report using concise agent statistics""" + try: + code_stats = code_agent.get_implementation_statistics() + memory_stats = memory_agent.get_memory_statistics( + code_stats["files_implemented_count"] + ) + + if self.mcp_agent: + history_result = await self.mcp_agent.call_tool( + "get_operation_history", {"last_n": 30} + ) + history_data = ( + json.loads(history_result) + if isinstance(history_result, str) + else history_result + ) + else: + history_data = {"total_operations": 0, "history": []} + + write_operations = 0 + files_created = [] + if "history" in history_data: + for item in history_data["history"]: + if item.get("action") == "write_file": + write_operations += 1 + file_path = item.get("details", {}).get("file_path", "unknown") + files_created.append(file_path) + + report = f""" +# Pure Code Implementation Completion Report (Write-File-Based Memory Mode) + +## Execution Summary +- Implementation iterations: {iterations} +- Total elapsed time: {elapsed_time:.2f} seconds +- Files implemented: {code_stats['total_files_implemented']} +- File write operations: {write_operations} +- Total MCP operations: {history_data.get('total_operations', 0)} + +## Read Tools Configuration +- Read tools enabled: {code_stats['read_tools_status']['read_tools_enabled']} +- Status: {code_stats['read_tools_status']['status']} +- Tools affected: {', '.join(code_stats['read_tools_status']['tools_affected'])} + +## Agent Performance +### Code Implementation Agent +- Files tracked: {code_stats['files_implemented_count']} +- Technical decisions: {code_stats['technical_decisions_count']} +- Constraints tracked: {code_stats['constraints_count']} +- Architecture notes: {code_stats['architecture_notes_count']} +- Dependency analysis performed: {code_stats['dependency_analysis_count']} +- Files read for dependencies: {code_stats['files_read_for_dependencies']} +- Last summary triggered at file count: {code_stats['last_summary_file_count']} + +### Concise Memory Agent (Write-File-Based) +- Last write_file detected: {memory_stats['last_write_file_detected']} +- Should clear memory next: {memory_stats['should_clear_memory_next']} +- Files implemented count: {memory_stats['implemented_files_tracked']} +- Current round: {memory_stats['current_round']} +- Concise mode active: {memory_stats['concise_mode_active']} +- Current round tool results: {memory_stats['current_round_tool_results']} +- Essential tools recorded: {memory_stats['essential_tools_recorded']} + +## Files Created +""" + for file_path in files_created[-20:]: + report += f"- {file_path}\n" + + if len(files_created) > 20: + report += f"... and {len(files_created) - 20} more files\n" + + report += """ +## Architecture Features +โœ… WRITE-FILE-BASED Memory Agent - Clear after each file generation +โœ… After write_file: Clear history โ†’ Keep system prompt + initial plan + tool results +โœ… Tool accumulation: read_code_mem, read_file, search_reference_code until next write_file +โœ… Clean memory cycle: write_file โ†’ clear โ†’ accumulate โ†’ write_file โ†’ clear +โœ… Essential tool recording with write_file detection +โœ… Specialized agent separation for clean code organization +โœ… MCP-compliant tool execution +โœ… Production-grade code with comprehensive type hints +โœ… Intelligent dependency analysis and file reading +โœ… Automated read_file usage for implementation context +โœ… Eliminates conversation clutter between file generations +โœ… Focused memory for efficient next file generation +""" + return report + + except Exception as e: + self.logger.error(f"Failed to generate final report: {e}") + return f"Failed to generate final report: {str(e)}" + + +async def main(): + """Main function for running the workflow""" + # Configure root logger carefully to avoid duplicates + root_logger = logging.getLogger() + if not root_logger.handlers: + handler = logging.StreamHandler() + formatter = logging.Formatter("%(levelname)s:%(name)s:%(message)s") + handler.setFormatter(formatter) + root_logger.addHandler(handler) + root_logger.setLevel(logging.INFO) + + workflow = CodeImplementationWorkflowWithIndex() + + print("=" * 60) + print("Code Implementation Workflow with UNIFIED Reference Indexer") + print("=" * 60) + print("Select mode:") + print("1. Test Code Reference Indexer Integration") + print("2. Run Full Implementation Workflow") + print("3. Run Implementation with Pure Code Mode") + print("4. Test Read Tools Configuration") + + # mode_choice = input("Enter choice (1-4, default: 3): ").strip() + + # For testing purposes, we'll run the test first + # if mode_choice == "4": + # print("Testing Read Tools Configuration...") + + # # Create a test workflow normally + # test_workflow = CodeImplementationWorkflow() + + # # Create a mock code agent for testing + # print("\n๐Ÿงช Testing with read tools DISABLED:") + # test_agent_disabled = CodeImplementationAgent(None, enable_read_tools=False) + # await test_agent_disabled.test_read_tools_configuration() + + # print("\n๐Ÿงช Testing with read tools ENABLED:") + # test_agent_enabled = CodeImplementationAgent(None, enable_read_tools=True) + # await test_agent_enabled.test_read_tools_configuration() + + # print("โœ… Read tools configuration testing completed!") + # return + + # print("Running Code Reference Indexer Integration Test...") + + test_success = True + if test_success: + print("\n" + "=" * 60) + print("๐ŸŽ‰ UNIFIED Code Reference Indexer Integration Test PASSED!") + print("๐Ÿ”ง Three-step process successfully merged into ONE tool") + print("=" * 60) + + # Ask if user wants to continue with actual workflow + print("\nContinuing with workflow execution...") + + plan_file = "/data2/bjdwhzzh/project-hku/Deepcode_collections/DeepCode/deepcode_lab/papers/54_only_code_gen/initial_plan.txt" + # plan_file = "/data2/bjdwhzzh/project-hku/Code-Agent2.0/Code-Agent/deepcode-mcp/agent_folders/papers/1/initial_plan.txt" + target_directory = "/data2/bjdwhzzh/project-hku/Deepcode_collections/DeepCode/deepcode_lab/papers/54_only_code_gen/" + print("Implementation Mode Selection:") + print("1. Pure Code Implementation Mode (Recommended)") + print("2. Iterative Implementation Mode") + + pure_code_mode = True + mode_name = "Pure Code Implementation Mode with Memory Agent Architecture + Code Reference Indexer" + print(f"Using: {mode_name}") + + # Configure read tools - modify this parameter to enable/disable read tools + enable_read_tools = ( + True # Set to False to disable read_file and read_code_mem tools + ) + read_tools_status = "ENABLED" if enable_read_tools else "DISABLED" + print(f"๐Ÿ”ง Read tools (read_file, read_code_mem): {read_tools_status}") + + # NOTE: To test without read tools, change the line above to: + # enable_read_tools = False + + result = await workflow.run_workflow( + plan_file, + target_directory=target_directory, + pure_code_mode=pure_code_mode, + enable_read_tools=enable_read_tools, + ) + + print("=" * 60) + print("Workflow Execution Results:") + print(f"Status: {result['status']}") + print(f"Mode: {mode_name}") + + if result["status"] == "success": + print(f"Code Directory: {result['code_directory']}") + print(f"MCP Architecture: {result.get('mcp_architecture', 'unknown')}") + print("Execution completed!") + else: + print(f"Error Message: {result['message']}") + + print("=" * 60) + print( + "โœ… Using Standard MCP Architecture with Memory Agent + Code Reference Indexer" + ) + + else: + print("\n" + "=" * 60) + print("โŒ Code Reference Indexer Integration Test FAILED!") + print("Please check the configuration and try again.") + print("=" * 60) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/DeepCode/workflows/codebase_index_workflow.py b/DeepCode/workflows/codebase_index_workflow.py new file mode 100644 index 00000000..b42a7ffc --- /dev/null +++ b/DeepCode/workflows/codebase_index_workflow.py @@ -0,0 +1,732 @@ +""" +Codebase Index Workflow + +This workflow integrates the functionality of run_indexer.py and code_indexer.py +to build intelligent relationships between existing codebase and target structure. + +Features: +- Extract target file structure from initial_plan.txt +- Analyze codebase and build indexes +- Generate relationship mappings and statistical reports +- Provide reference basis for code reproduction +""" + +import asyncio +import json +import logging +import os +import re +import sys +from pathlib import Path +from typing import Dict, Any, Optional +import yaml + +# Add tools directory to path +sys.path.append(str(Path(__file__).parent.parent / "tools")) + +from tools.code_indexer import CodeIndexer + + +class CodebaseIndexWorkflow: + """Codebase Index Workflow Class""" + + def __init__(self, logger=None): + """ + Initialize workflow + + Args: + logger: Logger instance + """ + self.logger = logger or self._setup_default_logger() + self.indexer = None + + def _setup_default_logger(self) -> logging.Logger: + """Setup default logger""" + logger = logging.getLogger("CodebaseIndexWorkflow") + logger.setLevel(logging.INFO) + + if not logger.handlers: + handler = logging.StreamHandler() + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + handler.setFormatter(formatter) + logger.addHandler(handler) + + return logger + + def extract_file_tree_from_plan(self, plan_content: str) -> Optional[str]: + """ + Extract file tree structure from initial_plan.txt content + + Args: + plan_content: Content of the initial_plan.txt file + + Returns: + Extracted file tree structure as string + """ + # Look for file structure section, specifically "## File Structure" format + file_structure_pattern = r"## File Structure[^\n]*\n```[^\n]*\n(.*?)\n```" + + match = re.search(file_structure_pattern, plan_content, re.DOTALL) + if match: + file_tree = match.group(1).strip() + lines = file_tree.split("\n") + + # Clean tree structure - remove empty lines and comments not part of structure + cleaned_lines = [] + for line in lines: + # Keep tree structure lines + if line.strip() and ( + any(char in line for char in ["โ”œโ”€โ”€", "โ””โ”€โ”€", "โ”‚"]) + or line.strip().endswith("/") + or "." in line.split("/")[-1] # has file extension + or line.strip().endswith(".py") + or line.strip().endswith(".txt") + or line.strip().endswith(".md") + or line.strip().endswith(".yaml") + ): + cleaned_lines.append(line) + + if len(cleaned_lines) >= 5: + file_tree = "\n".join(cleaned_lines) + self.logger.info( + f"๐Ÿ“Š Extracted file tree structure from ## File Structure section ({len(cleaned_lines)} lines)" + ) + return file_tree + + # Fallback: look for any code block containing project structure + code_block_patterns = [ + r"```[^\n]*\n(project/.*?(?:โ”œโ”€โ”€|โ””โ”€โ”€).*?)\n```", + r"```[^\n]*\n(src/.*?(?:โ”œโ”€โ”€|โ””โ”€โ”€).*?)\n```", + r"```[^\n]*\n(core/.*?(?:โ”œโ”€โ”€|โ””โ”€โ”€).*?)\n```", + r"```[^\n]*\n(.*?(?:โ”œโ”€โ”€|โ””โ”€โ”€).*?(?:\.py|\.txt|\.md|\.yaml).*?)\n```", + ] + + for pattern in code_block_patterns: + match = re.search(pattern, plan_content, re.DOTALL) + if match: + file_tree = match.group(1).strip() + lines = [line for line in file_tree.split("\n") if line.strip()] + if len(lines) >= 5: + self.logger.info( + f"๐Ÿ“Š Extracted file tree structure from code block ({len(lines)} lines)" + ) + return file_tree + + # Final fallback: extract file paths from file mentions and create basic structure + self.logger.warning( + "โš ๏ธ No standard file tree found, trying to extract from file mentions..." + ) + + # Search for file paths in backticks throughout the document + file_mentions = re.findall( + r"`([^`]*(?:\.py|\.txt|\.md|\.yaml|\.yml)[^`]*)`", plan_content + ) + + if file_mentions: + # Organize files into directory structure + dirs = set() + files_by_dir = {} + + for file_path in file_mentions: + file_path = file_path.strip() + if "/" in file_path: + dir_path = "/".join(file_path.split("/")[:-1]) + filename = file_path.split("/")[-1] + dirs.add(dir_path) + if dir_path not in files_by_dir: + files_by_dir[dir_path] = [] + files_by_dir[dir_path].append(filename) + else: + if "root" not in files_by_dir: + files_by_dir["root"] = [] + files_by_dir["root"].append(file_path) + + # Create tree structure + structure_lines = [] + + # Determine root directory name from common patterns + if any("src/" in f for f in file_mentions): + root_name = "src" + elif any("core/" in f for f in file_mentions): + root_name = "core" + elif any("lib/" in f for f in file_mentions): + root_name = "lib" + else: + root_name = "project" + structure_lines.append(f"{root_name}/") + + # Add directories and files + sorted_dirs = sorted(dirs) if dirs else [] + for i, dir_path in enumerate(sorted_dirs): + is_last_dir = i == len(sorted_dirs) - 1 + prefix = "โ””โ”€โ”€" if is_last_dir else "โ”œโ”€โ”€" + structure_lines.append(f"{prefix} {dir_path}/") + + if dir_path in files_by_dir: + files = sorted(files_by_dir[dir_path]) + for j, filename in enumerate(files): + is_last_file = j == len(files) - 1 + if is_last_dir: + file_prefix = " โ””โ”€โ”€" if is_last_file else " โ”œโ”€โ”€" + else: + file_prefix = "โ”‚ โ””โ”€โ”€" if is_last_file else "โ”‚ โ”œโ”€โ”€" + structure_lines.append(f"{file_prefix} {filename}") + + # Add root files (if any) + if "root" in files_by_dir: + root_files = sorted(files_by_dir["root"]) + for i, filename in enumerate(root_files): + is_last = (i == len(root_files) - 1) and not sorted_dirs + prefix = "โ””โ”€โ”€" if is_last else "โ”œโ”€โ”€" + structure_lines.append(f"{prefix} {filename}") + + if len(structure_lines) >= 3: + file_tree = "\n".join(structure_lines) + self.logger.info( + f"๐Ÿ“Š Generated file tree from file mentions ({len(structure_lines)} lines)" + ) + return file_tree + + # If no file tree found, return None + self.logger.warning("โš ๏ธ No file tree structure found in initial plan") + return None + + def load_target_structure_from_plan(self, plan_path: str) -> str: + """ + Load target structure from initial_plan.txt and extract file tree + + Args: + plan_path: Path to initial_plan.txt file + + Returns: + Extracted file tree structure + """ + try: + # Load complete plan content + with open(plan_path, "r", encoding="utf-8") as f: + plan_content = f.read() + + self.logger.info(f"๐Ÿ“„ Loaded initial plan ({len(plan_content)} characters)") + + # Extract file tree structure + file_tree = self.extract_file_tree_from_plan(plan_content) + + if file_tree: + self.logger.info( + "โœ… Successfully extracted file tree from initial plan" + ) + self.logger.info("๐Ÿ“‹ Extracted structure preview:") + # Show first few lines of extracted tree + preview_lines = file_tree.split("\n")[:8] + for line in preview_lines: + self.logger.info(f" {line}") + if len(file_tree.split("\n")) > 8: + self.logger.info( + f" ... {len(file_tree.split('\n')) - 8} more lines" + ) + return file_tree + else: + self.logger.warning("โš ๏ธ Unable to extract file tree from initial plan") + self.logger.info("๐Ÿ”„ Falling back to default target structure") + return self.get_default_target_structure() + + except Exception as e: + self.logger.error(f"โŒ Failed to load initial plan file {plan_path}: {e}") + self.logger.info("๐Ÿ”„ Falling back to default target structure") + return self.get_default_target_structure() + + def get_default_target_structure(self) -> str: + """Get default target structure""" + return """ +project/ +โ”œโ”€โ”€ src/ +โ”‚ โ”œโ”€โ”€ core/ +โ”‚ โ”‚ โ”œโ”€โ”€ gcn.py # GCN encoder +โ”‚ โ”‚ โ”œโ”€โ”€ diffusion.py # forward/reverse processes +โ”‚ โ”‚ โ”œโ”€โ”€ denoiser.py # denoising MLP +โ”‚ โ”‚ โ””โ”€โ”€ fusion.py # fusion combiner +โ”‚ โ”œโ”€โ”€ models/ # model wrapper classes +โ”‚ โ”‚ โ””โ”€โ”€ recdiff.py +โ”‚ โ”œโ”€โ”€ utils/ +โ”‚ โ”‚ โ”œโ”€โ”€ data.py # loading & preprocessing +โ”‚ โ”‚ โ”œโ”€โ”€ predictor.py # scoring functions +โ”‚ โ”‚ โ”œโ”€โ”€ loss.py # loss functions +โ”‚ โ”‚ โ”œโ”€โ”€ metrics.py # NDCG, Recall etc. +โ”‚ โ”‚ โ””โ”€โ”€ sched.py # beta/alpha schedule utils +โ”‚ โ””โ”€โ”€ configs/ +โ”‚ โ””โ”€โ”€ default.yaml # hyperparameters, paths +โ”œโ”€โ”€ tests/ +โ”‚ โ”œโ”€โ”€ test_gcn.py +โ”‚ โ”œโ”€โ”€ test_diffusion.py +โ”‚ โ”œโ”€โ”€ test_denoiser.py +โ”‚ โ”œโ”€โ”€ test_loss.py +โ”‚ โ””โ”€โ”€ test_pipeline.py +โ”œโ”€โ”€ docs/ +โ”‚ โ”œโ”€โ”€ architecture.md +โ”‚ โ”œโ”€โ”€ api_reference.md +โ”‚ โ””โ”€โ”€ README.md +โ”œโ”€โ”€ experiments/ +โ”‚ โ”œโ”€โ”€ run_experiment.py +โ”‚ โ””โ”€โ”€ notebooks/ +โ”‚ โ””โ”€โ”€ analysis.ipynb +โ”œโ”€โ”€ requirements.txt +โ””โ”€โ”€ setup.py +""" + + def load_or_create_indexer_config(self, paper_dir: str) -> Dict[str, Any]: + """ + Load or create indexer configuration + + Args: + paper_dir: Paper directory path + + Returns: + Configuration dictionary + """ + # Try to load existing configuration file + config_path = Path(__file__).parent.parent / "tools" / "indexer_config.yaml" + + try: + if config_path.exists(): + with open(config_path, "r", encoding="utf-8") as f: + config = yaml.safe_load(f) + + # Update path configuration to current paper directory + if "paths" not in config: + config["paths"] = {} + config["paths"]["code_base_path"] = os.path.join(paper_dir, "code_base") + config["paths"]["output_dir"] = os.path.join(paper_dir, "indexes") + + # Adjust performance settings for workflow + if "performance" in config: + config["performance"]["enable_concurrent_analysis"] = ( + False # Disable concurrency to avoid API limits + ) + if "debug" in config: + config["debug"]["verbose_output"] = True # Enable verbose output + if "llm" in config: + config["llm"]["request_delay"] = 0.5 # Increase request delay + + self.logger.info(f"Loaded configuration file: {config_path}") + return config + + except Exception as e: + self.logger.warning(f"Failed to load configuration file: {e}") + + # If loading fails, use default configuration + self.logger.info("Using default configuration") + default_config = { + "paths": { + "code_base_path": os.path.join(paper_dir, "code_base"), + "output_dir": os.path.join(paper_dir, "indexes"), + }, + "llm": { + "model_provider": "anthropic", + "max_tokens": 4000, + "temperature": 0.3, + "request_delay": 0.5, # Increase request delay + "max_retries": 3, + "retry_delay": 1.0, + }, + "file_analysis": { + "max_file_size": 1048576, # 1MB + "max_content_length": 3000, + "supported_extensions": [ + ".py", + ".js", + ".ts", + ".java", + ".cpp", + ".c", + ".h", + ".hpp", + ".cs", + ".php", + ".rb", + ".go", + ".rs", + ".scala", + ".kt", + ".yaml", + ".yml", + ".json", + ".xml", + ".toml", + ".md", + ".txt", + ], + "skip_directories": [ + "__pycache__", + "node_modules", + "target", + "build", + "dist", + "venv", + "env", + ".git", + ".svn", + "data", + "datasets", + ], + }, + "relationships": { + "min_confidence_score": 0.3, + "high_confidence_threshold": 0.7, + "relationship_types": { + "direct_match": 1.0, + "partial_match": 0.8, + "reference": 0.6, + "utility": 0.4, + }, + }, + "performance": { + "enable_concurrent_analysis": False, # Disable concurrency to avoid API limits + "max_concurrent_files": 3, + "enable_content_caching": True, + "max_cache_size": 100, + }, + "debug": { + "verbose_output": True, + "save_raw_responses": False, + "mock_llm_responses": False, + }, + "output": { + "generate_summary": True, + "generate_statistics": True, + "include_metadata": True, + "json_indent": 2, + }, + "logging": {"level": "INFO", "log_to_file": False}, + } + + return default_config + + async def run_indexing_workflow( + self, + paper_dir: str, + initial_plan_path: Optional[str] = None, + config_path: str = "mcp_agent.secrets.yaml", + ) -> Dict[str, Any]: + """ + Run the complete code indexing workflow + + Args: + paper_dir: Paper directory path + initial_plan_path: Initial plan file path (optional) + config_path: API configuration file path + + Returns: + Index result dictionary + """ + try: + self.logger.info("๐Ÿš€ Starting codebase index workflow...") + + # Step 1: Determine initial plan file path + if not initial_plan_path: + initial_plan_path = os.path.join(paper_dir, "initial_plan.txt") + + # Step 2: Load target structure + if os.path.exists(initial_plan_path): + self.logger.info( + f"๐Ÿ“ Loading target structure from {initial_plan_path}" + ) + target_structure = self.load_target_structure_from_plan( + initial_plan_path + ) + else: + self.logger.warning( + f"โš ๏ธ Initial plan file does not exist: {initial_plan_path}" + ) + self.logger.info("๐Ÿ“ Using default target structure") + target_structure = self.get_default_target_structure() + + # Step 3: Check codebase path + code_base_path = os.path.join(paper_dir, "code_base") + if not os.path.exists(code_base_path): + self.logger.error(f"โŒ Codebase path does not exist: {code_base_path}") + return { + "status": "error", + "message": f"Code base path does not exist: {code_base_path}", + "output_files": {}, + } + + # Step 4: Create output directory + output_dir = os.path.join(paper_dir, "indexes") + os.makedirs(output_dir, exist_ok=True) + + # Step 5: Load configuration + indexer_config = self.load_or_create_indexer_config(paper_dir) + + self.logger.info(f"๐Ÿ“ Codebase path: {code_base_path}") + self.logger.info(f"๐Ÿ“ค Output directory: {output_dir}") + + # Step 6: Create code indexer + self.indexer = CodeIndexer( + code_base_path=code_base_path, + target_structure=target_structure, + output_dir=output_dir, + config_path=config_path, + enable_pre_filtering=True, + ) + + # Apply configuration settings + self.indexer.indexer_config = indexer_config + + # Directly set configuration attributes to indexer + if "file_analysis" in indexer_config: + file_config = indexer_config["file_analysis"] + self.indexer.supported_extensions = set( + file_config.get( + "supported_extensions", self.indexer.supported_extensions + ) + ) + self.indexer.skip_directories = set( + file_config.get("skip_directories", self.indexer.skip_directories) + ) + self.indexer.max_file_size = file_config.get( + "max_file_size", self.indexer.max_file_size + ) + self.indexer.max_content_length = file_config.get( + "max_content_length", self.indexer.max_content_length + ) + + if "llm" in indexer_config: + llm_config = indexer_config["llm"] + self.indexer.model_provider = llm_config.get( + "model_provider", self.indexer.model_provider + ) + self.indexer.llm_max_tokens = llm_config.get( + "max_tokens", self.indexer.llm_max_tokens + ) + self.indexer.llm_temperature = llm_config.get( + "temperature", self.indexer.llm_temperature + ) + self.indexer.request_delay = llm_config.get( + "request_delay", self.indexer.request_delay + ) + self.indexer.max_retries = llm_config.get( + "max_retries", self.indexer.max_retries + ) + self.indexer.retry_delay = llm_config.get( + "retry_delay", self.indexer.retry_delay + ) + + if "relationships" in indexer_config: + rel_config = indexer_config["relationships"] + self.indexer.min_confidence_score = rel_config.get( + "min_confidence_score", self.indexer.min_confidence_score + ) + self.indexer.high_confidence_threshold = rel_config.get( + "high_confidence_threshold", self.indexer.high_confidence_threshold + ) + self.indexer.relationship_types = rel_config.get( + "relationship_types", self.indexer.relationship_types + ) + + if "performance" in indexer_config: + perf_config = indexer_config["performance"] + self.indexer.enable_concurrent_analysis = perf_config.get( + "enable_concurrent_analysis", + self.indexer.enable_concurrent_analysis, + ) + self.indexer.max_concurrent_files = perf_config.get( + "max_concurrent_files", self.indexer.max_concurrent_files + ) + self.indexer.enable_content_caching = perf_config.get( + "enable_content_caching", self.indexer.enable_content_caching + ) + self.indexer.max_cache_size = perf_config.get( + "max_cache_size", self.indexer.max_cache_size + ) + + if "debug" in indexer_config: + debug_config = indexer_config["debug"] + self.indexer.verbose_output = debug_config.get( + "verbose_output", self.indexer.verbose_output + ) + self.indexer.save_raw_responses = debug_config.get( + "save_raw_responses", self.indexer.save_raw_responses + ) + self.indexer.mock_llm_responses = debug_config.get( + "mock_llm_responses", self.indexer.mock_llm_responses + ) + + if "output" in indexer_config: + output_config = indexer_config["output"] + self.indexer.generate_summary = output_config.get( + "generate_summary", self.indexer.generate_summary + ) + self.indexer.generate_statistics = output_config.get( + "generate_statistics", self.indexer.generate_statistics + ) + self.indexer.include_metadata = output_config.get( + "include_metadata", self.indexer.include_metadata + ) + + self.logger.info("๐Ÿ”ง Indexer configuration completed") + self.logger.info(f"๐Ÿค– Model provider: {self.indexer.model_provider}") + self.logger.info( + f"โšก Concurrent analysis: {'Enabled' if self.indexer.enable_concurrent_analysis else 'Disabled'}" + ) + self.logger.info( + f"๐Ÿ—„๏ธ Content caching: {'Enabled' if self.indexer.enable_content_caching else 'Disabled'}" + ) + self.logger.info( + f"๐Ÿ” Pre-filtering: {'Enabled' if self.indexer.enable_pre_filtering else 'Disabled'}" + ) + + self.logger.info("=" * 60) + self.logger.info("๐Ÿš€ Starting code indexing process...") + + # Step 7: Build all indexes + output_files = await self.indexer.build_all_indexes() + + # Step 8: Generate summary report + if output_files: + summary_report = self.indexer.generate_summary_report(output_files) + + self.logger.info("=" * 60) + self.logger.info("โœ… Indexing completed successfully!") + self.logger.info(f"๐Ÿ“Š Processed {len(output_files)} repositories") + self.logger.info("๐Ÿ“ Generated index files:") + for repo_name, file_path in output_files.items(): + self.logger.info(f" ๐Ÿ“„ {repo_name}: {file_path}") + self.logger.info(f"๐Ÿ“‹ Summary report: {summary_report}") + + # Statistics (if enabled) + if self.indexer.generate_statistics: + self.logger.info("\n๐Ÿ“ˆ Processing statistics:") + total_relationships = 0 + high_confidence_relationships = 0 + + for file_path in output_files.values(): + try: + with open(file_path, "r", encoding="utf-8") as f: + index_data = json.load(f) + relationships = index_data.get("relationships", []) + total_relationships += len(relationships) + high_confidence_relationships += len( + [ + r + for r in relationships + if r.get("confidence_score", 0) + > self.indexer.high_confidence_threshold + ] + ) + except Exception as e: + self.logger.warning( + f" โš ๏ธ Unable to load statistics from {file_path}: {e}" + ) + + self.logger.info( + f" ๐Ÿ”— Total relationships found: {total_relationships}" + ) + self.logger.info( + f" โญ High confidence relationships: {high_confidence_relationships}" + ) + self.logger.info( + f" ๐Ÿ“Š Average relationships per repository: {total_relationships / len(output_files) if output_files else 0:.1f}" + ) + + self.logger.info("\n๐ŸŽ‰ Code indexing process completed successfully!") + + return { + "status": "success", + "message": f"Successfully indexed {len(output_files)} repositories", + "output_files": output_files, + "summary_report": summary_report, + "statistics": { + "total_repositories": len(output_files), + "total_relationships": total_relationships, + "high_confidence_relationships": high_confidence_relationships, + } + if self.indexer.generate_statistics + else None, + } + else: + self.logger.warning("โš ๏ธ No index files generated") + return { + "status": "warning", + "message": "No index files were generated", + "output_files": {}, + } + + except Exception as e: + self.logger.error(f"โŒ Index workflow failed: {e}") + # If there are detailed error messages, log them + import traceback + + self.logger.error(f"Detailed error information: {traceback.format_exc()}") + return {"status": "error", "message": str(e), "output_files": {}} + + def print_banner(self): + """Print application banner""" + banner = """ +โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•— +โ•‘ ๐Ÿ” Codebase Index Workflow v1.0 โ•‘ +โ•‘ Intelligent Code Relationship Analysis Tool โ•‘ +โ• โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ฃ +โ•‘ ๐Ÿ“ Analyzes existing codebases โ•‘ +โ•‘ ๐Ÿ”— Builds intelligent relationships with target structure โ•‘ +โ•‘ ๐Ÿค– Powered by LLM analysis โ•‘ +โ•‘ ๐Ÿ“Š Generates detailed JSON indexes โ•‘ +โ•‘ ๐ŸŽฏ Provides reference for code reproduction โ•‘ +โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + """ + print(banner) + + +# Convenience function for direct workflow invocation +async def run_codebase_indexing( + paper_dir: str, + initial_plan_path: Optional[str] = None, + config_path: str = "mcp_agent.secrets.yaml", + logger=None, +) -> Dict[str, Any]: + """ + Convenience function to run codebase indexing + + Args: + paper_dir: Paper directory path + initial_plan_path: Initial plan file path (optional) + config_path: API configuration file path + logger: Logger instance (optional) + + Returns: + Index result dictionary + """ + workflow = CodebaseIndexWorkflow(logger=logger) + workflow.print_banner() + + return await workflow.run_indexing_workflow( + paper_dir=paper_dir, + initial_plan_path=initial_plan_path, + config_path=config_path, + ) + + +# Main function for testing +async def main(): + """Main function for testing workflow""" + import logging + + # Setup logging + logging.basicConfig(level=logging.INFO) + logger = logging.getLogger(__name__) + + # Test parameters + paper_dir = "./deepcode_lab/papers/1" + initial_plan_path = os.path.join(paper_dir, "initial_plan.txt") + + # Run workflow + result = await run_codebase_indexing( + paper_dir=paper_dir, initial_plan_path=initial_plan_path, logger=logger + ) + + logger.info(f"Index result: {result}") + + +if __name__ == "__main__": + asyncio.run(main())