project.justfile

## Add your own just recipes here. This is imported by the main justfile.

# NOTE: The old single-mode analyze-experimental recipe has been replaced by a dual-mode version
# in the main justfile. The new version (defined in justfile, not project.justfile) supports both
# absolute and relative analysis modes. See main justfile for the updated recipe.

# DEPRECATED: Old single-mode experimental analysis (replaced by dual-mode version in main justfile)
# Analyze experimental plate data (statistical processing + visualization + clustering)
# [group('analysis')]
# analyze-experimental-legacy DATA_DIR="data/experimental/plate_designs_v10_maxprooptblock_long__results" OUTPUT_DIR="outputs/experimental_analysis":
#     #!/usr/bin/env bash
# #     set -euo pipefail
#     ... (commented out - see git history for full recipe)

# Perform hierarchical clustering on replicate data
[group('analysis')]
cluster-replicates DATA_FILE="outputs/experimental_analysis/replicate_statistics.tsv" OUTPUT_DIR="outputs/clustering" FEATURE_TYPE="growth" N_CLUSTERS="6" SOURCE_DATA_ID="":
    #!/usr/bin/env bash
    set -euo pipefail

    DATA_FILE="{{DATA_FILE}}"
    OUTPUT_DIR="{{OUTPUT_DIR}}"
    FEATURE_TYPE="{{FEATURE_TYPE}}"
    N_CLUSTERS="{{N_CLUSTERS}}"
    SOURCE_DATA_ID="{{SOURCE_DATA_ID}}"

    echo "================================================================================"
    echo "Hierarchical Clustering Analysis of Replicates"
    echo "================================================================================"
    echo ""
    echo "📊 Data file: ${DATA_FILE}"
    echo "📁 Output directory: ${OUTPUT_DIR}"
    echo "🔬 Feature type: ${FEATURE_TYPE}"
    echo "📊 Number of clusters: ${N_CLUSTERS}"
    if [ -n "${SOURCE_DATA_ID}" ]; then
        echo "🏷️  Source data ID: ${SOURCE_DATA_ID}"
    fi
    echo ""

    # Check if data file exists
    if [ ! -f "${DATA_FILE}" ]; then
        echo "❌ Error: Data file not found: ${DATA_FILE}"
        echo ""
        echo "Run 'just analyze-experimental' first to generate replicate statistics."
        exit 1
    fi

    # Run clustering analysis (with optional source data ID)
    if [ -n "${SOURCE_DATA_ID}" ]; then
        uv run python scripts/cluster_heatmap_replicates.py "${DATA_FILE}" \
            --output-dir "${OUTPUT_DIR}" \
            --source-data-id "${SOURCE_DATA_ID}" \
            --feature-type "${FEATURE_TYPE}" \
            --n-clusters "${N_CLUSTERS}"
    else
        uv run python scripts/cluster_heatmap_replicates.py "${DATA_FILE}" \
            --output-dir "${OUTPUT_DIR}" \
            --feature-type "${FEATURE_TYPE}" \
            --n-clusters "${N_CLUSTERS}"
    fi

    echo ""
    echo "================================================================================"
    echo "✅ Clustering Complete!"
    echo "================================================================================"
    echo ""
    echo "📊 Results saved to: ${OUTPUT_DIR}/"
    echo ""
    echo "Generated files:"
    echo "  - clustered_heatmap_${FEATURE_TYPE}.pdf (main visualization)"
    echo "  - clustered_heatmap_${FEATURE_TYPE}.png (for easy viewing)"
    echo "  - cluster_assignments_${FEATURE_TYPE}.csv (which cluster each replicate belongs to)"
    echo "  - cluster_summary_${FEATURE_TYPE}.pdf (cluster sizes and OD600 distributions)"
    echo "  - cluster_descriptions_${FEATURE_TYPE}.txt (detailed text summary)"
    echo ""
    echo "View heatmap:"
    echo "  open ${OUTPUT_DIR}/clustered_heatmap_${FEATURE_TYPE}.png"
    echo ""

# Run complete clustering analysis (both growth and ingredient-based)
[group('analysis')]
cluster-all DATA_FILE="outputs/experimental_analysis/replicate_statistics.tsv" OUTPUT_DIR="outputs/clustering":
    #!/usr/bin/env bash
    set -euo pipefail

    echo "================================================================================"
    echo "Complete Clustering Analysis Pipeline"
    echo "================================================================================"
    echo ""
    echo "Running both growth-based and ingredient-based clustering..."
    echo ""

    # Growth-based clustering
    echo "════════════════════════════════════════════════════════════════════════════════"
    echo "1. Growth-Based Clustering (OD600 trajectories)"
    echo "════════════════════════════════════════════════════════════════════════════════"
    echo ""
    just cluster-replicates "{{DATA_FILE}}" "{{OUTPUT_DIR}}" "growth" "6"

    echo ""
    echo "════════════════════════════════════════════════════════════════════════════════"
    echo "2. Ingredient-Based Clustering (media composition)"
    echo "════════════════════════════════════════════════════════════════════════════════"
    echo ""
    just cluster-replicates "{{DATA_FILE}}" "{{OUTPUT_DIR}}" "ingredients" "6"

    echo ""
    echo "════════════════════════════════════════════════════════════════════════════════"
    echo "3. Combined Clustering (growth + ingredients)"
    echo "════════════════════════════════════════════════════════════════════════════════"
    echo ""
    just cluster-replicates "{{DATA_FILE}}" "{{OUTPUT_DIR}}" "combined" "6"

    echo ""
    echo "================================================================================"
    echo "✅ All Clustering Complete!"
    echo "================================================================================"
    echo ""
    echo "📊 All results in: {{OUTPUT_DIR}}/"
    echo ""
    echo "Key insights:"
    echo "  - Growth clusters show which conditions have similar performance"
    echo "  - Ingredient clusters show which media are similar in composition"
    echo "  - Combined clusters integrate both perspectives"
    echo ""
    echo "View summary:"
    echo "  cat {{OUTPUT_DIR}}/CLUSTERING_ANALYSIS_SUMMARY.md"
    echo "  open {{OUTPUT_DIR}}/clustered_heatmap_growth.png"
    echo ""

# Generate a new LHS design version (creates all required output files)
[group('project management')]
generate-lhs-design VERSION SCRIPT_NAME:
    #!/usr/bin/env bash
    set -euo pipefail

    VERSION_TAG="{{VERSION}}"
    SCRIPT="scripts/{{SCRIPT_NAME}}"
    DATE=$(date +"%Y-%m-%d %H:%M:%S")

    echo "=== Generating LHS Design ${VERSION_TAG} ==="
    echo ""
    echo "📋 Using script: ${SCRIPT}"
    echo "🕐 Generated: ${DATE}"
    echo ""

    # Check if script exists
    if [ ! -f "${SCRIPT}" ]; then
        echo "❌ Error: Script not found: ${SCRIPT}"
        echo ""
        echo "Available LHS generation scripts:"
        ls -1 scripts/generate_lhs*.py scripts/*lhs*.py 2>/dev/null || echo "   No scripts found"
        exit 1
    fi

    # Run the generation script
    echo "🔧 Running generation script..."
    uv run python "${SCRIPT}"

    # Check if output directory was created
    OUTPUT_DIR="data/designs/MP_latinhypercube/plate_designs_${VERSION_TAG}"
    if [ ! -d "${OUTPUT_DIR}" ]; then
        echo "❌ Error: Output directory not created: ${OUTPUT_DIR}"
        exit 1
    fi

    echo ""
    echo "✅ Design generation complete!"
    echo ""
    echo "📊 Verifying output files..."
    FILE_COUNT=$(ls -1 "${OUTPUT_DIR}" | wc -l | tr -d ' ')
    echo "   Files created: ${FILE_COUNT}"

    # Check for required files
    echo ""
    echo "📋 Checking standard output files..."

    REQUIRED_FILES=(
        "README_${VERSION_TAG}.md"
        "DESIGN_CRITERIA_${VERSION_TAG}.md"
        "CONCENTRATION_SUMMARY_${VERSION_TAG}.md"
        "DELIVERY_SUMMARY_${VERSION_TAG}.md"
        "pipetting_protocol_${VERSION_TAG}.csv"
        "transfer_protocol_${VERSION_TAG}.csv"
        "complete_formulations_${VERSION_TAG}.json"
        "lhs_scatterplot_matrix_${VERSION_TAG}.png"
        "lhs_scatterplot_matrix_${VERSION_TAG}.pdf"
    )

    MISSING_COUNT=0
    for file in "${REQUIRED_FILES[@]}"; do
        if [ -f "${OUTPUT_DIR}/${file}" ]; then
            echo "   ✓ ${file}"
        else
            echo "   ✗ MISSING: ${file}"
            MISSING_COUNT=$((MISSING_COUNT + 1))
        fi
    done

    if [ $MISSING_COUNT -gt 0 ]; then
        echo ""
        echo "⚠️  Warning: ${MISSING_COUNT} required files missing!"
        echo "   Run completion script if needed:"
        echo "   uv run python scripts/complete_${VERSION_TAG}_outputs.py"
    fi

    echo ""
    echo "📦 Next steps:"
    echo "   1. Review output files in: ${OUTPUT_DIR}"
    echo "   2. Package design: just package-lhs-design ${VERSION_TAG}"
    echo ""

# Create tarball of LHS design with proper naming (includes date stamp)
[group('project management')]
package-lhs-design VERSION:
    #!/usr/bin/env bash
    set -euo pipefail

    # Variables
    DESIGN_VERSION="{{VERSION}}"
    DATE=$(date +"%Y%m%d")
    TARBALL_NAME="MP_latinhypercube_${DESIGN_VERSION}_${DATE}.tar.gz"
    DESIGN_DIR="data/designs/MP_latinhypercube"

    echo "=== Packaging LHS Design ${DESIGN_VERSION} ==="
    echo "📋 Date: ${DATE}"
    echo ""

    # Check if design directory exists
    if [ ! -d "${DESIGN_DIR}/plate_designs_${DESIGN_VERSION}" ]; then
        echo "❌ Error: Design directory not found: ${DESIGN_DIR}/plate_designs_${DESIGN_VERSION}"
        echo ""
        echo "Available designs:"
        ls -1d ${DESIGN_DIR}/plate_designs_* 2>/dev/null || echo "   No designs found"
        exit 1
    fi

    # Navigate to design directory
    cd "${DESIGN_DIR}"

    # Create tarball directly (no temp directory needed)
    echo "📦 Creating tarball: ${TARBALL_NAME}"
    tar -czf "${TARBALL_NAME}" \
        plate_designs_${DESIGN_VERSION}/ \
        MP_latinhypercube_list_ranges_${DESIGN_VERSION}.txt 2>/dev/null || \
    tar -czf "${TARBALL_NAME}" \
        plate_designs_${DESIGN_VERSION}/

    # Calculate size
    SIZE=$(du -h "${TARBALL_NAME}" | cut -f1)
    FILE_COUNT=$(tar -tzf "${TARBALL_NAME}" | wc -l | tr -d ' ')

    echo ""
    echo "✅ Package created successfully!"
    echo ""
    echo "   File: ${TARBALL_NAME}"
    echo "   Size: ${SIZE}"
    echo "   Files: ${FILE_COUNT}"
    echo "   Location: ${DESIGN_DIR}/${TARBALL_NAME}"
    echo ""
    echo "📋 Contents:"
    tar -tzf "${TARBALL_NAME}" | head -20
    if [ ${FILE_COUNT} -gt 20 ]; then
        echo "   ... and $((FILE_COUNT - 20)) more files"
    fi
    echo ""
    echo "To extract:"
    echo "   cd ${DESIGN_DIR}"
    echo "   tar -xzf ${TARBALL_NAME}"
    echo ""

# Initialize provenance tracking tables in database
[group('database')]
init-provenance DB_PATH="data/processed/microgrow.duckdb":
    uv run python scripts/migrate_provenance_schema.py {{DB_PATH}}

# Query provenance data from database
[group('database')]
query-provenance QUERY:
    @echo "Querying provenance data..."
    uv run python -c "import duckdb; conn = duckdb.connect('data/processed/microgrow.duckdb'); print(conn.execute('{{QUERY}}').fetchdf())"

# Show recent agent sessions
[group('database')]
show-sessions LIMIT="10":
    @just query-provenance "SELECT session_id, agent_type, query, duration_ms, success FROM provenance_sessions ORDER BY created_at DESC LIMIT {{LIMIT}}"

# Show provenance statistics
[group('database')]
provenance-stats:
    @echo "\n=== Provenance Statistics ==="
    @just query-provenance "SELECT 'Total Sessions' as metric, COUNT(*) as count FROM provenance_sessions UNION ALL SELECT 'Total Events', COUNT(*) FROM provenance_events UNION ALL SELECT 'Total Tool Calls', COUNT(*) FROM provenance_tool_calls"

# ============== Research Auditor Recipes ==============

# Audit tool output (file validation, schema check)
[group('audit')]
audit-tool TOOL_NAME OUTPUT_PATH EXPECTED_COLUMNS="":
    #!/usr/bin/env bash
    set -euo pipefail

    echo "================================================================================"
    echo "Research Audit: Tool Output"
    echo "================================================================================"
    echo ""
    echo "🔍 Tool: {{TOOL_NAME}}"
    echo "📄 Output: {{OUTPUT_PATH}}"

    if [ -n "{{EXPECTED_COLUMNS}}" ]; then
        echo "📋 Expected columns: {{EXPECTED_COLUMNS}}"
        uv run python scripts/run_research_audit.py \
            --scope tool \
            --tool-name "{{TOOL_NAME}}" \
            --output-path "{{OUTPUT_PATH}}" \
            --expected-columns {{EXPECTED_COLUMNS}}
    else
        uv run python scripts/run_research_audit.py \
            --scope tool \
            --tool-name "{{TOOL_NAME}}" \
            --output-path "{{OUTPUT_PATH}}"
    fi

# Audit agent execution (output file check, provenance trace)
[group('audit')]
audit-agent SESSION_ID EXPECTED_OUTPUTS="" OUTPUT_DIR="":
    #!/usr/bin/env bash
    set -euo pipefail

    echo "================================================================================"
    echo "Research Audit: Agent Execution"
    echo "================================================================================"
    echo ""
    echo "🤖 Session: {{SESSION_ID}}"

    if [ -n "{{EXPECTED_OUTPUTS}}" ] && [ -n "{{OUTPUT_DIR}}" ]; then
        echo "📋 Expected outputs: {{EXPECTED_OUTPUTS}}"
        echo "📁 Output directory: {{OUTPUT_DIR}}"
        uv run python scripts/run_research_audit.py \
            --scope agent \
            --session-id "{{SESSION_ID}}" \
            --expected-outputs {{EXPECTED_OUTPUTS}} \
            --agent-output-dir "{{OUTPUT_DIR}}"
    else
        uv run python scripts/run_research_audit.py \
            --scope agent \
            --session-id "{{SESSION_ID}}"
    fi

# Audit multi-agent workflow (checkpoint aggregation, hierarchy analysis)
[group('audit')]
audit-workflow WORKFLOW_ID ROOT_SESSION_ID:
    #!/usr/bin/env bash
    set -euo pipefail

    echo "================================================================================"
    echo "Research Audit: Workflow"
    echo "================================================================================"
    echo ""
    echo "🔗 Workflow: {{WORKFLOW_ID}}"
    echo "📊 Root session: {{ROOT_SESSION_ID}}"
    echo ""

    uv run python scripts/run_research_audit.py \
        --scope workflow \
        --workflow-id "{{WORKFLOW_ID}}" \
        --root-session-id "{{ROOT_SESSION_ID}}"

# Audit end-to-end pipeline (checksum verification, data integrity)
[group('audit')]
audit-pipeline PIPELINE_ID INPUT_DIR:
    #!/usr/bin/env bash
    set -euo pipefail

    echo "================================================================================"
    echo "Research Audit: Pipeline"
    echo "================================================================================"
    echo ""
    echo "⚙️  Pipeline: {{PIPELINE_ID}}"
    echo "📂 Input directory: {{INPUT_DIR}}"
    echo ""

    uv run python scripts/run_research_audit.py \
        --scope pipeline \
        --pipeline-id "{{PIPELINE_ID}}" \
        --input-dir "{{INPUT_DIR}}"

# Example: Audit experimental analysis pipeline end-to-end
[group('audit')]
audit-experimental-pipeline SOURCE_DATA_ID:
    #!/usr/bin/env bash
    set -euo pipefail

    INPUT_DIR="data/experimental/{{SOURCE_DATA_ID}}"
    PIPELINE_ID="experimental-analysis-{{SOURCE_DATA_ID}}"

    echo "================================================================================"
    echo "Research Audit: Experimental Analysis Pipeline"
    echo "================================================================================"
    echo ""
    echo "📊 Source data: {{SOURCE_DATA_ID}}"
    echo "📂 Input directory: ${INPUT_DIR}"
    echo ""

    just audit-pipeline "${PIPELINE_ID}" "${INPUT_DIR}"

    echo ""
    echo "✅ Pipeline audit complete"
    echo "📄 View full report: .claude/audits/AUDIT_REPORT.md"

# ============================================================================
# AUDIT-ENABLED WORKFLOWS
# ============================================================================

# Generate MP_plus v10 with full audit trail (demonstration)
[group('audit')]
generate-mp-plus-v10-audited:
    #!/usr/bin/env bash
    set -euo pipefail

    echo "================================================================================"
    echo "MP_plus v10 Generation with ResearchAuditor - DEMONSTRATION"
    echo "================================================================================"
    echo ""
    echo "This demonstrates audit integration in multi-agent workflows."
    echo "Creates simplified MP_plus v10 recommendations with full audit trails."
    echo ""

    uv run python scripts/generate_mp_plus_v10_with_audit.py

    echo ""
    echo "✅ Generation complete with audit trail"
    echo "📁 Outputs:"
    echo "   - data/designs/MP_plus/MP_plus_v10_audited/"
    echo "   - data/designs/MP_plus/MP_plus_v10_audited/audits/{workflow_id}/"
    echo "📄 Audit reports:"
    echo "   - audit_report.json (machine-readable)"
    echo "   - AUDIT_REPORT.md (human-readable)"
    echo "   - checkpoints/ (4 checkpoint files)"

# Run experimental analysis with ResearchAuditor monitoring
[group('audit')]
analyze-experimental-audited DATA_DIR OUTPUT_BASE="outputs":
    #!/usr/bin/env bash
    set -euo pipefail

    DATA_DIR="{{DATA_DIR}}"
    OUTPUT_BASE="{{OUTPUT_BASE}}"

    echo "================================================================================"
    echo "Experimental Analysis with ResearchAuditor"
    echo "================================================================================"
    echo ""
    echo "📊 Data directory: ${DATA_DIR}"
    echo "📁 Output base: ${OUTPUT_BASE}"
    echo ""

    uv run python scripts/run_dual_analysis_with_audit.py "${DATA_DIR}" \
        --output-base "${OUTPUT_BASE}"

    echo ""
    echo "✅ Analysis complete with audit checkpoints"
    echo "📄 Audit reports:"
    echo "   - ${OUTPUT_BASE}/.claude/audits/{workflow_id}/audit_report.json"
    echo "   - ${OUTPUT_BASE}/.claude/audits/{workflow_id}/AUDIT_REPORT.md"
    echo "   - ${OUTPUT_BASE}/.claude/audits/{workflow_id}/checkpoints/"

# Run experimental analysis with audit (disable optional steps)
[group('audit')]
analyze-experimental-audited-minimal DATA_DIR OUTPUT_BASE="outputs":
    #!/usr/bin/env bash
    set -euo pipefail

    DATA_DIR="{{DATA_DIR}}"
    OUTPUT_BASE="{{OUTPUT_BASE}}"

    echo "================================================================================"
    echo "Experimental Analysis with ResearchAuditor (Minimal)"
    echo "================================================================================"
    echo ""

    uv run python scripts/run_dual_analysis_with_audit.py "${DATA_DIR}" \
        --output-base "${OUTPUT_BASE}" \
        --disable-response-surfaces \
        --disable-optimization-report

    echo ""
    echo "✅ Minimal analysis complete with audit"