Add web profile endpoint discovery

README.md

@@ -56,6 +56,7 @@ Instead of manually running separate commands and collecting notes from differen
 * collect HTTP status, headers, redirects, page titles, and technology hints
 * collect TLS certificate metadata for HTTPS services
 * query common DNS records
+* run a richer web reconnaissance workflow from the `web` scan profile
 * generate Markdown and JSON reports
 * highlight interesting signals for follow-up review
 
@@ -75,6 +76,7 @@ ActiveRecon currently supports:
 | HTTP      | Status codes, titles, redirects, headers, technology hints |
 | TLS       | TLS version, cipher, certificate metadata                  |
 | DNS       | A, MX, and TXT lookups                                     |
+| Web       | Endpoint discovery from HTML, headers, JavaScript, and safe well-known paths |
 | Reporting | Markdown and JSON output                                   |
 | Safety    | Scope guard, dry-run mode, doctor checks                   |
 | Analysis  | Interesting signals for follow-up review                   |
@@ -153,6 +155,7 @@ Generated Markdown reports include sections such as:
 ## Scan Information
 ## Port Scan Results
 ## HTTP Analysis
+## Endpoint Discovery
 ## TLS Analysis
 ## DNS Analysis
 ## Interesting Signals
@@ -254,6 +257,25 @@ scan_profiles:
   standard: "-Pn -n -sT -sV -sC -T3"
   full: "-Pn -n -sT -p- -sV -sC -T4"
   udp: "-Pn -n -sU --top-ports 100 -sC --script-timeout 5m"
+
+web_recon:
+  enabled_profiles:
+    - web
+  endpoint_probe_limit: 50
+  fetch_javascript: true
+  same_origin_only: true
+  well_known_paths:
+    - /robots.txt
+    - /sitemap.xml
+    - /.well-known/security.txt
+    - /api
+    - /rest
+    - /ftp
+    - /admin
+    - /login
+    - /debug
+    - /swagger
+    - /api-docs
 ```
 
 ---
@@ -310,6 +332,8 @@ Attention
 
 Markdown reports use the heading `Interesting Signals`. JSON output keeps the `Attention` key for compatibility.
 
+When the `web` profile is used, reports also include `Endpoint Discovery`.
+
 ---
 
 ## Project Structure
@@ -324,6 +348,7 @@ ActiveRecon/
 |       |-- config_loader.py
 |       |-- dns_analysis.py
 |       |-- doctor.py
+|       |-- endpoint_discovery.py
 |       |-- http_enum.py
 |       |-- json_report.py
 |       |-- nmap_scan.py

activerecon/main.py

@@ -12,6 +12,7 @@
 from .modules.json_report import generate_json_report
 from .modules.config_loader import load_config
 from .modules.doctor import run_doctor
+from .modules.endpoint_discovery import discover_endpoints
 from .modules.risk_analysis import generate_attention_findings
 from .modules.scope_guard import is_target_in_scope
 from .modules.tls_analysis import analyze_tls
@@ -81,6 +82,12 @@ def _dns_skip_result():
     }
 
 
+def _web_recon_enabled(config, scan_profile):
+    web_recon = config.get("web_recon", {}) if isinstance(config, dict) else {}
+    enabled_profiles = web_recon.get("enabled_profiles", [])
+    return scan_profile in enabled_profiles
+
+
 def _safe_report_name(target):
     safe_name = re.sub(r"[^A-Za-z0-9_.-]+", "_", target).strip("._-")
     return safe_name or "target"
@@ -219,6 +226,14 @@ def main():
         logging.error(f"Error during TLS analysis: {e}")
         results["TLS Analysis"] = {"error": f"TLS analysis failed: {e}"}
 
+    if _web_recon_enabled(config, chosen_profile):
+        try:
+            logging.info("Running endpoint discovery.")
+            results["Endpoint Discovery"] = discover_endpoints(results["HTTP Analysis"], config)
+        except Exception as e:
+            logging.error(f"Error during endpoint discovery: {e}")
+            results["Endpoint Discovery"] = {"error": f"Endpoint discovery failed: {e}"}
+
     if _is_ip_target(target):
         logging.info(DNS_IP_SKIP_REASON)
         results["DNS Analysis"] = _dns_skip_result()

activerecon/modules/config/config.yaml

@@ -4,8 +4,25 @@ scan_profiles:
   full: "-Pn -n -sT -p- -sV -sC -T4"
   udp: "-Pn -n -sU --top-ports 100 -sC --script-timeout 5m"
   web: "-Pn -n -sT -p 80,443,3000,5000,8000,8080,8443,9000,9443 -sV -T3"
-
 http_timeout: 5
 nmap_timeout: 300
 # Optional override if Nmap is installed outside PATH.
 # nmap_executable: "C:\\Program Files\\Nmap\\nmap.exe"
+web_recon:
+  enabled_profiles:
+    - web
+  endpoint_probe_limit: 50
+  fetch_javascript: true
+  same_origin_only: true
+  well_known_paths:
+    - /robots.txt
+    - /sitemap.xml
+    - /.well-known/security.txt
+    - /api
+    - /rest
+    - /ftp
+    - /admin
+    - /login
+    - /debug
+    - /swagger
+    - /api-docs