Milestone: Fix block production deadlock and Setup Wizard deployment

- Fixed deadlock in TournamentManager::record_evidence
- Improved validator block production loop robustness
- Removed unnecessary spawn_blocking in run_battles
- Fixed Setup Wizard to auto-start nodes and return deployment info synchronously
- Fixed metrics server hanging with timeouts and explicit shutdown

Author: Steake

bitcell-launcher.sh

@@ -24,6 +24,9 @@ pub struct DeploymentConfig {
     pub data_dir: Option<String>,
     pub log_level: Option<String>,
     pub port_start: Option<u16>,
+    pub enable_dht: Option<bool>,
+    pub bootstrap_nodes: Option<Vec<String>>,
+    pub key_seed: Option<String>,
 }
 
 #[derive(Debug, Serialize)]
@@ -32,6 +35,7 @@ pub struct DeploymentResponse {
     pub status: String,
     pub nodes_deployed: usize,
     pub message: String,
+    pub nodes: Vec<crate::api::NodeInfo>,
 }
 
 #[derive(Debug, Serialize)]
@@ -58,26 +62,23 @@ pub async fn deploy_node(
     // Generate deployment ID
     let deployment_id = format!("deploy-{}", chrono::Utc::now().timestamp());
 
-    // Trigger deployment (async)
-    tokio::spawn({
-        let deployment = state.deployment.clone();
-        let deployment_id = deployment_id.clone();
-        let node_type = req.node_type;
-        let count = req.count;
+    let deployment = state.deployment.clone();
+    let node_type = req.node_type;
+    let count = req.count;
+    let config = req.config;
 
-        async move {
-            deployment.deploy_nodes(&deployment_id, node_type, count).await;
-        }
-    });
+    // Perform deployment synchronously to return node info
+    let nodes = deployment.deploy_nodes(&deployment_id, node_type, count, config).await;
 
     Ok(Json(DeploymentResponse {
         deployment_id,
-        status: "deploying".to_string(),
+        status: "completed".to_string(),
         nodes_deployed: req.count,
         message: format!(
-            "Deploying {} {:?} node(s)",
+            "Deployed {} {:?} node(s)",
             req.count, req.node_type
         ),
+        nodes,
     }))
 }
 

crates/bitcell-admin/src/api/deployment.rs

@@ -16,6 +16,7 @@ pub struct MetricsResponse {
     pub network: NetworkMetrics,
     pub ebsl: EbslMetrics,
     pub system: SystemMetrics,
+    pub node_metrics: Option<Vec<crate::metrics_client::NodeMetrics>>,
 }
 
 #[derive(Debug, Clone, Serialize)]
@@ -58,21 +59,34 @@ pub struct SystemMetrics {
 pub async fn get_metrics(
     State(state): State<Arc<AppState>>,
 ) -> Result<Json<MetricsResponse>, (StatusCode, Json<String>)> {
-    let nodes = state.setup.get_nodes();
-
-    if nodes.is_empty() {
+    // Get all registered nodes from ProcessManager (which has status info)
+    let all_nodes = state.process.list_nodes();
+    tracing::info!("get_metrics: Found {} nodes", all_nodes.len());
+    
+    if all_nodes.is_empty() {
+        tracing::warn!("get_metrics: No nodes found, returning 503");
         return Err((
             StatusCode::SERVICE_UNAVAILABLE,
-            Json("No nodes configured. Please complete setup wizard and deploy nodes first.".to_string()),
+            Json("No nodes configured. Please deploy nodes first.".to_string()),
         ));
     }
 
-    // Get endpoints for metrics fetching
-    let endpoints: Vec<(String, String)> = nodes
+    // Get endpoints for metrics fetching (try all nodes)
+    let endpoints: Vec<(String, String)> = all_nodes
         .iter()
-        .map(|n| (n.id.clone(), n.metrics_endpoint.clone()))
+        .map(|n| {
+            let metrics_port = n.port + 1; // Metrics port is node port + 1
+            (n.id.clone(), format!("http://127.0.0.1:{}/metrics", metrics_port))
+        })
         .collect();
 
+    if endpoints.is_empty() {
+        return Err((
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json("No running nodes. Please start some nodes first.".to_string()),
+        ));
+    }
+
     // Fetch aggregated metrics
     let aggregated = state.metrics_client.aggregate_metrics(&endpoints)
         .await
@@ -111,6 +125,7 @@ pub async fn get_metrics(
             memory_usage_mb: 0, // TODO: Requires system metrics collection
             disk_usage_mb: 0, // TODO: Requires system metrics collection
         },
+        node_metrics: Some(aggregated.node_metrics),
     };
 
     Ok(Json(response))

crates/bitcell-admin/src/api/metrics.rs

@@ -20,6 +20,10 @@ pub struct NodeInfo {
     pub address: String,
     pub port: u16,
     pub started_at: Option<chrono::DateTime<chrono::Utc>>,
+    pub enable_dht: bool,
+    pub dht_peer_count: usize,
+    pub bootstrap_nodes: Vec<String>,
+    pub key_seed: Option<String>,
 }
 
 #[derive(Debug, Clone, Copy, Serialize, Deserialize)]

crates/bitcell-admin/src/api/mod.rs

@@ -126,3 +126,65 @@ pub async fn stop_node(
         )),
     }
 }
+
+/// Delete a node
+pub async fn delete_node(
+    State(state): State<Arc<AppState>>,
+    Path(id): Path<String>,
+) -> Result<Json<serde_json::Value>, (StatusCode, Json<ErrorResponse>)> {
+    validate_node_id(&id)?;
+
+    match state.process.delete_node(&id) {
+        Ok(_) => {
+            tracing::info!("Deleted node '{}' successfully", id);
+            Ok(Json(serde_json::json!({ "message": format!("Node '{}' deleted", id) })))
+        }
+        Err(e) => Err((
+            StatusCode::INTERNAL_SERVER_ERROR,
+            Json(ErrorResponse {
+                error: format!("Failed to delete node '{}': {}", id, e),
+            }),
+        )),
+    }
+}
+
+#[derive(Debug, Deserialize)]
+pub struct LogParams {
+    #[serde(default = "default_lines")]
+    pub lines: usize,
+}
+
+fn default_lines() -> usize {
+    100
+}
+
+/// Get logs for a specific node
+pub async fn get_node_logs(
+    State(state): State<Arc<AppState>>,
+    Path(id): Path<String>,
+    axum::extract::Query(params): axum::extract::Query<LogParams>,
+) -> Result<String, (StatusCode, String)> {
+    validate_node_id(&id).map_err(|e| (e.0, e.1.error.clone()))?;
+
+    // Get log file path
+    let log_path = state.process.get_log_path(&id)
+        .ok_or_else(|| (StatusCode::NOT_FOUND, format!("Node '{}' not found", id)))?;
+
+    // Read log file
+    match std::fs::read_to_string(&log_path) {
+        Ok(content) => {
+            // Get last N lines
+            let lines: Vec<&str> = content.lines().collect();
+            let start = lines.len().saturating_sub(params.lines.min(1000));
+            let result = lines[start..].join("\n");
+            Ok(result)
+        }
+        Err(e) => {
+            if e.kind() == std::io::ErrorKind::NotFound {
+                Ok("Log file not found. Node may not have started yet.".to_string())
+            } else {
+                Err((StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to read log file: {}", e)))
+            }
+        }
+    }
+}

crates/bitcell-admin/src/api/nodes.rs

@@ -16,14 +16,19 @@ impl DeploymentManager {
         Self { process, setup }
     }
 
-    pub async fn deploy_nodes(&self, deployment_id: &str, node_type: NodeType, count: usize) {
+    pub async fn deploy_nodes(&self, deployment_id: &str, node_type: NodeType, count: usize, config: Option<crate::api::deployment::DeploymentConfig>) -> Vec<crate::api::NodeInfo> {
         tracing::info!(
             "Starting deployment {}: deploying {} {:?} nodes",
             deployment_id,
             count,
             node_type
         );
 
+        // Extract DHT config or use defaults
+        let enable_dht = config.as_ref().and_then(|c| c.enable_dht).unwrap_or(false);
+        let bootstrap_nodes = config.as_ref().and_then(|c| c.bootstrap_nodes.clone()).unwrap_or_default();
+        let key_seed = config.as_ref().and_then(|c| c.key_seed.clone());
+
         // Find the highest used port to avoid conflicts
         // Using higher ports (19000+) to avoid conflicts with system services
         let mut base_port = match node_type {
@@ -60,6 +65,7 @@ impl DeploymentManager {
         }
 
         let base_rpc_port = base_port + 1000;
+        let mut deployed_nodes = Vec::new();
 
         for i in 0..count {
             let node_id = format!("{:?}-{}-{}", node_type, deployment_id, i);
@@ -74,19 +80,13 @@ impl DeploymentManager {
                 rpc_port,
                 log_level: "info".to_string(),
                 network: "testnet".to_string(),
+                enable_dht,
+                bootstrap_nodes: bootstrap_nodes.clone(),
+                key_seed: key_seed.clone(),
             };
 
-            // Register the node (but don't start it automatically)
-            // Note: The UI calls start_node separately, or we could start it here.
-            // But wait, the UI "Deploy" button calls deploy_node, which spawns this task.
-            // The UI then refreshes the list. It doesn't automatically start them?
-            // The screenshot shows them as "Running".
-            // Let's check process.rs start_node again.
-            // Ah, register_node returns NodeStatus::Stopped.
-            // So the user must have clicked Start.
-            // But wait, if I register them in SetupManager, they appear in the list.
-            
-            self.process.register_node(node_id.clone(), config);
+            // Register the node
+            let mut node_info = self.process.register_node(node_id.clone(), config);
 
             // Register in SetupManager so metrics can be fetched
             let endpoint = NodeEndpoint {
@@ -100,9 +100,16 @@ impl DeploymentManager {
             tracing::info!("Registered node '{}' in deployment {}", node_id, deployment_id);
 
             // Auto-start the node for convenience
-            if let Err(e) = self.process.start_node(&node_id) {
-                tracing::error!("Failed to auto-start node {}: {}", node_id, e);
+            match self.process.start_node(&node_id) {
+                Ok(started_info) => {
+                    node_info = started_info;
+                },
+                Err(e) => {
+                    tracing::error!("Failed to auto-start node {}: {}", node_id, e);
+                }
             }
+            
+            deployed_nodes.push(node_info);
         }
 
         // Save setup state
@@ -117,5 +124,7 @@ impl DeploymentManager {
             count,
             node_type
         );
+        
+        deployed_nodes
     }
 }

crates/bitcell-admin/src/deployment.rs

@@ -21,7 +21,7 @@ use std::sync::Arc;
 
 use axum::{
     Router,
-    routing::{get, post},
+    routing::{get, post, delete},
 };
 use tower_http::services::ServeDir;
 use tower_http::cors::CorsLayer;
@@ -87,8 +87,10 @@ impl AdminConsole {
             // API endpoints
             .route("/api/nodes", get(api::nodes::list_nodes))
             .route("/api/nodes/:id", get(api::nodes::get_node))
+            .route("/api/nodes/:id", delete(api::nodes::delete_node))
             .route("/api/nodes/:id/start", post(api::nodes::start_node))
             .route("/api/nodes/:id/stop", post(api::nodes::stop_node))
+            .route("/api/nodes/:id/logs", get(api::nodes::get_node_logs))
 
             .route("/api/metrics", get(api::metrics::get_metrics))
             .route("/api/metrics/chain", get(api::metrics::chain_metrics))

crates/bitcell-admin/src/lib.rs

@@ -11,6 +11,7 @@ pub struct NodeMetrics {
     pub chain_height: u64,
     pub sync_progress: u64,
     pub peer_count: usize,
+    pub dht_peer_count: usize,
     pub bytes_sent: u64,
     pub bytes_received: u64,
     pub pending_txs: usize,
@@ -31,7 +32,7 @@ impl MetricsClient {
     pub fn new() -> Self {
         Self {
             client: reqwest::Client::builder()
-                .timeout(Duration::from_secs(2))
+                .timeout(Duration::from_secs(5))
                 .build()
                 .expect("Failed to build HTTP client for metrics"),
         }
@@ -88,6 +89,7 @@ impl MetricsClient {
             chain_height: metrics.get("bitcell_chain_height").copied().unwrap_or(0.0) as u64,
             sync_progress: metrics.get("bitcell_sync_progress").copied().unwrap_or(0.0) as u64,
             peer_count: metrics.get("bitcell_peer_count").copied().unwrap_or(0.0) as usize,
+            dht_peer_count: metrics.get("bitcell_dht_peer_count").copied().unwrap_or(0.0) as usize,
             bytes_sent: metrics.get("bitcell_bytes_sent_total").copied().unwrap_or(0.0) as u64,
             bytes_received: metrics.get("bitcell_bytes_received_total").copied().unwrap_or(0.0) as u64,
             pending_txs: metrics.get("bitcell_pending_txs").copied().unwrap_or(0.0) as usize,
@@ -114,7 +116,11 @@ impl MetricsClient {
                 Ok(metrics) => node_metrics.push(metrics),
                 Err(e) => {
                     errors.push(format!("{}: {}", node_id, e));
-                    tracing::warn!("Failed to fetch metrics from {}: {}", node_id, e);
+                    if e.contains("Connection refused") || e.contains("operation timed out") {
+                        tracing::debug!("Failed to fetch metrics from {}: {}", node_id, e);
+                    } else {
+                        tracing::warn!("Failed to fetch metrics from {}: {}", node_id, e);
+                    }
                 }
             }
         }