From c2ff20e7e9e25000d4a6580d387b7534e929404c Mon Sep 17 00:00:00 2001 From: Tarun Telang Date: Wed, 8 Apr 2026 00:22:22 +0530 Subject: [PATCH 01/12] Upgrade Docker base image to Java 21 --- code/chapter08/payment/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/chapter08/payment/Dockerfile b/code/chapter08/payment/Dockerfile index 77e6dde6..dbabd4f9 100644 --- a/code/chapter08/payment/Dockerfile +++ b/code/chapter08/payment/Dockerfile @@ -1,4 +1,4 @@ -FROM icr.io/appcafe/open-liberty:full-java17-openj9-ubi +FROM icr.io/appcafe/open-liberty:full-java21-openj9-ubi # Copy configuration files COPY --chown=1001:0 src/main/liberty/config/ /config/ From 4cfc547d60f252de0c38b7a6277df6902672e1b2 Mon Sep 17 00:00:00 2001 From: Tarun Telang Date: Wed, 8 Apr 2026 00:25:02 +0530 Subject: [PATCH 02/12] Upgrade Java version and dependencies in pom.xml Updated Maven configuration for Java 21 and updated dependencies. --- code/chapter08/payment/pom.xml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/code/chapter08/payment/pom.xml b/code/chapter08/payment/pom.xml index 12b8fada..ecc75ae2 100644 --- a/code/chapter08/payment/pom.xml +++ b/code/chapter08/payment/pom.xml @@ -12,8 +12,8 @@ UTF-8 - 17 - 17 + 21 + 21 UTF-8 UTF-8 @@ -33,7 +33,7 @@ org.projectlombok lombok - 1.18.26 + 1.18.36 provided @@ -49,7 +49,7 @@ org.eclipse.microprofile microprofile - 6.1 + 7.1 pom provided @@ -65,6 +65,16 @@ ${project.artifactId} + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 21 + + + io.openliberty.tools @@ -82,4 +92,4 @@ - \ No newline at end of file + From c77883afae4944d7a21b75af4c34ffe33ed0f510 Mon Sep 17 00:00:00 2001 From: Tarun Telang Date: Wed, 8 Apr 2026 00:29:48 +0530 Subject: [PATCH 03/12] Revise README for Payment Service Fault Tolerance Updated README to enhance clarity on MicroProfile Fault Tolerance features and implementation details. --- code/chapter08/payment/README.adoc | 616 +++++++++++++++++++++++++---- 1 file changed, 547 insertions(+), 69 deletions(-) diff --git a/code/chapter08/payment/README.adoc b/code/chapter08/payment/README.adoc index eb53e9dc..423bc079 100644 --- a/code/chapter08/payment/README.adoc +++ b/code/chapter08/payment/README.adoc @@ -1,132 +1,610 @@ -= Payment Service += Payment Service - MicroProfile Fault Tolerance :toc: macro -:toclevels: 3 +:toclevels: 1 :icons: font :source-highlighter: highlight.js :experimental: toc::[] -This microservice is part of the Jakarta EE 10 and MicroProfile 6.1-based e-commerce application. It handles payment processing and transaction management. +This microservice demonstrates MicroProfile Fault Tolerance patterns for building resilient payment processing services. It is part of the Jakarta EE and MicroProfile based e-commerce tutorial. == Features -* Payment transaction processing -* Dynamic configuration management via MicroProfile Config -* RESTful API endpoints with JSON support -* Custom ConfigSource implementation -* OpenAPI documentation -* **MicroProfile Fault Tolerance with Retry Policies** -* **Circuit Breaker protection for external services** -* **Fallback mechanisms for service resilience** -* **Bulkhead pattern for concurrency control** -* **Timeout protection for long-running operations** +* **Payment Authorization** with Retry, Timeout, and Fallback +* **Gateway Health Checks** with Circuit Breaker protection +* **Async Notifications** with Bulkhead resource isolation == MicroProfile Fault Tolerance Implementation -The Payment Service implements comprehensive fault tolerance patterns using MicroProfile Fault Tolerance annotations: +The Payment Service demonstrates three distinct fault tolerance patterns, each applied to different service methods: -=== Retry Policies +=== 1. Payment Authorization (Retry + Timeout + Fallback) -The service implements different retry strategies based on operation criticality: +**Method**: `PaymentService.authorizePayment()` -==== Payment Authorization Retry (@Retry) -* **Max Retries**: 3 attempts -* **Delay**: 1000ms with 500ms jitter -* **Max Duration**: 10 seconds -* **Retry On**: RuntimeException, WebApplicationException -* **Use Case**: Standard payment authorization with exponential backoff +**Fault Tolerance Strategy:** +* **@Retry**: Handles transient network failures (up to 3 retries with jitter) +* **@Timeout**: Prevents indefinite waits (3 second limit per attempt) +* **@Fallback**: Provides degraded service when gateway unavailable + +**Configuration:** [source,java] ---- @Retry( maxRetries = 3, delay = 2000, - maxDuration = 10000 jitter = 500, - retryOn = {RuntimeException.class, WebApplicationException.class} + retryOn = PaymentProcessingException.class, + abortOn = CriticalPaymentException.class ) +@Timeout(3000) +@Fallback(fallbackMethod = "fallbackAuthorizePayment") +public String authorizePayment(PaymentDetails paymentDetails) ---- -=== Circuit Breaker Protection +**Behavior:** + +* Retries transient failures (network timeouts, temporary gateway issues) +* Does NOT retry critical errors (invalid card, insufficient funds) +* Falls back to offline processing queue if all retries fail +* Each retry attempt has its own 3-second timeout + +=== 2. Gateway Health Check (Circuit Breaker + Timeout) + +**Method**: `PaymentService.checkGatewayHealth()` -Payment capture operations use circuit breaker pattern: +**Fault Tolerance Strategy:** +* **@CircuitBreaker**: Prevents hammering failed gateway (opens at 50% failure rate) +* **@Timeout**: Quick health check (2 second limit) + +**Configuration:** [source,java] ---- @CircuitBreaker( failureRatio = 0.5, requestVolumeThreshold = 4, - delay = 5000 + delay = 5000, + successThreshold = 2 ) +@Timeout(2000) +public boolean checkGatewayHealth() ---- -* **Failure Ratio**: 50% failure rate triggers circuit opening -* **Request Volume**: Minimum 4 requests for evaluation -* **Recovery Delay**: 5 seconds before attempting recovery +**Circuit Breaker States:** + +* **CLOSED**: Normal operation, all health checks pass through +* **OPEN**: Circuit breaker activated after 50% failure rate (2 of 4 requests fail) +* **HALF_OPEN**: After 5-second delay, allows 2 test requests +* **CLOSED**: Returns to normal after 2 consecutive successful tests + +**Behavior:** -=== Timeout Protection +* Prevents repeated health checks to failed gateway +* Automatically recovers when gateway comes back online +* Reduces load on failing systems -Operations with potential long delays are protected with timeouts: +=== 3. Payment Notification (Asynchronous + Bulkhead + Fallback) +**Method**: `PaymentService.sendPaymentNotification()` + +**Fault Tolerance Strategy:** + +* **@Asynchronous**: Non-blocking execution in separate thread +* **@Bulkhead**: Limits concurrent notifications (max 10, queue 20) +* **@Timeout**: Prevents stuck notification threads (5 second limit) +* **@Fallback**: Logs failed notifications for retry + +**Configuration:** [source,java] ---- -@Timeout(value = 3000) +@Asynchronous +@Bulkhead(value = 10, waitingTaskQueue = 20) +@Timeout(5000) +@Fallback(fallbackMethod = "fallbackSendNotification") +public CompletionStage sendPaymentNotification(String paymentId, String recipient) ---- -=== Bulkhead Pattern +**Behavior:** -The bulkhead pattern limits concurrent requests to prevent system overload: +* Returns immediately (non-blocking) via CompletionStage +* Up to 10 concurrent notifications processing +* Additional 20 notifications queued +* Excess notifications rejected to prevent overload +* Failed notifications logged for later retry -[source,java] +== REST Endpoints + +=== POST /payment/api/authorize + +Authorize a payment with full fault tolerance protection. + +**Request:** +[source,bash] ---- -@Bulkhead(value = 5) +curl -X POST "http://localhost:9080/payment/api/authorize?amount=100.50" ---- -* **Concurrent Requests**: Limited to 5 concurrent requests -* **Excess Requests**: Rejected immediately instead of queuing -* **Use Case**: Protect service from traffic spikes and cascading failures +**Success Response (200 OK):** +[source,json] +---- +{ + "status": "success", + "message": "Payment authorized", + "amount": 100.50 +} +---- -=== Fallback Mechanisms +**Fallback Response (200 OK - degraded service):** +[source,json] +---- +{ + "status": "pending", + "message": "Payment queued for processing", + "amount": 100.50 +} +---- -All critical operations have fallback methods that provide graceful degradation: +**Error Response (400 Bad Request):** +[source,json] +---- +{ + "error": "Invalid payment amount: -10" +} +---- -* **Payment Authorization Fallback**: Returns service unavailable with retry instructions +=== GET /payment/api/health/gateway -== Endpoints +Check payment gateway health with circuit breaker protection. -=== GET /payment/api/payment-config -* Returns all current payment configuration values -* Example: `GET http://localhost:9080/payment/api/payment-config` -* Response: `{"gateway.endpoint":"https://api.paymentgateway.com"}` +**Request:** +[source,bash] +---- +curl http://localhost:9080/payment/api/health/gateway +---- -=== POST /payment/api/payment-config -* Updates a payment configuration value -* Example: `POST http://localhost:9080/payment/api/payment-config` -* Request body: `{"key": "payment.gateway.endpoint", "value": "https://new-api.paymentgateway.com"}` -* Response: `{"key":"payment.gateway.endpoint","value":"https://new-api.paymentgateway.com","message":"Configuration updated successfully"}` +**Healthy Response (200 OK):** +[source,json] +---- +{ + "status": "healthy", + "message": "Payment gateway is operational" +} +---- -=== POST /payment/api/authorize -* Processes a payment authorization with retry policy -* **Retry Configuration**: 3 attempts, 1s delay, 500ms jitter -* **Fallback**: Service unavailable response -* Example: `POST http://localhost:9080/payment/api/authorize` -* Request body: `{"cardNumber":"4111111111111111", "cardHolderName":"Test User", "expiryDate":"12/25", "securityCode":"123", "amount":100.00}` -* Response: `{"status":"success", "message":"Payment authorized successfully", "transactionId":"TXN1234567890", "amount":100.00}` -* Fallback Response: `{"status":"failed", "message":"Payment gateway unavailable. Please try again later.", "fallback":true}` - -=== POST /payment/api/payment-config/process-example -* Example endpoint demonstrating payment processing with configuration -* Example: `POST http://localhost:9080/payment/api/payment-config/process-example` -* Request body: `{"cardNumber":"4111111111111111", "cardHolderName":"Test User", "expiryDate":"12/25", "securityCode":"123", "amount":100.00}` -* Response: `{"amount":100.00,"message":"Payment processed successfully","status":"success","configUsed":{"gatewayEndpoint":"https://new-api.paymentgateway.com"}}` +**Circuit Open Response (503 Service Unavailable):** +[source,json] +---- +{ + "status": "circuit_open", + "message": "Circuit breaker is open - gateway appears to be down" +} +---- + +=== POST /payment/api/notify/{paymentId} + +Send asynchronous payment notification with bulkhead protection. + +**Request:** +[source,bash] +---- +curl -X POST "http://localhost:9080/payment/api/notify/PAY-12345?recipient=customer@example.com" +---- + +**Success Response (200 OK):** +[source,json] +---- +{ + "status": "success", + "message": "Notification sent successfully" +} +---- + +**Bulkhead Rejected Response (503 Service Unavailable):** +[source,json] +---- +{ + "status": "rejected", + "message": "Too many concurrent notifications - please try again later" +} +---- + +== Testing and Verifying Fault Tolerance + +This section explains how to test and verify that each fault tolerance pattern is working correctly. + +=== Automated Test Scripts + +The payment service includes comprehensive test scripts that demonstrate all fault tolerance patterns: + +[source,bash] +---- +cd code/chapter08/payment + +# 1. Basic smoke test - verify all endpoints work +./test-payment-basic.sh + +# 2. Test retry behavior (30% simulated failure rate) +./test-payment-retry.sh + +# 3. Test circuit breaker state transitions +./test-payment-circuit-breaker.sh + +# 4. Test asynchronous execution +./test-payment-async.sh + +# 5. Test bulkhead resource isolation +./test-payment-bulkhead.sh + +# 6. Comprehensive load test (all patterns) +./test-payment-concurrent-load.sh +---- + +=== Manual Testing: Retry Pattern + +**What to test:** The authorization endpoint retries transient failures but aborts on critical errors. + +**Test valid payment (may trigger retries):** +[source,bash] +---- +curl -X POST "http://localhost:9080/payment/api/authorize?amount=100.50" +---- + +**Expected behavior:** +* 70% success rate (simulated) +* Failed requests automatically retry up to 3 times +* Each retry has 2-2.5 second delay (2000ms + 500ms jitter) +* Total duration reveals retry count: + - ~1.5s = Success on 1st attempt + - ~4s = Needed 1 retry + - ~6.5s = Needed 2 retries + - ~9-12s = All 3 retries or fallback + +**Test abort condition (no retries):** +[source,bash] +---- +curl -X POST "http://localhost:9080/payment/api/authorize?amount=0" +curl -X POST "http://localhost:9080/payment/api/authorize?amount=-50" +---- + +**Expected behavior:** +* Immediate failure (~1.5s) with HTTP 400 +* No retries attempted (abort on CriticalPaymentException) + +**Verification:** +[source,bash] +---- +# Check retry metrics +curl http://localhost:9080/metrics?scope=base | grep "ft_authorizePayment_retry" + +# Look for: +# - retry_callsSucceededNotRetried_total (first-attempt successes) +# - retry_callsSucceededRetried_total (succeeded after retry) +# - retry_retries_total (total number of retries executed) +---- + +=== Manual Testing: Circuit Breaker Pattern + +**What to test:** The health check endpoint opens the circuit after repeated failures, preventing cascade. + +**Trigger circuit breaker (50% simulated failure rate):** +[source,bash] +---- +# Send 10 health checks - circuit should open after ~4-5 requests +for i in {1..10}; do + echo "Request $i:" + curl http://localhost:9080/payment/api/health/gateway + echo "" + sleep 0.5 +done +---- + +**Expected state transitions:** +1. **CLOSED** (0): First few requests pass through normally +2. **OPEN** (1): After 2 failures out of 4 requests (50% threshold), circuit opens +3. Subsequent requests fail immediately with "Circuit breaker is OPEN" message +4. **HALF_OPEN** (2): After 5-second delay, circuit allows test requests +5. **CLOSED** (0): After 2 consecutive successes, circuit returns to normal + +**Wait for recovery:** +[source,bash] +---- +# Wait 5 seconds for circuit delay +sleep 5 + +# Send requests to test HALF_OPEN → CLOSED transition +curl http://localhost:9080/payment/api/health/gateway # Test 1 +curl http://localhost:9080/payment/api/health/gateway # Test 2 (should close circuit) +---- + +**Verification:** +[source,bash] +---- +# Check circuit breaker state +curl http://localhost:9080/metrics?scope=base | grep "ft_checkGatewayHealth_circuitbreaker" + +# Look for: +# - circuitbreaker_state_total{state="0"} = CLOSED count +# - circuitbreaker_state_total{state="1"} = OPEN count +# - circuitbreaker_state_total{state="2"} = HALF_OPEN count +# - circuitbreaker_opened_total = Number of times circuit opened +---- + +=== Manual Testing: Bulkhead Pattern + +**What to test:** The notification endpoint limits concurrent executions and rejects excess requests. + +**Send concurrent requests to test limits:** +[source,bash] +---- +# Send 15 concurrent notifications +# Expected: 3 concurrent + 2 queued = 5 accepted, 10 rejected +for i in {1..15}; do + curl -X POST "http://localhost:9080/payment/api/notify/PAY-$(printf "%05d" $i)" & +done +wait +---- + +**Expected behavior:** +* **Bulkhead capacity:** 3 concurrent + 2 waiting queue = 5 total +* Requests 1-3: Execute immediately (concurrent slots) +* Requests 4-5: Queued for execution (waiting queue) +* Requests 6-15: **Rejected** with BulkheadException (HTTP 503) + +**Verification:** +[source,bash] +---- +# Check bulkhead metrics +curl http://localhost:9080/metrics?scope=base | grep "ft_sendPaymentNotification_bulkhead" + +# Look for: +# - bulkhead_callsAccepted_total (accepted requests) +# - bulkhead_callsRejected_total (rejected requests) +# - bulkhead_executionDuration_* (execution time stats) +# - bulkhead_queuePopulation_current (current queue size) +---- + +=== Manual Testing: Asynchronous Pattern + +**What to test:** The notification endpoint returns immediately (non-blocking) despite 2-second processing time. + +**Send sequential async requests:** +[source,bash] +---- +# Time 5 sequential async requests +time for i in {1..5}; do + curl -X POST "http://localhost:9080/payment/api/notify/PAY-0000$i" +done + +# Expected: < 2.5 seconds total (if truly async) +# If blocking: ~10 seconds (5 requests × 2s each) +---- + +**Expected behavior:** +* Each request returns immediately (< 0.5s) +* Total time for 5 requests: < 2.5 seconds +* Processing continues in background +* Uses CompletionStage (not Future) for proper fault tolerance integration + +**Verification:** +* Check response time (should be < 500ms per request) +* Monitor server logs for async processing +* Verify bulkhead metrics show concurrent executions + +=== Observing Fault Tolerance in Server Logs + +Monitor the Liberty server logs to see fault tolerance in action: + +[source,bash] +---- +# Tail the server logs +tail -f target/liberty/wlp/usr/servers/defaultServer/logs/messages.log + +# Look for log messages indicating: +# - "Processing payment..." (method entry) +# - "Retry attempt X of 3" (retry behavior) +# - "Circuit breaker opened/closed" (state changes) +# - "Bulkhead rejected request" (resource limits) +# - "Fallback invoked" (degraded service) +---- + +=== Validating Configuration + +Test that fault tolerance configuration is loaded correctly: + +[source,bash] +---- +# Check all fault tolerance configuration properties +curl http://localhost:9080/payment/api/payment-config | grep -E "(retry|timeout|circuitbreaker|bulkhead)" +---- + +=== Complete Verification Checklist + +Use this checklist to verify all fault tolerance patterns: + +- [ ] **Retry**: Failed requests retry automatically (check logs and metrics) +- [ ] **Retry Abort**: Critical errors don't retry (test with amount=0) +- [ ] **Timeout**: Requests timeout after configured duration +- [ ] **Circuit Breaker**: Circuit opens after failure threshold +- [ ] **Circuit Recovery**: Circuit closes after successful probes +- [ ] **Bulkhead**: Concurrent requests limited (check rejected count) +- [ ] **Async**: Requests return immediately (< 500ms response time) +- [ ] **Fallback**: Degraded service provided on failures +- [ ] **Metrics**: All fault tolerance metrics exposed and updating + +**Success criteria:** All test scripts complete successfully, and metrics show expected fault tolerance activations. + +== Monitoring with MicroProfile Metrics + +Access fault tolerance metrics at the metrics endpoint: + +[source,bash] +---- +# View all base scope metrics (includes fault tolerance) +curl http://localhost:9080/metrics?scope=base + +# Filter for payment authorization retry metrics +curl http://localhost:9080/metrics?scope=base | grep "authorizePayment.*retry" + +# Check circuit breaker state (0=CLOSED, 1=OPEN, 2=HALF_OPEN) +curl http://localhost:9080/metrics?scope=base | grep "circuitbreaker.state" + +# View bulkhead metrics +curl http://localhost:9080/metrics?scope=base | grep "bulkhead" +---- + +**Key Metrics to Monitor:** + +[cols="2,3", options="header"] +|=== +| Metric | Description + +| `ft_authorizePayment_retry_callsSucceededNotRetried_total` +| Payments that succeeded without retry + +| `ft_authorizePayment_retry_callsSucceededRetried_total` +| Payments that succeeded after retrying + +| `ft_authorizePayment_fallback_calls_total` +| Number of times fallback was invoked + +| `ft_checkGatewayHealth_circuitbreaker_state_total` +| Circuit breaker state (0/1/2) + +| `ft_sendPaymentNotification_bulkhead_callsRejected_total` +| Notifications rejected by bulkhead +|=== + +== Building and Running + +=== Prerequisites + +* JDK 21 or higher +* Maven 3.13.x or higher + +=== Local Development + +[source,bash] +---- +# Navigate to payment service directory +cd code/chapter08/payment + +# Build the application +mvn clean package + +# Run with Liberty +mvn liberty:run + +# In another terminal, test the endpoints +curl -X POST "http://localhost:9080/payment/api/authorize?amount=100" +---- + +The server will start on: +* HTTP: `http://localhost:9080` +* HTTPS: `https://localhost:9081` + +=== Docker + +[source,bash] +---- +# Build and run with Docker +./run-docker.sh +---- + +== Project Structure + +[source] +---- +payment/ +├── src/ +│ ├── main/ +│ │ ├── java/ +│ │ │ └── io/microprofile/tutorial/store/payment/ +│ │ │ ├── PaymentRestApplication.java +│ │ │ ├── service/ +│ │ │ │ └── PaymentService.java # Fault tolerance patterns +│ │ │ ├── resource/ +│ │ │ │ ├── PaymentResource.java # REST endpoints +│ │ │ │ └── PaymentConfigResource.java +│ │ │ ├── entity/ +│ │ │ │ └── PaymentDetails.java +│ │ │ ├── exception/ +│ │ │ │ ├── PaymentProcessingException.java # Retryable errors +│ │ │ │ └── CriticalPaymentException.java # Non-retryable errors +│ │ │ └── config/ +│ │ │ ├── PaymentServiceConfigSource.java +│ │ │ └── PaymentConfig.java +│ │ ├── liberty/config/ +│ │ │ └── server.xml # mpFaultTolerance feature +│ │ └── resources/ +│ │ └── META-INF/ +│ │ └── microprofile-config.properties +│ └── test/ +│ └── java/ +├── pom.xml +└── README.adoc +---- + +== Key Concepts Demonstrated + +This chapter demonstrates: + +**1. Retry Pattern** +- Automatically retry transient failures +- Configure retry delays with jitter +- Distinguish retryable vs non-retryable exceptions + +**2. Circuit Breaker Pattern** +- Prevent cascading failures +- Automatic recovery detection +- State transitions (CLOSED → OPEN → HALF_OPEN → CLOSED) + +**3. Bulkhead Pattern ** +- Resource isolation +- Thread pool style with queuing +- Prevent resource exhaustion + +**4. Timeout Pattern** +- Prevent indefinite blocking +- Per-attempt timeout with retry +- Free resources for other operations + +**5. Fallback Pattern** +- Graceful degradation +- Alternative responses during failures +- Maintain service availability + +**6. Asynchronous Execution** +- Non-blocking operations +- CompletionStage for proper fault tolerance integration +- Improved scalability + +**7. Configuration Externalization** +- Runtime configuration via MicroProfile Config +- Environment-specific fault tolerance parameters +- No code changes required + +## Related Chapters + +- **Chapter 3**: Idempotency (safe retries) and Audit Logging +- **Chapter 5**: MicroProfile Config (externalize fault tolerance settings) +- **Chapter 7**: MicroProfile Metrics (monitor fault tolerance behavior) +- **Chapter 11**: Complete integrated e-commerce application + +== Further Reading + +- link:https://download.eclipse.org/microprofile/microprofile-fault-tolerance-4.1/microprofile-fault-tolerance-spec-4.1.html[MicroProfile Fault Tolerance 4.1 Specification] +- link:https://openliberty.io/docs/latest/fault-tolerance.html[Open Liberty Fault Tolerance Guide] +- link:https://openliberty.io/blog/2020/06/04/asynchronous-programming-microprofile-fault-tolerance.html[Asynchronous Programming with MicroProfile Fault Tolerance] == Building and Running the Service === Prerequisites -* JDK 17 or higher -* Maven 3.6.0 or higher +* JDK 21 or higher +* Maven 3.13.x or higher === Local Development From 5f08247b55f47656fb7857d50dc92165523c0cb2 Mon Sep 17 00:00:00 2001 From: Tarun Telang Date: Wed, 8 Apr 2026 00:32:51 +0530 Subject: [PATCH 04/12] Refactor async payment test script Refactor test script for asynchronous payment notifications to improve readability and functionality. Dynamically determine base URL based on environment and enhance testing structure for async behavior. --- code/chapter08/payment/test-payment-async.sh | 226 ++++++++++++------- 1 file changed, 144 insertions(+), 82 deletions(-) diff --git a/code/chapter08/payment/test-payment-async.sh b/code/chapter08/payment/test-payment-async.sh index 2803b66e..eb15b28e 100755 --- a/code/chapter08/payment/test-payment-async.sh +++ b/code/chapter08/payment/test-payment-async.sh @@ -1,10 +1,7 @@ #!/bin/bash -# Enhanced test script for verifying asynchronous processing -# This script shows the asynchronous behavior by: -# 1. Checking for concurrent processing -# 2. Monitoring response times to verify non-blocking behavior -# 3. Analyzing the server logs to confirm retry patterns +# Test script for asynchronous payment notification processing +# Demonstrates @Asynchronous annotation behavior # Color definitions RED='\033[0;31m' @@ -15,107 +12,172 @@ PURPLE='\033[0;35m' CYAN='\033[0;36m' NC='\033[0m' # No Color -# Check if bc is installed and install it if not +# Check if bc is installed if ! command -v bc &> /dev/null; then - echo -e "${YELLOW}The 'bc' command is not found. Installing bc...${NC}" + echo -e "${YELLOW}Installing bc...${NC}" sudo apt-get update && sudo apt-get install -y bc - if [ $? -ne 0 ]; then - echo -e "${RED}Failed to install bc. Please install it manually.${NC}" - exit 1 - fi - echo -e "${GREEN}bc installed successfully.${NC}" fi -# Set payment endpoint URL -HOST="localhost" -PAYMENT_URL="http://${HOST}:9080/payment/api/authorize" +# Dynamically determine the base URL +if [ -n "$CODESPACE_NAME" ] && [ -n "$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN" ]; then + BASE_URL="http://localhost:9080/payment/api" + echo -e "${CYAN}Detected GitHub Codespaces environment${NC}" +elif [ -n "$GITPOD_WORKSPACE_URL" ]; then + GITPOD_HOST=$(echo $GITPOD_WORKSPACE_URL | sed 's|https://||' | sed 's|/||') + BASE_URL="https://9080-$GITPOD_HOST/payment/api" + echo -e "${CYAN}Detected Gitpod environment${NC}" +else + BASE_URL="http://localhost:9080/payment/api" + echo -e "${CYAN}Using local environment${NC}" +fi -echo -e "${BLUE}=== Enhanced Asynchronous Testing ====${NC}" -echo -e "${CYAN}This test will send a series of requests to demonstrate asynchronous processing${NC}" +echo "" +echo -e "${BLUE}=== Testing Asynchronous Payment Notifications ===${NC}" +echo -e "${CYAN}Endpoint: POST /notify/{paymentId}${NC}" +echo -e "${CYAN}Base URL: $BASE_URL${NC}" echo "" -# First, let's check server logs before test -echo -e "${YELLOW}Checking server logs before test...${NC}" -echo -e "${CYAN}(This establishes a baseline for comparison)${NC}" -cd /workspaces/liberty-rest-app/payment -MESSAGES_LOG="target/liberty/wlp/usr/servers/mpServer/logs/messages.log" - -if [ -f "$MESSAGES_LOG" ]; then - echo -e "${PURPLE}Server log file exists at: $MESSAGES_LOG${NC}" - # Count initial payment processing messages - INITIAL_PROCESSING_COUNT=$(grep -c "Processing payment for amount" "$MESSAGES_LOG") - INITIAL_FALLBACK_COUNT=$(grep -c "Fallback invoked for payment" "$MESSAGES_LOG") - - echo -e "${CYAN}Initial payment processing count: $INITIAL_PROCESSING_COUNT${NC}" - echo -e "${CYAN}Initial fallback count: $INITIAL_FALLBACK_COUNT${NC}" -else - echo -e "${RED}Server log file not found at: $MESSAGES_LOG${NC}" - INITIAL_PROCESSING_COUNT=0 - INITIAL_FALLBACK_COUNT=0 -fi +echo -e "${YELLOW}Asynchronous Configuration:${NC}" +echo " • Method: sendPaymentNotification()" +echo " • Return Type: CompletionStage" +echo " • Processing: Non-blocking" +echo " • Simulated delay: ~2 seconds" +echo "" +echo -e "${CYAN}🔍 Testing Goals:${NC}" +echo " 1. Verify requests return immediately (non-blocking)" +echo " 2. Demonstrate concurrent processing" +echo " 3. Show CompletionStage behavior" echo "" -echo -e "${YELLOW}Now sending 3 requests in rapid succession...${NC}" -# Function to send request and measure time -send_request() { +# Function to send notification and measure response time +send_async_notification() { local id=$1 - local amount=$2 - local start_time=$(date +%s.%N) + local payment_id=$2 - response=$(curl -s -X POST "${PAYMENT_URL}?amount=${amount}") + start_time=$(date +%s.%N) + response=$(curl -s -w "\nHTTP_STATUS:%{http_code}\nTIME_TOTAL:%{time_total}" \ + -X POST "${BASE_URL}/notify/${payment_id}" 2>/dev/null || echo "HTTP_STATUS:000") + end_time=$(date +%s.%N) - local end_time=$(date +%s.%N) - local duration=$(echo "$end_time - $start_time" | bc) + http_code=$(echo "$response" | grep "HTTP_STATUS:" | cut -d: -f2) + curl_time=$(echo "$response" | grep "TIME_TOTAL:" | cut -d: -f2) + body=$(echo "$response" | sed '/HTTP_STATUS:/d' | sed '/TIME_TOTAL:/d') - echo -e "${GREEN}[Request $id] Completed in ${duration}s${NC}" - echo -e "${CYAN}[Request $id] Response: $response${NC}" + wall_time=$(echo "$end_time - $start_time" | bc) + wall_time_formatted=$(printf "%.3f" $wall_time) - return 0 + if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then + # Check if response was truly async (should be fast, not 2 seconds) + is_async=$(echo "$curl_time < 0.5" | bc) + if [ "$is_async" -eq 1 ]; then + echo -e "${GREEN}[Request $id] ✓ ASYNC - Returned immediately (${wall_time_formatted}s)${NC}" + echo -e "${GREEN} Response: $body${NC}" + else + echo -e "${YELLOW}[Request $id] ⚠ SYNC? - Took ${wall_time_formatted}s (expected < 0.5s)${NC}" + echo -e "${YELLOW} Response: $body${NC}" + fi + else + echo -e "${RED}[Request $id] ✗ ERROR (HTTP $http_code, ${wall_time_formatted}s)${NC}" + echo -e "${RED} Response: $body${NC}" + fi + + echo "$wall_time_formatted" } -# Send 3 requests in rapid succession -for i in {1..3}; do - # Use a fixed amount for consistency - amount=25.99 - echo -e "${PURPLE}[Request $i] Sending request for \$$amount...${NC}" - send_request $i $amount & - # Sleep briefly to ensure log messages are distinguishable - sleep 0.1 +# Test 1: Single async request +echo -e "${BLUE}=== Test 1: Single Asynchronous Request ===${NC}" +echo -e "${CYAN}Sending single notification to verify async behavior...${NC}" +echo "" + +send_async_notification 1 "PAY-00001" + +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Test 2: Multiple sequential requests +echo -e "${BLUE}=== Test 2: Sequential Asynchronous Requests ===${NC}" +echo -e "${CYAN}Sending 5 sequential requests...${NC}" +echo -e "${YELLOW}Each should return immediately (< 0.5s) despite 2s processing time${NC}" +echo "" + +total_start=$(date +%s.%N) + +for i in {2..6}; do + payment_id=$(printf "PAY-%05d" $i) + send_async_notification $i $payment_id done -# Wait for all background processes to complete -wait +total_end=$(date +%s.%N) +total_time=$(echo "$total_end - $total_start" | bc) +total_time_formatted=$(printf "%.2f" $total_time) echo "" -echo -e "${YELLOW}Waiting 5 seconds for processing to complete...${NC}" -sleep 5 +echo -e "${PURPLE}Total time for 5 sequential requests: ${total_time_formatted}s${NC}" +echo -e "${CYAN}Expected: < 2.5s if async, ~10s if blocking${NC}" -# Check the server logs after test -echo -e "${YELLOW}Checking server logs after test...${NC}" +if (( $(echo "$total_time < 3" | bc -l) )); then + echo -e "${GREEN}✓ CONFIRMED: Requests are truly asynchronous!${NC}" +else + echo -e "${YELLOW}⚠ WARNING: Requests may be blocking (total time too high)${NC}" +fi -if [ -f "$MESSAGES_LOG" ]; then - # Count final payment processing messages - FINAL_PROCESSING_COUNT=$(grep -c "Processing payment for amount" "$MESSAGES_LOG") - FINAL_FALLBACK_COUNT=$(grep -c "Fallback invoked for payment" "$MESSAGES_LOG") - - NEW_PROCESSING=$(($FINAL_PROCESSING_COUNT - $INITIAL_PROCESSING_COUNT)) - NEW_FALLBACKS=$(($FINAL_FALLBACK_COUNT - $INITIAL_FALLBACK_COUNT)) - - echo -e "${CYAN}New payment processing events: $NEW_PROCESSING${NC}" - echo -e "${CYAN}New fallback events: $NEW_FALLBACKS${NC}" - - # Extract the latest log entries - echo "" - echo -e "${BLUE}Latest server log entries related to payment processing:${NC}" - grep "Processing payment for amount\|Fallback invoked for payment" "$MESSAGES_LOG" | tail -10 +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Test 3: Concurrent requests +echo -e "${BLUE}=== Test 3: Concurrent Asynchronous Requests ===${NC}" +echo -e "${CYAN}Sending 5 concurrent requests in parallel...${NC}" +echo -e "${YELLOW}All should complete quickly, demonstrating true async processing${NC}" +echo "" + +concurrent_start=$(date +%s.%N) + +# Launch background jobs +for i in {7..11}; do + payment_id=$(printf "PAY-%05d" $i) + (send_async_notification $i $payment_id) & + sleep 0.05 # Small stagger to make output readable +done + +# Wait for all background jobs +wait + +concurrent_end=$(date +%s.%N) +concurrent_time=$(echo "$concurrent_end - $concurrent_start" | bc) +concurrent_time_formatted=$(printf "%.2f" $concurrent_time) + +echo "" +echo -e "${PURPLE}Total time for 5 concurrent requests: ${concurrent_time_formatted}s${NC}" +echo -e "${CYAN}Expected: ~0.5-1.0s (all start immediately)${NC}" + +if (( $(echo "$concurrent_time < 2" | bc -l) )); then + echo -e "${GREEN}✓ EXCELLENT: Concurrent async processing confirmed!${NC}" else - echo -e "${RED}Server log file not found after test${NC}" + echo -e "${YELLOW}⚠ Note: Time higher than expected${NC}" fi echo "" -echo -e "${BLUE}=== Asynchronous Behavior Analysis ====${NC}" -echo -e "${CYAN}1. Rapid response times indicate non-blocking behavior${NC}" -echo -e "${CYAN}2. Multiple processing entries in logs show concurrent execution${NC}" -echo -e "${CYAN}3. Fallbacks demonstrate the fault tolerance mechanism${NC}" -echo -e "${CYAN}4. All @Asynchronous methods return quickly while processing continues in background${NC}" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Summary +echo -e "${BLUE}=== Asynchronous Processing Summary ===${NC}" +echo "" +echo -e "${CYAN}Key Observations:${NC}" +echo " • @Asynchronous methods return CompletionStage" +echo " • JAX-RS endpoint returns CompletionStage" +echo " • Client receives immediate response (non-blocking)" +echo " • Server processes work in background thread pool" +echo " • Multiple requests can execute concurrently" +echo "" + +echo -e "${CYAN}CompletionStage vs Future:${NC}" +echo " • CompletionStage: Fault tolerance applied, supports chaining" +echo " • Future: Fault tolerance NOT applied (avoid!)" +echo "" + +echo -e "${GREEN}=== Test Complete ===${NC}" +echo "" From 245aa4b4fec5c3f3e029f2f11bf9a0acd1d8e241 Mon Sep 17 00:00:00 2001 From: Tarun Telang Date: Wed, 8 Apr 2026 00:33:44 +0530 Subject: [PATCH 05/12] Refactor payment test script for environment detection Updated the payment test script to dynamically determine the base URL based on the environment. Enhanced comments and improved error handling for the 'bc' command installation check. --- code/chapter08/payment/test-payment-basic.sh | 115 +++++++++++++++---- 1 file changed, 90 insertions(+), 25 deletions(-) diff --git a/code/chapter08/payment/test-payment-basic.sh b/code/chapter08/payment/test-payment-basic.sh index 1eecb6ff..ab85d096 100755 --- a/code/chapter08/payment/test-payment-basic.sh +++ b/code/chapter08/payment/test-payment-basic.sh @@ -1,55 +1,120 @@ #!/bin/bash -# Simple test script for payment service +# Basic test script for payment service +# Tests all three fault-tolerant endpoints # Color definitions RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' NC='\033[0m' # No Color -# Check if bc is installed and install it if not +# Check if bc is installed if ! command -v bc &> /dev/null; then echo -e "${YELLOW}The 'bc' command is not found. Installing bc...${NC}" sudo apt-get update && sudo apt-get install -y bc - if [ $? -ne 0 ]; then - echo -e "${RED}Failed to install bc. Please install it manually.${NC}" - exit 1 - fi - echo -e "${GREEN}bc installed successfully.${NC}" fi -HOST="localhost" -PAYMENT_URL="http://${HOST}:9080/payment/api/authorize" +# Dynamically determine the base URL +if [ -n "$CODESPACE_NAME" ] && [ -n "$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN" ]; then + BASE_URL="http://localhost:9080/payment/api" + echo -e "${CYAN}Detected GitHub Codespaces environment${NC}" +elif [ -n "$GITPOD_WORKSPACE_URL" ]; then + GITPOD_HOST=$(echo $GITPOD_WORKSPACE_URL | sed 's|https://||' | sed 's|/||') + BASE_URL="https://9080-$GITPOD_HOST/payment/api" + echo -e "${CYAN}Detected Gitpod environment${NC}" +else + BASE_URL="http://localhost:9080/payment/api" + echo -e "${CYAN}Using local environment${NC}" +fi -# Hardcoded amount for testing -AMOUNT=25.99 +echo -e "${BLUE}=== Payment Service Basic Test ===${NC}" +echo -e "${CYAN}Base URL: $BASE_URL${NC}" +echo "" -echo -e "${YELLOW}Sending payment request for \$$AMOUNT...${NC}" +# Test 1: Payment Authorization +echo -e "${YELLOW}Test 1: Payment Authorization (Retry + Timeout + Fallback)${NC}" +echo -e "${CYAN}Endpoint: POST /authorize?amount=100${NC}" echo "" -# Capture start time start_time=$(date +%s.%N) +response=$(curl -s -X POST "${BASE_URL}/authorize?amount=100") +end_time=$(date +%s.%N) +duration=$(echo "$end_time - $start_time" | bc) -# Send request -response=$(curl -s -X POST "${PAYMENT_URL}?amount=${AMOUNT}") +echo -e "${GREEN}Response:${NC} $response" +echo -e "${GREEN}Duration:${NC} ${duration}s" +echo "" -# Capture end time -end_time=$(date +%s.%N) +if echo "$response" | grep -q "success"; then + echo -e "${GREEN}✓ SUCCESS: Payment authorized${NC}" +elif echo "$response" | grep -q "fallback"; then + echo -e "${YELLOW}⚠ FALLBACK: Used fallback response${NC}" +else + echo -e "${RED}✗ ERROR: Unexpected response${NC}" +fi -# Calculate duration +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Test 2: Gateway Health Check +echo -e "${YELLOW}Test 2: Gateway Health Check (Circuit Breaker + Timeout)${NC}" +echo -e "${CYAN}Endpoint: GET /health/gateway${NC}" +echo "" + +start_time=$(date +%s.%N) +response=$(curl -s -X GET "${BASE_URL}/health/gateway") +end_time=$(date +%s.%N) duration=$(echo "$end_time - $start_time" | bc) +echo -e "${GREEN}Response:${NC} $response" +echo -e "${GREEN}Duration:${NC} ${duration}s" echo "" -echo -e "${GREEN}✓ Request completed in ${duration} seconds${NC}" -echo -e "${YELLOW}Response:${NC} $response" + +if echo "$response" | grep -q "healthy"; then + echo -e "${GREEN}✓ SUCCESS: Gateway is healthy${NC}" +elif echo "$response" | grep -q "degraded"; then + echo -e "${YELLOW}⚠ WARNING: Gateway health degraded (Circuit Breaker may be OPEN)${NC}" +else + echo -e "${RED}✗ ERROR: Unexpected response${NC}" +fi + +echo "" +echo -e "${BLUE}----------------------------------------${NC}" echo "" -# Show if the response indicates success or fallback -if echo "$response" | grep -q "success"; then - echo -e "${GREEN}✓ SUCCESS: Payment processed successfully${NC}" -elif echo "$response" | grep -q "failed"; then - echo -e "${RED}✗ FALLBACK: Payment service unavailable${NC}" +# Test 3: Async Payment Notification +echo -e "${YELLOW}Test 3: Payment Notification (Async + Bulkhead + Timeout + Fallback)${NC}" +echo -e "${CYAN}Endpoint: POST /notify/PAY-12345${NC}" +echo "" + +start_time=$(date +%s.%N) +response=$(curl -s -X POST "${BASE_URL}/notify/PAY-12345") +end_time=$(date +%s.%N) +duration=$(echo "$end_time - $start_time" | bc) + +echo -e "${GREEN}Response:${NC} $response" +echo -e "${GREEN}Duration:${NC} ${duration}s" +echo "" + +if echo "$response" | grep -q "queued\|sent"; then + echo -e "${GREEN}✓ SUCCESS: Notification queued/sent${NC}" +elif echo "$response" | grep -q "fallback"; then + echo -e "${YELLOW}⚠ FALLBACK: Used fallback notification${NC}" else echo -e "${RED}✗ ERROR: Unexpected response${NC}" fi + +echo "" +echo -e "${BLUE}========================================${NC}" +echo -e "${GREEN}Basic tests complete!${NC}" +echo "" +echo -e "${CYAN}Next steps:${NC}" +echo -e " • Run ${YELLOW}test-payment-retry.sh${NC} to test retry behavior" +echo -e " • Run ${YELLOW}test-payment-circuit-breaker.sh${NC} to test circuit breaker states" +echo -e " • Run ${YELLOW}test-payment-bulkhead.sh${NC} to test bulkhead limits" +echo -e " • Check metrics at ${BLUE}http://localhost:9080/metrics?scope=base${NC}" +echo "" From 557e3ca6b0ebb9129024e9242e315be75a4aac9d Mon Sep 17 00:00:00 2001 From: Tarun Telang Date: Wed, 8 Apr 2026 00:34:38 +0530 Subject: [PATCH 06/12] Refactor payment bulkhead test script for clarity Updated the test script for Payment Service Bulkhead functionality to clarify its purpose and improve output messages. Adjusted environment detection and added bulkhead configuration details. --- .../payment/test-payment-bulkhead.sh | 186 +++++++++++++++--- 1 file changed, 164 insertions(+), 22 deletions(-) diff --git a/code/chapter08/payment/test-payment-bulkhead.sh b/code/chapter08/payment/test-payment-bulkhead.sh index 86199af5..2e26d6e4 100755 --- a/code/chapter08/payment/test-payment-bulkhead.sh +++ b/code/chapter08/payment/test-payment-bulkhead.sh @@ -1,8 +1,7 @@ #!/bin/bash # Test script for Payment Service Bulkhead functionality -# This script demonstrates the @Bulkhead annotation by sending many concurrent requests -# and observing how the service handles concurrent load up to its configured limit (5) +# Tests the @Bulkhead annotation on the sendPaymentNotification method # Color definitions RED='\033[0;31m' @@ -13,45 +12,188 @@ PURPLE='\033[0;35m' CYAN='\033[0;36m' NC='\033[0m' # No Color -# Check if bc is installed and install it if not +# Check if bc is installed if ! command -v bc &> /dev/null; then - echo -e "${YELLOW}The 'bc' command is not found. Installing bc...${NC}" + echo -e "${YELLOW}Installing bc for duration calculations...${NC}" sudo apt-get update && sudo apt-get install -y bc - if [ $? -ne 0 ]; then - echo -e "${RED}Failed to install bc. Please install it manually.${NC}" - exit 1 - fi - echo -e "${GREEN}bc installed successfully.${NC}" fi -# Header -echo -e "${BLUE}==============================================${NC}" -echo -e "${BLUE} Payment Service Bulkhead Test ${NC}" -echo -e "${BLUE}==============================================${NC}" -echo "" - # Dynamically determine the base URL if [ -n "$CODESPACE_NAME" ] && [ -n "$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN" ]; then - # GitHub Codespaces environment - use localhost for internal testing BASE_URL="http://localhost:9080/payment/api" - echo -e "${CYAN}Detected GitHub Codespaces environment (using localhost)${NC}" - echo -e "${CYAN}Note: External access would be https://$CODESPACE_NAME-9080.$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN/payment/api${NC}" + METRICS_URL="http://localhost:9080/metrics" + echo -e "${CYAN}Detected GitHub Codespaces environment${NC}" elif [ -n "$GITPOD_WORKSPACE_URL" ]; then - # Gitpod environment GITPOD_HOST=$(echo $GITPOD_WORKSPACE_URL | sed 's|https://||' | sed 's|/||') BASE_URL="https://9080-$GITPOD_HOST/payment/api" + METRICS_URL="https://9080-$GITPOD_HOST/metrics" echo -e "${CYAN}Detected Gitpod environment${NC}" else - # Local or other environment BASE_URL="http://localhost:9080/payment/api" + METRICS_URL="http://localhost:9080/metrics" echo -e "${CYAN}Using local environment${NC}" fi +echo "" +echo -e "${BLUE}=== Payment Notification Bulkhead Test ===${NC}" +echo -e "${CYAN}Endpoint: POST /notify/{paymentId}${NC}" echo -e "${CYAN}Base URL: $BASE_URL${NC}" echo "" -# Set up log monitoring -LOG_FILE="/workspaces/liberty-rest-app/payment/target/liberty/wlp/usr/servers/mpServer/logs/messages.log" +echo -e "${YELLOW}Bulkhead Configuration:${NC}" +echo " • Maximum Concurrent Requests: 3" +echo " • Waiting Queue Size: 2" +echo " • Total Capacity: 5 (3 concurrent + 2 queued)" +echo " • Asynchronous: Yes (@Asynchronous)" +echo " • Processing delay: ~2 seconds per notification" +echo "" + +echo -e "${CYAN}🔍 Expected Behavior:${NC}" +echo " • First 3 requests: Execute immediately (concurrent slots)" +echo " • Next 2 requests: Queue for execution (waiting queue)" +echo " • 6th+ requests: ${RED}REJECTED${NC} - Bulkhead full" +echo "" + +read -p "Press Enter to start the bulkhead test..." +echo "" + +# Function to send async notification request +send_notification() { + local id=$1 + local payment_id="PAY-$(printf "%05d" $id)" + + start_time=$(date +%s.%N) + response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" -X POST "${BASE_URL}/notify/${payment_id}" 2>/dev/null || echo "HTTP_STATUS:000") + end_time=$(date +%s.%N) + + duration=$(echo "$end_time - $start_time" | bc) + duration_formatted=$(printf "%.3f" $duration) + + http_code=$(echo "$response" | grep "HTTP_STATUS:" | cut -d: -f2) + body=$(echo "$response" | sed '/HTTP_STATUS:/d') + + if [ "$http_code" -eq 200 ] || [ "$http_code" -eq 202 ]; then + if echo "$body" | grep -q "queued\|sent"; then + echo -e "${GREEN}[Request $id] ✓ Accepted (${duration_formatted}s) - Response: $body${NC}" + elif echo "$body" | grep -q "fallback"; then + echo -e "${PURPLE}[Request $id] ⚡ Fallback (${duration_formatted}s) - Response: $body${NC}" + else + echo -e "${GREEN}[Request $id] ✓ Success (${duration_formatted}s) - Response: $body${NC}" + fi + elif echo "$body" | grep -q "BulkheadException\|Bulkhead"; then + echo -e "${RED}[Request $id] ✗ REJECTED: Bulkhead full (${duration_formatted}s)${NC}" + elif [ "$http_code" -eq 503 ]; then + echo -e "${YELLOW}[Request $id] ⚠ Service Unavailable (${duration_formatted}s)${NC}" + else + echo -e "${PURPLE}[Request $id] ? Unknown (HTTP $http_code, ${duration_formatted}s): $body${NC}" + fi +} + +# Phase 1: Single request baseline +echo -e "${BLUE}=== Phase 1: Single Request (Baseline) ===${NC}" +echo -e "${CYAN}Establishing baseline performance...${NC}" +echo "" + +send_notification 1 +sleep 3 + +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Phase 2: Test bulkhead capacity +echo -e "${BLUE}=== Phase 2: Testing Bulkhead Capacity (10 concurrent requests) ===${NC}" +echo -e "${CYAN}Sending 10 concurrent requests...${NC}" +echo -e "${YELLOW}Expected:${NC}" +echo -e " • Requests 1-3: ${GREEN}Execute immediately (concurrent slots)${NC}" +echo -e " • Requests 4-5: ${CYAN}Queued (waiting queue)${NC}" +echo -e " • Requests 6-10: ${RED}REJECTED (bulkhead full)${NC}" +echo "" + +# Send 10 requests concurrently in background +for i in {1..10}; do + send_notification $i & + sleep 0.1 # Small delay to stagger requests slightly +done + +# Wait for all background jobs to complete +wait + +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Phase 3: Check metrics +echo -e "${BLUE}=== Phase 3: Bulkhead Metrics ===${NC}" +echo -e "${CYAN}Fetching fault tolerance metrics...${NC}" +echo "" + +sleep 1 +metrics=$(curl -s "$METRICS_URL" 2>/dev/null | grep -i "bulkhead.*sendPaymentNotification") + +if [ -n "$metrics" ]; then + echo "$metrics" | while IFS= read -r line; do + if echo "$line" | grep -q "callsAccepted\|accepted"; then + echo -e "${GREEN}$line${NC}" + elif echo "$line" | grep -q "callsRejected\|rejected"; then + echo -e "${RED}$line${NC}" + elif echo "$line" | grep -q "executionDuration\|duration\|running"; then + echo -e "${CYAN}$line${NC}" + else + echo "$line" + fi + done +else + echo -e "${YELLOW}No bulkhead metrics found${NC}" + echo -e "${CYAN}Make sure the service is running and has processed some requests${NC}" + echo -e "${CYAN}Checking all available fault tolerance metrics...${NC}" + echo "" + all_ft_metrics=$(curl -s "$METRICS_URL" 2>/dev/null | grep -i "ft.*bulkhead" | head -20) + if [ -n "$all_ft_metrics" ]; then + echo -e "${BLUE}Available bulkhead metrics:${NC}" + echo "$all_ft_metrics" + else + echo -e "${YELLOW}No fault tolerance bulkhead metrics found at all${NC}" + fi +fi + +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Phase 4: Verify recovery after wait +echo -e "${BLUE}=== Phase 4: Testing Recovery ===${NC}" +echo -e "${CYAN}Waiting 3 seconds for in-progress notifications to complete...${NC}" +sleep 3 + +echo "" +echo -e "${CYAN}Sending new requests to verify bulkhead has freed up...${NC}" +echo "" + +for i in {11..13}; do + send_notification $i + sleep 0.5 +done + +echo "" +echo -e "${GREEN}=== Bulkhead Test Complete ===${NC}" +echo "" + +echo -e "${CYAN}Summary:${NC}" +echo " • Bulkhead limits concurrent async executions" +echo " • Requests beyond capacity are rejected immediately" +echo " • Protects system resources from exhaustion" +echo " • Works with @Asynchronous for non-blocking behavior" +echo "" + +echo -e "${CYAN}To view all bulkhead metrics:${NC}" +echo -e "${BLUE}curl $METRICS_URL | grep -i bulkhead${NC}" +echo "" +echo "" + +# Set up log monitoring - use relative path from script location +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LOG_FILE="$SCRIPT_DIR/target/liberty/wlp/usr/servers/mpServer/logs/messages.log" # Get initial log position for later analysis if [ -f "$LOG_FILE" ]; then From 004a43b9251ac184a7dab21fc2f23ad125501813 Mon Sep 17 00:00:00 2001 From: Tarun Telang Date: Wed, 8 Apr 2026 00:36:03 +0530 Subject: [PATCH 07/12] Add test script for Payment Service Circuit Breaker This script tests the Circuit Breaker functionality of the Payment Service by simulating requests and monitoring the circuit states. --- .../payment/test-payment-circuit-breaker.sh | 319 ++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 code/chapter08/payment/test-payment-circuit-breaker.sh diff --git a/code/chapter08/payment/test-payment-circuit-breaker.sh b/code/chapter08/payment/test-payment-circuit-breaker.sh new file mode 100644 index 00000000..e11a20de --- /dev/null +++ b/code/chapter08/payment/test-payment-circuit-breaker.sh @@ -0,0 +1,319 @@ +#!/bin/bash + +# Test script for Payment Service Circuit Breaker functionality +# Tests the @CircuitBreaker annotation on the checkGatewayHealth method + +# Color definitions +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +PURPLE='\033[0;35m' +NC='\033[0m' # No Color + +# Dynamically determine the base URL +if [ -n "$CODESPACE_NAME" ] && [ -n "$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN" ]; then + BASE_URL="http://localhost:9080/payment/api" + METRICS_URL="http://localhost:9080/metrics?scope=base" + echo -e "${CYAN}Detected GitHub Codespaces environment${NC}" +elif [ -n "$GITPOD_WORKSPACE_URL" ]; then + GITPOD_HOST=$(echo $GITPOD_WORKSPACE_URL | sed 's|https://||' | sed 's|/||') + BASE_URL="https://9080-$GITPOD_HOST/payment/api" + METRICS_URL="https://9080-$GITPOD_HOST/metrics?scope=base" + echo -e "${CYAN}Detected Gitpod environment${NC}" +else + BASE_URL="http://localhost:9080/payment/api" + METRICS_URL="http://localhost:9080/metrics?scope=base" + echo -e "${CYAN}Using local environment${NC}" +fi + +echo "" +echo -e "${BLUE}=== Payment Gateway Circuit Breaker Test ===${NC}" +echo -e "${CYAN}Endpoint: GET /health/gateway${NC}" +echo -e "${CYAN}Base URL: $BASE_URL${NC}" +echo "" + +echo -e "${YELLOW}Circuit Breaker Configuration:${NC}" +echo " • Request Volume Threshold: 4" +echo " • Failure Ratio: 0.5 (50%)" +echo " • Delay: 5000ms" +echo " • Success Threshold: 2" +echo " • Simulated failure rate: 50%" +echo "" + +echo -e "${CYAN}🔍 Circuit Breaker States:${NC}" +echo -e " • ${GREEN}CLOSED${NC} (0): Normal operation, requests pass through" +echo -e " • ${RED}OPEN${NC} (1): Too many failures, requests immediately fail" +echo -e " • ${YELLOW}HALF_OPEN${NC} (2): Testing if service recovered" +echo "" + +echo -e "${PURPLE}This test will:${NC}" +echo " 1. Send initial requests to establish baseline" +echo " 2. Trigger failures to trip the circuit breaker to OPEN" +echo " 3. Wait for delay period (5s)" +echo " 4. Test HALF_OPEN state (probing for recovery)" +echo " 5. Verify circuit returns to CLOSED on success" +echo "" + +read -p "Press Enter to start the circuit breaker test..." +echo "" + +# Function to check circuit breaker state from metrics +check_circuit_state() { + # Fetch all metrics + metrics=$(curl -s "$METRICS_URL" 2>/dev/null) + + # Try to find circuit breaker state metric (MicroProfile FT uses labels for method name) + # Look for ft_circuitbreaker_state_total_nanoseconds with checkGatewayHealth in method label + state_metrics=$(echo "$metrics" | grep "ft_circuitbreaker_state_total" | grep "checkGatewayHealth") + + if [ -z "$state_metrics" ]; then + # Check if we can reach the metrics endpoint at all + if [ -z "$metrics" ]; then + echo -e "${RED}✗ Unable to reach metrics endpoint${NC}" + echo -e "${YELLOW} Make sure the payment service is running on port 9080${NC}" + else + # Metrics endpoint works, but circuit breaker metrics not found + echo -e "${YELLOW}⚠ Circuit breaker metrics not yet available${NC}" + echo -e "${CYAN} (This is normal if no health check requests have been made yet)${NC}" + fi + return + fi + + # Determine current state by checking which state has recent activity + # The state with the highest time value is likely the current state + closed_time=$(echo "$state_metrics" | grep 'state="closed"' | grep -o '[0-9.E+]*$' || echo "0") + open_time=$(echo "$state_metrics" | grep 'state="open"' | grep -o '[0-9.E+]*$' || echo "0") + halfopen_time=$(echo "$state_metrics" | grep 'state="halfOpen"' | grep -o '[0-9.E+]*$' || echo "0") + + # Simple heuristic: Check if open or halfOpen have recent values + if echo "$state_metrics" | grep -q 'state="open"' && [ "$open_time" != "0" ]; then + # Check if we're in half-open by looking at recent halfOpen time + if echo "$state_metrics" | grep -q 'state="halfOpen"' && [ "$halfopen_time" != "0" ]; then + echo -e "${YELLOW}Circuit Breaker State: HALF_OPEN (2) - Testing recovery${NC}" + else + echo -e "${RED}Circuit Breaker State: OPEN (1) - Blocking requests${NC}" + fi + else + echo -e "${GREEN}Circuit Breaker State: CLOSED (0) - Normal operation${NC}" + fi +} + +# Function to make health check request +make_health_request() { + local request_num=$1 + + start_time=$(date +%s.%N) + response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" -X GET "${BASE_URL}/health/gateway" 2>/dev/null || echo "HTTP_STATUS:000") + end_time=$(date +%s.%N) + + duration=$(echo "$end_time - $start_time" | bc) + duration_formatted=$(printf "%.3f" $duration) + + http_code=$(echo "$response" | grep "HTTP_STATUS:" | cut -d: -f2) + body=$(echo "$response" | sed '/HTTP_STATUS:/d') + + if echo "$body" | grep -q "\"status\":\"healthy\""; then + echo -e "${GREEN}[Request $request_num] ✓ Gateway healthy (${duration_formatted}s)${NC}" + elif echo "$body" | grep -q "\"status\":\"unhealthy\""; then + echo -e "${YELLOW}[Request $request_num] ⚠ Gateway unhealthy (${duration_formatted}s)${NC}" + elif echo "$body" | grep -q "\"status\":\"circuit_open\""; then + echo -e "${RED}[Request $request_num] ✗ Circuit OPEN - Request blocked (${duration_formatted}s)${NC}" + else + echo -e "${PURPLE}[Request $request_num] ? Unknown status (${duration_formatted}s): $body${NC}" + fi +} + +# Phase 0: Ensure circuit is CLOSED before starting test +echo -e "${BLUE}=== Phase 0: Initialization ===${NC}" +echo -e "${CYAN}Checking current circuit breaker state...${NC}" +echo "" + +# Check initial state and wait if needed +metrics=$(curl -s "$METRICS_URL" 2>/dev/null) +state_metrics=$(echo "$metrics" | grep "ft_circuitbreaker_state_total" | grep "checkGatewayHealth") + +if [ -z "$state_metrics" ]; then + echo -e "${YELLOW}⚠ Circuit breaker metrics not yet available${NC}" + echo -e "${CYAN}Sending initial requests to initialize circuit breaker...${NC}" + # Send a few requests to initialize the circuit breaker + for i in {1..2}; do + curl -s "${BASE_URL}/health/gateway" > /dev/null 2>&1 + sleep 0.5 + done + sleep 1 + echo -e "${GREEN}✓ Circuit breaker initialized${NC}" +elif echo "$state_metrics" | grep -q 'state="open"'; then + # Check if there's significant open time (circuit is OPEN) + open_time=$(echo "$state_metrics" | grep 'state="open"' | grep -o '[0-9.E+]*$') + if [ -n "$open_time" ] && [ "$open_time" != "0" ]; then + echo -e "${YELLOW}⚠ Circuit is currently OPEN (from previous test run)${NC}" + echo -e "${CYAN}Waiting 6 seconds for circuit to transition to HALF_OPEN...${NC}" + for i in {6..1}; do + echo -ne "${CYAN} Waiting... $i seconds\r${NC}" + sleep 1 + done + echo "" + echo -e "${CYAN}Sending probe requests to close the circuit...${NC}" + # Send a few successful probes to close the circuit + for i in {1..3}; do + curl -s "${BASE_URL}/health/gateway" > /dev/null 2>&1 + sleep 0.5 + done + sleep 2 + echo -e "${GREEN}✓ Circuit should now be CLOSED${NC}" + fi +elif echo "$state_metrics" | grep -q 'state="halfOpen"'; then + halfopen_time=$(echo "$state_metrics" | grep 'state="halfOpen"' | grep -o '[0-9.E+]*$') + if [ -n "$halfopen_time" ] && [ "$halfopen_time" != "0" ]; then + echo -e "${YELLOW}⚠ Circuit is currently HALF_OPEN${NC}" + echo -e "${CYAN}Sending probe requests to close the circuit...${NC}" + for i in {1..3}; do + curl -s "${BASE_URL}/health/gateway" > /dev/null 2>&1 + sleep 0.5 + done + sleep 1 + echo -e "${GREEN}✓ Circuit should now be CLOSED${NC}" + fi +else + echo -e "${GREEN}✓ Circuit is already CLOSED${NC}" +fi + +echo "" +check_circuit_state +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Phase 1: Initial requests (circuit should be CLOSED) +echo -e "${BLUE}=== Phase 1: Initial Requests (Circuit: CLOSED) ===${NC}" +echo -e "${CYAN}Sending initial requests to establish baseline...${NC}" +echo "" + +for i in {1..3}; do + make_health_request $i + sleep 0.5 +done + +echo "" +check_circuit_state +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Phase 2: Trigger failures to OPEN the circuit +echo -e "${BLUE}=== Phase 2: Triggering Circuit Breaker (CLOSED → OPEN) ===${NC}" +echo -e "${CYAN}Sending requests to trigger failures...${NC}" +echo -e "${YELLOW}Given 50% failure rate, we expect some failures${NC}" +echo "" + +for i in {4..10}; do + make_health_request $i + sleep 0.3 +done + +echo "" +check_circuit_state +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Phase 3: Circuit should be OPEN, requests should fail immediately +echo -e "${BLUE}=== Phase 3: Verify Circuit OPEN (Blocking Requests) ===${NC}" +echo -e "${CYAN}These requests should fail immediately without hitting the service...${NC}" +echo "" + +for i in {11..13}; do + make_health_request $i + sleep 0.5 +done + +echo "" +check_circuit_state +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Phase 4: Wait for delay period to transition to HALF_OPEN +echo -e "${BLUE}=== Phase 4: Waiting for Recovery Period ===${NC}" +echo -e "${CYAN}Waiting 5 seconds for circuit breaker delay...${NC}" +echo -e "${YELLOW}Circuit should transition from OPEN → HALF_OPEN${NC}" +echo "" + +for i in {5..1}; do + echo -ne "${CYAN}Waiting... $i seconds remaining\r${NC}" + sleep 1 +done +echo "" + +echo "" +check_circuit_state +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Phase 5: Test HALF_OPEN state +echo -e "${BLUE}=== Phase 5: Testing HALF_OPEN State ===${NC}" +echo -e "${CYAN}Circuit should allow probe requests to test recovery...${NC}" +echo -e "${YELLOW}Need 2 consecutive successes to return to CLOSED${NC}" +echo "" + +for i in {14..18}; do + make_health_request $i + sleep 1 + check_circuit_state + echo "" +done + +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Final state check +echo -e "${BLUE}=== Final Circuit Breaker State ===${NC}" +check_circuit_state +echo "" + +# Show metrics +echo -e "${BLUE}=== Circuit Breaker Metrics ===${NC}" +echo -e "${CYAN}Fetching fault tolerance metrics...${NC}" +echo "" + +# Use flexible pattern matching for metrics +all_metrics=$(curl -s "$METRICS_URL" 2>/dev/null) +metrics=$(echo "$all_metrics" | grep -i "circuitbreaker" | grep -i "checkGatewayHealth") + +if [ -n "$metrics" ]; then + echo "$metrics" | while IFS= read -r line; do + if echo "$line" | grep -q "opened"; then + echo -e "${YELLOW}$line${NC}" + elif echo "$line" | grep -q "state"; then + echo -e "${GREEN}$line${NC}" + elif echo "$line" | grep -q "succeeded\|failed"; then + echo -e "${CYAN}$line${NC}" + else + echo "$line" + fi + done +else + # Try alternative metric patterns + alt_metrics=$(echo "$all_metrics" | grep -E "ft.*Health.*circuit|circuit.*Health") + if [ -n "$alt_metrics" ]; then + echo -e "${CYAN}Found alternative circuit breaker metrics:${NC}" + echo "$alt_metrics" + else + echo -e "${RED}✗ No circuit breaker metrics found${NC}" + echo -e "${YELLOW}This could mean:${NC}" + echo -e "${YELLOW} • The service is not running${NC}" + echo -e "${YELLOW} • Metrics are not enabled${NC}" + echo -e "${YELLOW} • No health check requests were processed successfully${NC}" + fi +fi + +echo "" +echo -e "${GREEN}=== Circuit Breaker Test Complete ===${NC}" +echo "" +echo -e "${CYAN}To view all fault tolerance metrics:${NC}" +echo -e "${BLUE}curl $METRICS_URL | grep -i circuit${NC}" +echo "" From a8b8bd454995f3d373ec6362b8aa12f6e9b12498 Mon Sep 17 00:00:00 2001 From: Tarun Telang Date: Wed, 8 Apr 2026 00:36:54 +0530 Subject: [PATCH 08/12] Revise payment load test for comprehensive fault tolerance Updated the payment load test script to comprehensively test all fault tolerance patterns including retry, circuit breaker, bulkhead, and asynchronous processing under concurrent load. --- .../payment/test-payment-concurrent-load.sh | 393 ++++++++++++++---- 1 file changed, 308 insertions(+), 85 deletions(-) diff --git a/code/chapter08/payment/test-payment-concurrent-load.sh b/code/chapter08/payment/test-payment-concurrent-load.sh index 94452a42..70fabde6 100755 --- a/code/chapter08/payment/test-payment-concurrent-load.sh +++ b/code/chapter08/payment/test-payment-concurrent-load.sh @@ -1,10 +1,7 @@ #!/bin/bash -# Test script for asynchronous payment processing -# This script sends multiple concurrent requests to test: -# 1. Asynchronous processing (@Asynchronous) -# 2. Resource isolation (@Bulkhead) -# 3. Retry behavior (@Retry) +# Comprehensive load test for all fault tolerance patterns +# Tests retry, circuit breaker, bulkhead, and async under concurrent load # Color definitions RED='\033[0;31m' @@ -15,109 +12,335 @@ PURPLE='\033[0;35m' CYAN='\033[0;36m' NC='\033[0m' # No Color -# Check if bc is installed and install it if not +# Check if bc is installed if ! command -v bc &> /dev/null; then - echo -e "${YELLOW}The 'bc' command is not found. Installing bc...${NC}" + echo -e "${YELLOW}Installing bc...${NC}" sudo apt-get update && sudo apt-get install -y bc - if [ $? -ne 0 ]; then - echo -e "${RED}Failed to install bc. Please install it manually.${NC}" - exit 1 - fi - echo -e "${GREEN}bc installed successfully.${NC}" fi -# Set payment endpoint URL - automatically detect if running in GitHub Codespaces -if [ -n "$CODESPACES" ]; then - HOST="localhost" +# Dynamically determine the base URL +if [ -n "$CODESPACE_NAME" ] && [ -n "$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN" ]; then + BASE_URL="http://localhost:9080/payment/api" + METRICS_URL="http://localhost:9080/metrics" + echo -e "${CYAN}Detected GitHub Codespaces environment${NC}" +elif [ -n "$GITPOD_WORKSPACE_URL" ]; then + GITPOD_HOST=$(echo $GITPOD_WORKSPACE_URL | sed 's|https://||' | sed 's|/||') + BASE_URL="https://9080-$GITPOD_HOST/payment/api" + METRICS_URL="https://9080-$GITPOD_HOST/metrics" + echo -e "${CYAN}Detected Gitpod environment${NC}" else - HOST="localhost" + BASE_URL="http://localhost:9080/payment/api" + METRICS_URL="http://localhost:9080/metrics" + echo -e "${CYAN}Using local environment${NC}" fi -PAYMENT_URL="http://${HOST}:9080/payment/api/authorize" -NUM_REQUESTS=10 -CONCURRENCY=5 +echo "" +echo -e "${BLUE}╔════════════════════════════════════════════════╗${NC}" +echo -e "${BLUE}║ Payment Service Concurrent Load Test ║${NC}" +echo -e "${BLUE}║ Testing All Fault Tolerance Patterns ║${NC}" +echo -e "${BLUE}╚════════════════════════════════════════════════╝${NC}" +echo "" +echo -e "${CYAN}Base URL: $BASE_URL${NC}" +echo "" + +# Test configuration +NUM_AUTHORIZE_REQUESTS=20 +NUM_HEALTH_REQUESTS=15 +NUM_NOTIFY_REQUESTS=15 -echo -e "${BLUE}=== Testing Asynchronous Payment Processing ===${NC}" -echo -e "${CYAN}Endpoint: ${PAYMENT_URL}${NC}" -echo -e "${CYAN}Sending ${NUM_REQUESTS} requests with concurrency ${CONCURRENCY}${NC}" +echo -e "${YELLOW}Load Test Configuration:${NC}" +echo " • Authorization requests (Retry): $NUM_AUTHORIZE_REQUESTS" +echo " • Health check requests (Circuit Breaker): $NUM_HEALTH_REQUESTS" +echo " • Notification requests (Async + Bulkhead): $NUM_NOTIFY_REQUESTS" +echo "" +echo -e "${CYAN}Expected Behavior:${NC}" +echo -e "${CYAN} Phase 1 (Retry): Some failures will be retried and eventually succeed${NC}" +echo -e "${CYAN} Phase 2 (Circuit Breaker): Circuit will OPEN after 50% failures, then recover${NC}" +echo -e "${CYAN} Phase 3 (Bulkhead): Requests queued up to limit, excess rejected${NC}" echo "" -# Function to send a single request and capture timing -send_request() { +read -p "Press Enter to start the load test..." +echo "" + +# Counters +declare -A success_count=( ["authorize"]=0 ["health"]=0 ["notify"]=0 ) +declare -A failure_count=( ["authorize"]=0 ["health"]=0 ["notify"]=0 ) +declare -A fallback_count=( ["authorize"]=0 ["health"]=0 ["notify"]=0 ) + +# Function to send authorization request +send_authorize_request() { local id=$1 - local amount=$2 - local start_time=$(date +%s) + local amount=$((50 + RANDOM % 450)) - echo -e "${YELLOW}[Request $id] Sending payment request for \$$amount...${NC}" + response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" \ + -X POST "${BASE_URL}/authorize?amount=${amount}" 2>/dev/null || echo "HTTP_STATUS:000") - # Send request and capture response - response=$(curl -s -X POST "${PAYMENT_URL}?amount=${amount}") + http_code=$(echo "$response" | grep "HTTP_STATUS:" | cut -d: -f2) + body=$(echo "$response" | sed '/HTTP_STATUS:/d') - local end_time=$(date +%s) - local duration=$((end_time - start_time)) - - # Check if response contains "success" or "failed" - if echo "$response" | grep -q "success"; then - echo -e "${GREEN}[Request $id] SUCCESS: Payment processed in ${duration}s${NC}" - echo -e "${GREEN}[Request $id] Response: $response${NC}" - elif echo "$response" | grep -q "failed"; then - echo -e "${RED}[Request $id] FALLBACK: Service unavailable (took ${duration}s)${NC}" - echo -e "${RED}[Request $id] Response: $response${NC}" + if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then + if echo "$body" | grep -q "fallback\|pending"; then + echo -e "${YELLOW}[AUTH-$id] ⚡ Fallback${NC}" + echo "fallback" > /tmp/auth_result_$id.txt + else + echo -e "${GREEN}[AUTH-$id] ✓ Success${NC}" + echo "success" > /tmp/auth_result_$id.txt + fi else - echo -e "${RED}[Request $id] ERROR: Unexpected response (took ${duration}s)${NC}" - echo -e "${RED}[Request $id] Response: $response${NC}" + echo -e "${RED}[AUTH-$id] ✗ Failed${NC}" + echo "failed" > /tmp/auth_result_$id.txt fi - - # Return the duration for analysis - echo "$duration" } -# Run concurrent requests using GNU Parallel if available, or background processes if not -if command -v parallel > /dev/null; then - echo -e "${PURPLE}Using GNU Parallel for concurrent requests${NC}" - export -f send_request - export PAYMENT_URL RED GREEN YELLOW BLUE PURPLE CYAN NC +# Function to send health check request +send_health_request() { + local id=$1 - # Use predefined amounts instead of bc calculation - amounts=("15.99" "24.50" "19.95" "32.75" "12.99" "22.50" "18.75" "29.99" "14.50" "27.25") - for i in $(seq 1 $NUM_REQUESTS); do - amount_index=$((i-1 % 10)) - amount=${amounts[$amount_index]} - send_request $i $amount & - done -else - echo -e "${PURPLE}Running concurrent requests using background processes${NC}" - # Store PIDs - pids=() + response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" \ + -X GET "${BASE_URL}/health/gateway" 2>/dev/null || echo "HTTP_STATUS:000") - # Predefined amounts - amounts=("15.99" "24.50" "19.95" "32.75" "12.99" "22.50" "18.75" "29.99" "14.50" "27.25") + http_code=$(echo "$response" | grep "HTTP_STATUS:" | cut -d: -f2) + body=$(echo "$response" | sed '/HTTP_STATUS:/d') - # Launch requests in background - for i in $(seq 1 $NUM_REQUESTS); do - # Get amount from predefined list - amount_index=$((i-1 % 10)) - amount=${amounts[$amount_index]} - send_request $i $amount & - pids+=($!) - - # Control concurrency - if [ ${#pids[@]} -ge $CONCURRENCY ]; then - # Wait for one process to finish before starting more - wait "${pids[0]}" - pids=("${pids[@]:1}") + if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then + if echo "$body" | grep -q "healthy"; then + echo -e "${GREEN}[HEALTH-$id] ✓ Healthy${NC}" + echo "success" > /tmp/health_result_$id.txt + else + echo -e "${YELLOW}[HEALTH-$id] ⚠ Degraded${NC}" + echo "fallback" > /tmp/health_result_$id.txt fi - done + elif [ "$http_code" -eq 503 ]; then + if echo "$body" | grep -q "circuit_open"; then + echo -e "${PURPLE}[HEALTH-$id] ⚡ Circuit Breaker OPEN${NC}" + echo "failed" > /tmp/health_result_$id.txt + elif echo "$body" | grep -q "unhealthy"; then + echo -e "${YELLOW}[HEALTH-$id] ⚠ Gateway Unhealthy${NC}" + echo "fallback" > /tmp/health_result_$id.txt + else + echo -e "${RED}[HEALTH-$id] ✗ Service Unavailable${NC}" + echo "failed" > /tmp/health_result_$id.txt + fi + else + echo -e "${RED}[HEALTH-$id] ✗ Failed (HTTP $http_code)${NC}" + echo "failed" > /tmp/health_result_$id.txt + fi +} + +# Function to send notification request +send_notify_request() { + local id=$1 + local payment_id=$(printf "PAY-%05d" $id) + + response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" \ + -X POST "${BASE_URL}/notify/${payment_id}" 2>/dev/null || echo "HTTP_STATUS:000") + + http_code=$(echo "$response" | grep "HTTP_STATUS:" | cut -d: -f2) + body=$(echo "$response" | sed '/HTTP_STATUS:/d') - # Wait for remaining processes - for pid in "${pids[@]}"; do - wait $pid - done + if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then + if echo "$body" | grep -q "fallback"; then + echo -e "${YELLOW}[NOTIFY-$id] ⚡ Fallback${NC}" + echo "fallback" > /tmp/notify_result_$id.txt + else + echo -e "${GREEN}[NOTIFY-$id] ✓ Queued${NC}" + echo "success" > /tmp/notify_result_$id.txt + fi + else + if echo "$body" | grep -q "Bulkhead"; then + echo -e "${RED}[NOTIFY-$id] ✗ Rejected (Bulkhead)${NC}" + else + echo -e "${RED}[NOTIFY-$id] ✗ Failed${NC}" + fi + echo "failed" > /tmp/notify_result_$id.txt + fi +} + +# Phase 1: Concurrent authorization requests +echo -e "${BLUE}=== Phase 1: Authorization Load Test (Retry Pattern) ===${NC}" +echo -e "${CYAN}Sending $NUM_AUTHORIZE_REQUESTS concurrent authorization requests...${NC}" +echo "" + +start_time=$(date +%s.%N) + +for i in $(seq 1 $NUM_AUTHORIZE_REQUESTS); do + send_authorize_request $i & + sleep 0.05 +done + +wait + +end_time=$(date +%s.%N) +duration=$(echo "$end_time - $start_time" | bc) +duration_formatted=$(printf "%.2f" $duration) + +# Collect results from temp files +for i in $(seq 1 $NUM_AUTHORIZE_REQUESTS); do + if [ -f "/tmp/auth_result_$i.txt" ]; then + result=$(cat /tmp/auth_result_$i.txt) + case $result in + success) ((success_count["authorize"]++)) ;; + fallback) ((fallback_count["authorize"]++)) ;; + failed) ((failure_count["authorize"]++)) ;; + esac + rm /tmp/auth_result_$i.txt + fi +done + +echo "" +echo -e "${PURPLE}Authorization Phase Complete: ${duration_formatted}s${NC}" +echo -e "${GREEN}✓ Success: ${success_count["authorize"]}${NC}" +echo -e "${YELLOW}⚡ Fallback: ${fallback_count["authorize"]}${NC}" +echo -e "${RED}✗ Failed: ${failure_count["authorize"]}${NC}" +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +sleep 2 + +# Phase 2: Health check requests (circuit breaker) +echo -e "${BLUE}=== Phase 2: Health Check Load Test (Circuit Breaker) ===${NC}" +echo -e "${CYAN}Sending $NUM_HEALTH_REQUESTS health check requests (sequential to observe circuit breaker)...${NC}" +echo -e "${CYAN}Watch for circuit breaker to OPEN after failures, then potentially CLOSE after recovery${NC}" +echo "" + +start_time=$(date +%s.%N) + +# Send requests sequentially to better observe circuit breaker behavior +for i in $(seq 1 $NUM_HEALTH_REQUESTS); do + send_health_request $i + sleep 0.5 # Small delay to observe circuit breaker state changes +done + +end_time=$(date +%s.%N) +duration=$(echo "$end_time - $start_time" | bc) +duration_formatted=$(printf "%.2f" $duration) + +# Collect results from temp files +for i in $(seq 1 $NUM_HEALTH_REQUESTS); do + if [ -f "/tmp/health_result_$i.txt" ]; then + result=$(cat /tmp/health_result_$i.txt) + case $result in + success) ((success_count["health"]++)) ;; + fallback) ((fallback_count["health"]++)) ;; + failed) ((failure_count["health"]++)) ;; + esac + rm /tmp/health_result_$i.txt + fi +done + +echo "" +echo -e "${PURPLE}Health Check Phase Complete: ${duration_formatted}s${NC}" +echo -e "${GREEN}✓ Healthy: ${success_count["health"]}${NC}" +echo -e "${YELLOW}⚠ Degraded/Unhealthy: ${fallback_count["health"]}${NC}" +echo -e "${RED}✗ Failed/Circuit Open: ${failure_count["health"]}${NC}" +echo "" +echo -e "${CYAN}Circuit Breaker Behavior:${NC}" +echo -e "${CYAN}• Opens at 50% failure rate (after 4 requests)${NC}" +echo -e "${CYAN}• Simulated gateway has 60% failure rate${NC}" +echo -e "${CYAN}• Circuit stays open for 5 seconds, then tries to recover${NC}" +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +sleep 2 + +# Phase 3: Notification requests (bulkhead + async) +echo -e "${BLUE}=== Phase 3: Notification Load Test (Bulkhead + Async) ===${NC}" +echo -e "${CYAN}Sending $NUM_NOTIFY_REQUESTS concurrent notification requests...${NC}" +echo "" + +start_time=$(date +%s.%N) + +for i in $(seq 1 $NUM_NOTIFY_REQUESTS); do + send_notify_request $i & + sleep 0.05 +done + +wait + +end_time=$(date +%s.%N) +duration=$(echo "$end_time - $start_time" | bc) +duration_formatted=$(printf "%.2f" $duration) + +# Collect results from temp files +for i in $(seq 1 $NUM_NOTIFY_REQUESTS); do + if [ -f "/tmp/notify_result_$i.txt" ]; then + result=$(cat /tmp/notify_result_$i.txt) + case $result in + success) ((success_count["notify"]++)) ;; + fallback) ((fallback_count["notify"]++)) ;; + failed) ((failure_count["notify"]++)) ;; + esac + rm /tmp/notify_result_$i.txt + fi +done + +echo "" +echo -e "${PURPLE}Notification Phase Complete: ${duration_formatted}s${NC}" +echo -e "${GREEN}✓ Queued: ${success_count["notify"]}${NC}" +echo -e "${YELLOW}⚡ Fallback: ${fallback_count["notify"]}${NC}" +echo -e "${RED}✗ Rejected: ${failure_count["notify"]}${NC}" +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Summary +total_requests=$((NUM_AUTHORIZE_REQUESTS + NUM_HEALTH_REQUESTS + NUM_NOTIFY_REQUESTS)) +total_success=$((success_count["authorize"] + success_count["health"] + success_count["notify"])) +total_fallback=$((fallback_count["authorize"] + fallback_count["health"] + fallback_count["notify"])) +total_failure=$((failure_count["authorize"] + failure_count["health"] + failure_count["notify"])) + +echo -e "${BLUE}╔════════════════════════════════════════════════╗${NC}" +echo -e "${BLUE}║ Load Test Summary ║${NC}" +echo -e "${BLUE}╚════════════════════════════════════════════════╝${NC}" +echo "" +echo -e "${CYAN}Total Requests: ${total_requests}${NC}" +echo -e "${GREEN}✓ Successful: ${total_success}${NC}" +echo -e "${YELLOW}⚡ Fallback: ${total_fallback}${NC}" +echo -e "${RED}✗ Failed: ${total_failure}${NC}" +echo "" + +success_rate=$(echo "scale=2; ($total_success * 100) / $total_requests" | bc) +echo -e "${CYAN}Success Rate: ${success_rate}%${NC}" +echo "" + +# Fetch and display metrics +echo -e "${BLUE}=== Fault Tolerance Metrics Summary ===${NC}" +echo "" + +echo -e "${YELLOW}Retry Metrics (authorizePayment):${NC}" +retry_metrics=$(curl -s "$METRICS_URL" 2>/dev/null | grep -i "retry.*authorizePayment\|authorizePayment.*retry") +if [ -n "$retry_metrics" ]; then + echo "$retry_metrics" | head -10 +else + echo -e "${CYAN}No retry metrics found. Checking alternative patterns...${NC}" + curl -s "$METRICS_URL" 2>/dev/null | grep -iE "ft.*retry|retry.*total|retry.*calls" | head -5 fi echo "" -echo -e "${BLUE}=== Test Complete ===${NC}" -echo -e "${CYAN}Analyze the responses to verify:${NC}" -echo -e "${CYAN}1. Asynchronous processing (@Asynchronous)${NC}" -echo -e "${CYAN}2. Resource isolation (@Bulkhead)${NC}" -echo -e "${CYAN}3. Retry behavior on failures (@Retry)${NC}" +echo -e "${YELLOW}Circuit Breaker Metrics (checkGatewayHealth):${NC}" +cb_metrics=$(curl -s "$METRICS_URL" 2>/dev/null | grep -i "circuitbreaker.*checkGatewayHealth\|checkGatewayHealth.*circuit") +if [ -n "$cb_metrics" ]; then + echo "$cb_metrics" | head -10 +else + echo -e "${CYAN}No circuit breaker metrics found. Checking alternative patterns...${NC}" + curl -s "$METRICS_URL" 2>/dev/null | grep -iE "ft.*circuit|circuit.*state|circuit.*calls" | head -5 +fi + +echo "" +echo -e "${YELLOW}Bulkhead Metrics (sendPaymentNotification):${NC}" +bulkhead_metrics=$(curl -s "$METRICS_URL" 2>/dev/null | grep -i "bulkhead.*sendPaymentNotification\|sendPaymentNotification.*bulkhead") +if [ -n "$bulkhead_metrics" ]; then + echo "$bulkhead_metrics" | head -10 +else + echo -e "${CYAN}No bulkhead metrics found. Checking alternative patterns...${NC}" + curl -s "$METRICS_URL" 2>/dev/null | grep -iE "ft.*bulkhead|bulkhead.*accept|bulkhead.*reject" | head -5 +fi + +echo "" +echo -e "${GREEN}=== Load Test Complete ===${NC}" +echo "" +echo -e "${CYAN}To view all fault tolerance metrics:${NC}" +echo -e "${BLUE}curl $METRICS_URL | grep -i ft${NC}" +echo "" From 4dd0f4c5353230088f8e455b654b18e51487b673 Mon Sep 17 00:00:00 2001 From: Tarun Telang Date: Wed, 8 Apr 2026 00:38:07 +0530 Subject: [PATCH 09/12] Change test from processPayment to authorizePayment Updated test script to reflect changes in method name and improved output formatting. --- code/chapter08/payment/test-payment-retry.sh | 196 +++++++------------ 1 file changed, 71 insertions(+), 125 deletions(-) diff --git a/code/chapter08/payment/test-payment-retry.sh b/code/chapter08/payment/test-payment-retry.sh index dcbb62c3..85707bb0 100755 --- a/code/chapter08/payment/test-payment-retry.sh +++ b/code/chapter08/payment/test-payment-retry.sh @@ -1,180 +1,126 @@ #!/bin/bash # Test script for Payment Service Retry functionality -# Tests the @Retry annotation on the processPayment method +# Tests the @Retry annotation on the authorizePayment method -set -e - -echo "=== Payment Service Retry Test ===" -echo "" +# Color definitions +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color # Dynamically determine the base URL if [ -n "$CODESPACE_NAME" ] && [ -n "$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN" ]; then - # GitHub Codespaces environment - use localhost for internal testing BASE_URL="http://localhost:9080/payment/api" - echo "Detected GitHub Codespaces environment (using localhost)" - echo "Note: External access would be https://$CODESPACE_NAME-9080.$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN/payment/api" + echo -e "${CYAN}Detected GitHub Codespaces environment${NC}" elif [ -n "$GITPOD_WORKSPACE_URL" ]; then - # Gitpod environment GITPOD_HOST=$(echo $GITPOD_WORKSPACE_URL | sed 's|https://||' | sed 's|/||') BASE_URL="https://9080-$GITPOD_HOST/payment/api" - echo "Detected Gitpod environment" + echo -e "${CYAN}Detected Gitpod environment${NC}" else - # Local or other environment BASE_URL="http://localhost:9080/payment/api" - echo "Using local environment" + echo -e "${CYAN}Using local environment${NC}" fi -echo "Base URL: $BASE_URL" +echo "" +echo -e "${BLUE}=== Payment Service Retry Test ===${NC}" +echo -e "${CYAN}Base URL: $BASE_URL${NC}" echo "" -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color +echo -e "${YELLOW}Retry Configuration:${NC}" +echo " • Max Retries: 3" +echo " • Delay: 2000ms" +echo " • Jitter: 500ms" +echo " • Retry on: PaymentProcessingException" +echo " • Abort on: CriticalPaymentException" +echo " • Simulated failure rate: 60%" +echo " • Processing delay: 1500ms per attempt" +echo "" + +echo -e "${CYAN}🔍 How to identify retry behavior:${NC}" +echo " • ⚡ Fast (~1.5s) = Success on 1st attempt" +echo " • 🔄 Medium (~4s) = Succeeded after 1 retry" +echo " • 🔄🔄 Slow (~6.5s) = Succeeded after 2 retries" +echo " • 🔄🔄🔄 Very slow (~9-12s) = Needed all 3 retries or fallback" +echo "" # Function to make HTTP requests and display results make_request() { - local method=$1 - local url=$2 - local description=$3 + local amount=$1 + local description=$2 echo -e "${BLUE}Testing: $description${NC}" - echo "Request: $method $url" - echo "" + echo -e "${CYAN}Amount: \$$amount${NC}" - # Capture start time - start_time=$(date +%s%3N) + start_time=$(date +%s.%N) + response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" -X POST "${BASE_URL}/authorize?amount=${amount}" 2>/dev/null || echo "HTTP_STATUS:000") + end_time=$(date +%s.%N) - response=$(curl -s -w "\nHTTP_STATUS:%{http_code}\nTIME_TOTAL:%{time_total}" -X $method "$url" 2>/dev/null || echo "HTTP_STATUS:000") - - # Capture end time - end_time=$(date +%s%3N) - total_time=$((end_time - start_time)) + duration=$(echo "$end_time - $start_time" | bc) + duration_formatted=$(printf "%.2f" $duration) http_code=$(echo "$response" | grep "HTTP_STATUS:" | cut -d: -f2) - curl_time=$(echo "$response" | grep "TIME_TOTAL:" | cut -d: -f2) - body=$(echo "$response" | sed '/HTTP_STATUS:/d' | sed '/TIME_TOTAL:/d') + body=$(echo "$response" | sed '/HTTP_STATUS:/d') + + # Determine retry behavior based on duration + duration_int=$(echo "$duration * 10" | bc | cut -d. -f1) if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then - # Analyze timing to determine retry behavior - # Convert curl_time to integer for comparison (multiply by 10 to handle decimals) - curl_time_int=$(echo "$curl_time" | awk '{printf "%.0f", $1 * 10}') - - if [ "$curl_time_int" -lt 20 ]; then # < 2.0 seconds + if [ "$duration_int" -lt 20 ]; then echo -e "${GREEN}✓ Success (HTTP $http_code) - First attempt! ⚡${NC}" - elif [ "$curl_time_int" -lt 55 ]; then # < 5.5 seconds + elif [ "$duration_int" -lt 55 ]; then echo -e "${GREEN}✓ Success (HTTP $http_code) - After 1 retry 🔄${NC}" - elif [ "$curl_time_int" -lt 80 ]; then # < 8.0 seconds + elif [ "$duration_int" -lt 80 ]; then echo -e "${GREEN}✓ Success (HTTP $http_code) - After 2 retries 🔄🔄${NC}" else - echo -e "${GREEN}✓ Success (HTTP $http_code) - After 3 retries 🔄🔄🔄${NC}" + echo -e "${GREEN}✓ Success (HTTP $http_code) - After 3 retries or fallback 🔄🔄🔄${NC}" fi - elif [ "$http_code" -ge 400 ] && [ "$http_code" -lt 500 ]; then - echo -e "${YELLOW}⚠ Client Error (HTTP $http_code)${NC}" + echo -e "${GREEN}Response: $body${NC}" + elif [ "$http_code" -eq 400 ]; then + echo -e "${YELLOW}⚠ Client Error (HTTP $http_code) - No retries (abort condition)${NC}" + echo -e "${YELLOW}Response: $body${NC}" else echo -e "${RED}✗ Server Error (HTTP $http_code)${NC}" + echo -e "${RED}Response: $body${NC}" fi - echo "Response: $body" - echo "Total time: ${total_time}ms (curl: ${curl_time}s)" + echo -e "${CYAN}Duration: ${duration_formatted}s${NC}" echo "" echo "----------------------------------------" echo "" } -echo "Starting Payment Service Retry Tests..." -echo "" -echo "Your PaymentService has these retry settings:" -echo "• Max Retries: 3" -echo "• Delay: 2000ms" -echo "• Jitter: 500ms" -echo "• Retry on: PaymentProcessingException" -echo "• Abort on: CriticalPaymentException" -echo "• Simulated failure rate: 30% (Math.random() > 0.7)" -echo "• Processing delay: 1500ms per attempt" -echo "" -echo "🔍 HOW TO IDENTIFY RETRY BEHAVIOR:" -echo "• ⚡ Fast response (~1.5s) = Succeeded on 1st attempt" -echo "• 🔄 Medium response (~4s) = Needed 1 retry" -echo "• 🔄🔄 Slow response (~6.5s) = Needed 2 retries" -echo "• 🔄🔄🔄 Very slow response (~9-12s) = Needed 3 retries" -echo "" - -echo "Make sure the Payment Service is running on port 9080" -echo "You can start it with: cd payment && mvn liberty:run" -echo "" -read -p "Press Enter to continue..." -echo "" - -# Test 1: Valid payment (should succeed, may need retries due to random failures) -echo -e "${BLUE}=== Test 1: Valid Payment Authorization ===${NC}" -echo "This test uses a valid amount and may succeed immediately or after retries" -echo "Expected: Success after 1-4 attempts (due to 30% failure simulation)" -echo "" -make_request "POST" "$BASE_URL/authorize?amount=100.50" \ - "Valid payment amount (100.50) - may trigger retries due to random failures" +# Run multiple tests +echo -e "${BLUE}=== Test 1: Valid Payment (Will Likely Trigger Retries) ===${NC}" +make_request "100.50" "Valid payment - 60% failure rate per attempt" -# Test 2: Another valid payment to see retry behavior echo -e "${BLUE}=== Test 2: Another Valid Payment ===${NC}" -echo "Running another test to demonstrate retry variability" -echo "" -make_request "POST" "$BASE_URL/authorize?amount=250.00" \ - "Valid payment amount (250.00) - testing retry behavior" +make_request "250.00" "Valid payment - observe retry behavior" -# Test 3: Invalid payment amount (should abort immediately) -echo -e "${BLUE}=== Test 3: Invalid Payment Amount (Abort Condition) ===${NC}" -echo "This test uses an invalid amount which should trigger CriticalPaymentException" -echo "Expected: Immediate failure with no retries" -echo "" -make_request "POST" "$BASE_URL/authorize?amount=0" \ - "Invalid payment amount (0) - should abort immediately with CriticalPaymentException" +echo -e "${BLUE}=== Test 3: Large Payment ===${NC}" +make_request "999.99" "Large payment - testing retry behavior" -# Test 4: Negative amount (should abort immediately) -echo -e "${BLUE}=== Test 4: Negative Payment Amount ===${NC}" -echo "Expected: Immediate failure with no retries" -echo "" -make_request "POST" "$BASE_URL/authorize?amount=-50" \ - "Negative payment amount (-50) - should abort immediately" +echo -e "${BLUE}=== Test 4: Invalid Amount (Abort Condition) ===${NC}" +make_request "0" "Invalid amount - should abort immediately (no retries)" -# Test 5: No amount parameter (should abort immediately) -echo -e "${BLUE}=== Test 5: Missing Payment Amount ===${NC}" -echo "Expected: Immediate failure with no retries" -echo "" -make_request "POST" "$BASE_URL/authorize" \ - "Missing payment amount - should abort immediately" +echo -e "${BLUE}=== Test 5: Negative Amount (Abort Condition) ===${NC}" +make_request "-50" "Negative amount - should abort immediately (no retries)" -# Test 6: Multiple requests to observe retry patterns -echo -e "${BLUE}=== Test 6: Multiple Requests to Observe Retry Patterns ===${NC}" -echo "Running 5 valid payment requests to observe retry behavior patterns" +echo -e "${BLUE}=== Test 6: Multiple Requests to Observe Patterns ===${NC}" +echo -e "${CYAN}Sending 5 requests to observe retry patterns...${NC}" echo "" for i in {1..5}; do - echo "Request $i/5:" - amount=$((100 + i * 25)) - make_request "POST" "$BASE_URL/authorize?amount=$amount" \ - "Payment request $i with amount $amount" - - # Small delay between requests - sleep 2 + amount=$((50 + RANDOM % 150)) + echo -e "${YELLOW}Request $i:${NC}" + make_request "$amount" "Random amount test #$i" + sleep 1 done -echo -e "${GREEN}=== Retry Testing Complete ===${NC}" -echo "" -echo "Key observations to look for:" -echo -e "• ${GREEN}Successful requests:${NC} Should complete in ~1.5-12 seconds depending on retries" -echo -e "• ${YELLOW}Retry behavior:${NC} Failed attempts will retry up to 3 times with 2-2.5 second delays" -echo -e "• ${RED}Abort conditions:${NC} Invalid amounts should fail immediately (~1.5 seconds)" -echo -e "• ${BLUE}Random failures:${NC} ~30% of valid requests may need retries" +echo -e "${GREEN}=== Retry Test Complete ===${NC}" echo "" -echo "To see detailed retry logs, monitor the server logs:" -echo "tail -f target/liberty/wlp/usr/servers/mpServer/logs/messages.log" +echo -e "${CYAN}Check fault tolerance metrics:${NC}" +echo -e "${BLUE}curl http://localhost:9080/metrics?scope=base | grep retry${NC}" echo "" -echo "Expected timing patterns:" -echo "• Success on 1st attempt: ~1.5 seconds" -echo "• Success on 2nd attempt: ~4-4.5 seconds" -echo "• Success on 3rd attempt: ~6.5-7.5 seconds" -echo "• Success on 4th attempt: ~9-10.5 seconds" -echo "• Abort conditions: ~1.5 seconds (no retries)" From 6c1fc99860d574151c48ef1c987e6bc33d5f4e97 Mon Sep 17 00:00:00 2001 From: Tarun Telang Date: Wed, 8 Apr 2026 00:45:58 +0530 Subject: [PATCH 10/12] Update server.xml for MicroProfile and logging changes --- .../chapter08/payment/src/main/liberty/config/server.xml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/code/chapter08/payment/src/main/liberty/config/server.xml b/code/chapter08/payment/src/main/liberty/config/server.xml index 9d5f15b8..19577622 100644 --- a/code/chapter08/payment/src/main/liberty/config/server.xml +++ b/code/chapter08/payment/src/main/liberty/config/server.xml @@ -1,13 +1,12 @@ + microProfile-7.1 jakartaEE-10.0 - microProfile-6.1 restfulWS jsonp - jsonb cdi mpConfig - mpOpenAPI + mpOpenAPI-4.1 mpHealth mpMetrics mpFaultTolerance @@ -17,6 +16,6 @@ id="defaultHttpEndpoint" host="*" /> - + - \ No newline at end of file + From 8c5888abfe6563cc7cf4b8e413c6f06cbf4c0592 Mon Sep 17 00:00:00 2001 From: Tarun Telang Date: Wed, 8 Apr 2026 00:51:57 +0530 Subject: [PATCH 11/12] Update payment endpoint URL to use placeholders --- .../microprofile/tutorial/store/payment/service/payment.http | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/chapter08/payment/src/main/java/io/microprofile/tutorial/store/payment/service/payment.http b/code/chapter08/payment/src/main/java/io/microprofile/tutorial/store/payment/service/payment.http index 98ae2e52..2977a452 100644 --- a/code/chapter08/payment/src/main/java/io/microprofile/tutorial/store/payment/service/payment.http +++ b/code/chapter08/payment/src/main/java/io/microprofile/tutorial/store/payment/service/payment.http @@ -1,4 +1,4 @@ -POST https://orange-zebra-r745vp6rjcxp67-9080.app.github.dev/payment/authorize +POST https://:/payment/authorize { "cardNumber": "4111111111111111", @@ -6,4 +6,4 @@ POST https://orange-zebra-r745vp6rjcxp67-9080.app.github.dev/payment/authorize "expiryDate": "12/25", "securityCode": "123", "amount": 100.00 -} \ No newline at end of file +} From 22db6b9042e5e216fbe71a693220f037a914f89b Mon Sep 17 00:00:00 2001 From: Tarun Telang Date: Wed, 8 Apr 2026 00:53:24 +0530 Subject: [PATCH 12/12] Refactor PaymentService for fault tolerance and logging Enhanced PaymentService with detailed fault tolerance strategies and logging. --- .../store/payment/service/PaymentService.java | 209 ++++++++++++++---- 1 file changed, 170 insertions(+), 39 deletions(-) diff --git a/code/chapter08/payment/src/main/java/io/microprofile/tutorial/store/payment/service/PaymentService.java b/code/chapter08/payment/src/main/java/io/microprofile/tutorial/store/payment/service/PaymentService.java index db3ba2a0..7dd1667e 100644 --- a/code/chapter08/payment/src/main/java/io/microprofile/tutorial/store/payment/service/PaymentService.java +++ b/code/chapter08/payment/src/main/java/io/microprofile/tutorial/store/payment/service/PaymentService.java @@ -16,72 +16,203 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; +import java.util.logging.Logger; +/** + * Payment service demonstrating MicroProfile Fault Tolerance patterns. + * + * Each method demonstrates specific fault tolerance strategies: + * - authorizePayment: Retry + Timeout + Fallback for transient failures + * - checkGatewayHealth: Circuit Breaker + Timeout to prevent hammering failed services + * - sendPaymentNotification: Asynchronous + Bulkhead + Fallback for resource isolation + * + * These patterns work together to prevent cascading failures and ensure resilient payment processing. + */ @ApplicationScoped public class PaymentService { - @ConfigProperty(name = "payment.gateway.endpoint", defaultValue = "https://defaultapi.paymentgateway.com") + private static final Logger logger = Logger.getLogger(PaymentService.class.getName()); + + @ConfigProperty(name = "payment.gateway.endpoint", defaultValue = "https://api.paymentgateway.com") private String endpoint; /** - * Process the payment request. - * - * @param paymentDetails details of the payment - * @return response message indicating success or failure - * @throws PaymentProcessingException if a transient issue occurs + * Authorize a payment transaction with fault tolerance. + * + * Fault Tolerance Strategy: + * - @Retry: Handles transient network failures (up to 3 retries with jitter) + * - @Timeout: Prevents indefinite waits (3 second limit per attempt) + * - @Fallback: Provides degraded service when gateway unavailable + * + * @param paymentDetails Payment details to authorize + * @return Authorization result JSON + * @throws CriticalPaymentException For non-retryable failures (invalid card, insufficient funds) */ - @Asynchronous + @Retry( + maxRetries = 3, + delay = 2000, + jitter = 500, + retryOn = PaymentProcessingException.class, + abortOn = CriticalPaymentException.class + ) @Timeout(3000) - @Retry(maxRetries = 3, - delay = 2000, - jitter = 500, - retryOn = PaymentProcessingException.class, - abortOn = CriticalPaymentException.class) - @Fallback(fallbackMethod = "fallbackProcessPayment") - @Bulkhead(value=5) + @Fallback(fallbackMethod = "fallbackAuthorizePayment") + public String authorizePayment(PaymentDetails paymentDetails) + throws PaymentProcessingException { + + logger.info("Calling payment gateway at: " + endpoint); + logger.info("Processing payment for amount: " + paymentDetails.getAmount()); + + // Simulate network latency + simulateDelay(1500); + + // Simulate transient failures (60% failure rate to demonstrate retries) + if (Math.random() > 0.4) { + throw new PaymentProcessingException( + "Temporary payment gateway failure - will retry" + ); + } + + return String.format( + "{\"status\":\"success\",\"message\":\"Payment authorized\",\"amount\":%s}", + paymentDetails.getAmount() + ); + } + + /** + * Fallback method for payment authorization. + * Returns a pending status and queues payment for offline processing. + * + * @param paymentDetails Payment details + * @return Fallback response indicating payment is queued + */ + public String fallbackAuthorizePayment(PaymentDetails paymentDetails) { + logger.warning("Payment gateway unavailable - using fallback for amount: " + + paymentDetails.getAmount()); + + // In production: queue for offline processing, use backup gateway, etc. + return String.format( + "{\"status\":\"pending\",\"message\":\"Payment queued for processing\",\"amount\":%s}", + paymentDetails.getAmount() + ); + } + + /** + * Check payment gateway health with circuit breaker protection. + * + * Fault Tolerance Strategy: + * - @CircuitBreaker: Prevents hammering failed gateway (opens at 50% failure rate) + * - @Timeout: Quick health check (2 second limit) + * + * Circuit breaker will OPEN after 2 failures out of 4 requests (50% failure ratio), + * wait 5 seconds in OPEN state, then transition to HALF_OPEN to test recovery. + * Two consecutive successes in HALF_OPEN state will CLOSE the circuit. + * + * @return true if gateway is healthy, false otherwise + */ @CircuitBreaker( failureRatio = 0.5, requestVolumeThreshold = 4, - delay = 3000 + delay = 5000, + successThreshold = 2 ) - public CompletionStage processPayment(PaymentDetails paymentDetails) throws PaymentProcessingException { + @Timeout(2000) + public boolean checkGatewayHealth() { + logger.info("Checking payment gateway health"); - // Example logic to call the payment gateway API - System.out.println("Calling payment gateway API at: " + endpoint); - - simulateDelay(); - - System.out.println("Processing payment for amount: " + paymentDetails.getAmount()); - - // Simulating a transient failure - if (Math.random() > 0.7) { - throw new PaymentProcessingException("Temporary payment processing failure"); + // Simulate health check call + simulateDelay(500); + + // Simulate intermittent gateway failures (60% success rate) + if (Math.random() > 0.6) { + throw new RuntimeException("Gateway health check failed"); } + + return true; + } - // Simulating successful processing - return CompletableFuture.completedFuture("{\"status\":\"success\", \"message\":\"Payment processed successfully.\"}"); + /** + * Send payment notification asynchronously with resource isolation. + * + * Fault Tolerance Strategy: + * - @Asynchronous: Non-blocking execution in separate thread + * - @Bulkhead: Limits concurrent notifications (max 10 concurrent, queue up to 20) + * - @Timeout: Prevents stuck notification threads (5 second limit) + * - @Fallback: Logs failed notifications for later retry + * + * Uses CompletionStage (not Future) to ensure fault tolerance annotations + * react properly to asynchronous failures. With CompletionStage, exceptions + * in exceptionally-completed stages trigger @Retry, @CircuitBreaker, and @Fallback. + * + * @param paymentId Payment identifier + * @param recipient Notification recipient email/phone + * @return CompletionStage that completes when notification is sent + */ + @Asynchronous + @Bulkhead(value = 10, waitingTaskQueue = 20) + @Timeout(5000) + @Fallback(fallbackMethod = "fallbackSendNotification") + public CompletionStage sendPaymentNotification( + String paymentId, + String recipient + ) { + logger.info("Notification queued for payment: " + paymentId + " to " + recipient); + + // Schedule actual notification work in background (fire-and-forget) + // This allows the HTTP response to return immediately + CompletableFuture.runAsync(() -> { + try { + // Simulate notification sending delay (e.g., calling external SMS/email service) + simulateDelay(2000); + + // Simulate notification failures (80% success rate) + if (Math.random() > 0.8) { + logger.warning("Notification service unavailable for payment: " + paymentId); + throw new RuntimeException("Notification service unavailable"); + } + + logger.info("Notification sent successfully for payment: " + paymentId); + } catch (Exception e) { + logger.severe("Failed to send notification for payment: " + paymentId + " - " + e.getMessage()); + } + }); + + // Return immediately - client gets instant response + // @Asynchronous annotation ensures this executes on a background thread, + // but the CompletionStage itself completes immediately + return CompletableFuture.completedFuture("Notification queued for processing"); } /** - * Fallback method when payment processing fails. - * - * @param paymentDetails details of the payment - * @return response message for fallback + * Fallback for notification - logs failure for later retry. + * + * @param paymentId Payment identifier + * @param recipient Notification recipient + * @return CompletionStage indicating notification was queued */ - public CompletionStage fallbackProcessPayment(PaymentDetails paymentDetails) { - System.out.println("Fallback invoked for payment of amount: " + paymentDetails.getAmount()); - return CompletableFuture.completedFuture("{\"status\":\"failed\", \"message\":\"Payment service is currently unavailable.\"}"); + public CompletionStage fallbackSendNotification( + String paymentId, + String recipient + ) { + logger.warning("Failed to send notification for payment: " + paymentId); + + // In production: queue notification for retry, use backup channel (SMS if email failed), etc. + return CompletableFuture.completedFuture( + "Notification queued for retry" + ); } /** - * Simulate a delay in processing to demonstrate timeout. + * Simulate network/processing delay. + * + * @param milliseconds Delay duration in milliseconds */ - private void simulateDelay() { + private void simulateDelay(long milliseconds) { try { - Thread.sleep(1500); // Simulated long-running task + Thread.sleep(milliseconds); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new RuntimeException("Processing interrupted"); + throw new RuntimeException("Processing interrupted", e); } } }