diff --git a/code/chapter08/payment/Dockerfile b/code/chapter08/payment/Dockerfile index 77e6dde6..dbabd4f9 100644 --- a/code/chapter08/payment/Dockerfile +++ b/code/chapter08/payment/Dockerfile @@ -1,4 +1,4 @@ -FROM icr.io/appcafe/open-liberty:full-java17-openj9-ubi +FROM icr.io/appcafe/open-liberty:full-java21-openj9-ubi # Copy configuration files COPY --chown=1001:0 src/main/liberty/config/ /config/ diff --git a/code/chapter08/payment/README.adoc b/code/chapter08/payment/README.adoc index eb53e9dc..423bc079 100644 --- a/code/chapter08/payment/README.adoc +++ b/code/chapter08/payment/README.adoc @@ -1,132 +1,610 @@ -= Payment Service += Payment Service - MicroProfile Fault Tolerance :toc: macro -:toclevels: 3 +:toclevels: 1 :icons: font :source-highlighter: highlight.js :experimental: toc::[] -This microservice is part of the Jakarta EE 10 and MicroProfile 6.1-based e-commerce application. It handles payment processing and transaction management. +This microservice demonstrates MicroProfile Fault Tolerance patterns for building resilient payment processing services. It is part of the Jakarta EE and MicroProfile based e-commerce tutorial. == Features -* Payment transaction processing -* Dynamic configuration management via MicroProfile Config -* RESTful API endpoints with JSON support -* Custom ConfigSource implementation -* OpenAPI documentation -* **MicroProfile Fault Tolerance with Retry Policies** -* **Circuit Breaker protection for external services** -* **Fallback mechanisms for service resilience** -* **Bulkhead pattern for concurrency control** -* **Timeout protection for long-running operations** +* **Payment Authorization** with Retry, Timeout, and Fallback +* **Gateway Health Checks** with Circuit Breaker protection +* **Async Notifications** with Bulkhead resource isolation == MicroProfile Fault Tolerance Implementation -The Payment Service implements comprehensive fault tolerance patterns using MicroProfile Fault Tolerance annotations: +The Payment Service demonstrates three distinct fault tolerance patterns, each applied to different service methods: -=== Retry Policies +=== 1. Payment Authorization (Retry + Timeout + Fallback) -The service implements different retry strategies based on operation criticality: +**Method**: `PaymentService.authorizePayment()` -==== Payment Authorization Retry (@Retry) -* **Max Retries**: 3 attempts -* **Delay**: 1000ms with 500ms jitter -* **Max Duration**: 10 seconds -* **Retry On**: RuntimeException, WebApplicationException -* **Use Case**: Standard payment authorization with exponential backoff +**Fault Tolerance Strategy:** +* **@Retry**: Handles transient network failures (up to 3 retries with jitter) +* **@Timeout**: Prevents indefinite waits (3 second limit per attempt) +* **@Fallback**: Provides degraded service when gateway unavailable + +**Configuration:** [source,java] ---- @Retry( maxRetries = 3, delay = 2000, - maxDuration = 10000 jitter = 500, - retryOn = {RuntimeException.class, WebApplicationException.class} + retryOn = PaymentProcessingException.class, + abortOn = CriticalPaymentException.class ) +@Timeout(3000) +@Fallback(fallbackMethod = "fallbackAuthorizePayment") +public String authorizePayment(PaymentDetails paymentDetails) ---- -=== Circuit Breaker Protection +**Behavior:** + +* Retries transient failures (network timeouts, temporary gateway issues) +* Does NOT retry critical errors (invalid card, insufficient funds) +* Falls back to offline processing queue if all retries fail +* Each retry attempt has its own 3-second timeout + +=== 2. Gateway Health Check (Circuit Breaker + Timeout) + +**Method**: `PaymentService.checkGatewayHealth()` -Payment capture operations use circuit breaker pattern: +**Fault Tolerance Strategy:** +* **@CircuitBreaker**: Prevents hammering failed gateway (opens at 50% failure rate) +* **@Timeout**: Quick health check (2 second limit) + +**Configuration:** [source,java] ---- @CircuitBreaker( failureRatio = 0.5, requestVolumeThreshold = 4, - delay = 5000 + delay = 5000, + successThreshold = 2 ) +@Timeout(2000) +public boolean checkGatewayHealth() ---- -* **Failure Ratio**: 50% failure rate triggers circuit opening -* **Request Volume**: Minimum 4 requests for evaluation -* **Recovery Delay**: 5 seconds before attempting recovery +**Circuit Breaker States:** + +* **CLOSED**: Normal operation, all health checks pass through +* **OPEN**: Circuit breaker activated after 50% failure rate (2 of 4 requests fail) +* **HALF_OPEN**: After 5-second delay, allows 2 test requests +* **CLOSED**: Returns to normal after 2 consecutive successful tests + +**Behavior:** -=== Timeout Protection +* Prevents repeated health checks to failed gateway +* Automatically recovers when gateway comes back online +* Reduces load on failing systems -Operations with potential long delays are protected with timeouts: +=== 3. Payment Notification (Asynchronous + Bulkhead + Fallback) +**Method**: `PaymentService.sendPaymentNotification()` + +**Fault Tolerance Strategy:** + +* **@Asynchronous**: Non-blocking execution in separate thread +* **@Bulkhead**: Limits concurrent notifications (max 10, queue 20) +* **@Timeout**: Prevents stuck notification threads (5 second limit) +* **@Fallback**: Logs failed notifications for retry + +**Configuration:** [source,java] ---- -@Timeout(value = 3000) +@Asynchronous +@Bulkhead(value = 10, waitingTaskQueue = 20) +@Timeout(5000) +@Fallback(fallbackMethod = "fallbackSendNotification") +public CompletionStage sendPaymentNotification(String paymentId, String recipient) ---- -=== Bulkhead Pattern +**Behavior:** -The bulkhead pattern limits concurrent requests to prevent system overload: +* Returns immediately (non-blocking) via CompletionStage +* Up to 10 concurrent notifications processing +* Additional 20 notifications queued +* Excess notifications rejected to prevent overload +* Failed notifications logged for later retry -[source,java] +== REST Endpoints + +=== POST /payment/api/authorize + +Authorize a payment with full fault tolerance protection. + +**Request:** +[source,bash] ---- -@Bulkhead(value = 5) +curl -X POST "http://localhost:9080/payment/api/authorize?amount=100.50" ---- -* **Concurrent Requests**: Limited to 5 concurrent requests -* **Excess Requests**: Rejected immediately instead of queuing -* **Use Case**: Protect service from traffic spikes and cascading failures +**Success Response (200 OK):** +[source,json] +---- +{ + "status": "success", + "message": "Payment authorized", + "amount": 100.50 +} +---- -=== Fallback Mechanisms +**Fallback Response (200 OK - degraded service):** +[source,json] +---- +{ + "status": "pending", + "message": "Payment queued for processing", + "amount": 100.50 +} +---- -All critical operations have fallback methods that provide graceful degradation: +**Error Response (400 Bad Request):** +[source,json] +---- +{ + "error": "Invalid payment amount: -10" +} +---- -* **Payment Authorization Fallback**: Returns service unavailable with retry instructions +=== GET /payment/api/health/gateway -== Endpoints +Check payment gateway health with circuit breaker protection. -=== GET /payment/api/payment-config -* Returns all current payment configuration values -* Example: `GET http://localhost:9080/payment/api/payment-config` -* Response: `{"gateway.endpoint":"https://api.paymentgateway.com"}` +**Request:** +[source,bash] +---- +curl http://localhost:9080/payment/api/health/gateway +---- -=== POST /payment/api/payment-config -* Updates a payment configuration value -* Example: `POST http://localhost:9080/payment/api/payment-config` -* Request body: `{"key": "payment.gateway.endpoint", "value": "https://new-api.paymentgateway.com"}` -* Response: `{"key":"payment.gateway.endpoint","value":"https://new-api.paymentgateway.com","message":"Configuration updated successfully"}` +**Healthy Response (200 OK):** +[source,json] +---- +{ + "status": "healthy", + "message": "Payment gateway is operational" +} +---- -=== POST /payment/api/authorize -* Processes a payment authorization with retry policy -* **Retry Configuration**: 3 attempts, 1s delay, 500ms jitter -* **Fallback**: Service unavailable response -* Example: `POST http://localhost:9080/payment/api/authorize` -* Request body: `{"cardNumber":"4111111111111111", "cardHolderName":"Test User", "expiryDate":"12/25", "securityCode":"123", "amount":100.00}` -* Response: `{"status":"success", "message":"Payment authorized successfully", "transactionId":"TXN1234567890", "amount":100.00}` -* Fallback Response: `{"status":"failed", "message":"Payment gateway unavailable. Please try again later.", "fallback":true}` - -=== POST /payment/api/payment-config/process-example -* Example endpoint demonstrating payment processing with configuration -* Example: `POST http://localhost:9080/payment/api/payment-config/process-example` -* Request body: `{"cardNumber":"4111111111111111", "cardHolderName":"Test User", "expiryDate":"12/25", "securityCode":"123", "amount":100.00}` -* Response: `{"amount":100.00,"message":"Payment processed successfully","status":"success","configUsed":{"gatewayEndpoint":"https://new-api.paymentgateway.com"}}` +**Circuit Open Response (503 Service Unavailable):** +[source,json] +---- +{ + "status": "circuit_open", + "message": "Circuit breaker is open - gateway appears to be down" +} +---- + +=== POST /payment/api/notify/{paymentId} + +Send asynchronous payment notification with bulkhead protection. + +**Request:** +[source,bash] +---- +curl -X POST "http://localhost:9080/payment/api/notify/PAY-12345?recipient=customer@example.com" +---- + +**Success Response (200 OK):** +[source,json] +---- +{ + "status": "success", + "message": "Notification sent successfully" +} +---- + +**Bulkhead Rejected Response (503 Service Unavailable):** +[source,json] +---- +{ + "status": "rejected", + "message": "Too many concurrent notifications - please try again later" +} +---- + +== Testing and Verifying Fault Tolerance + +This section explains how to test and verify that each fault tolerance pattern is working correctly. + +=== Automated Test Scripts + +The payment service includes comprehensive test scripts that demonstrate all fault tolerance patterns: + +[source,bash] +---- +cd code/chapter08/payment + +# 1. Basic smoke test - verify all endpoints work +./test-payment-basic.sh + +# 2. Test retry behavior (30% simulated failure rate) +./test-payment-retry.sh + +# 3. Test circuit breaker state transitions +./test-payment-circuit-breaker.sh + +# 4. Test asynchronous execution +./test-payment-async.sh + +# 5. Test bulkhead resource isolation +./test-payment-bulkhead.sh + +# 6. Comprehensive load test (all patterns) +./test-payment-concurrent-load.sh +---- + +=== Manual Testing: Retry Pattern + +**What to test:** The authorization endpoint retries transient failures but aborts on critical errors. + +**Test valid payment (may trigger retries):** +[source,bash] +---- +curl -X POST "http://localhost:9080/payment/api/authorize?amount=100.50" +---- + +**Expected behavior:** +* 70% success rate (simulated) +* Failed requests automatically retry up to 3 times +* Each retry has 2-2.5 second delay (2000ms + 500ms jitter) +* Total duration reveals retry count: + - ~1.5s = Success on 1st attempt + - ~4s = Needed 1 retry + - ~6.5s = Needed 2 retries + - ~9-12s = All 3 retries or fallback + +**Test abort condition (no retries):** +[source,bash] +---- +curl -X POST "http://localhost:9080/payment/api/authorize?amount=0" +curl -X POST "http://localhost:9080/payment/api/authorize?amount=-50" +---- + +**Expected behavior:** +* Immediate failure (~1.5s) with HTTP 400 +* No retries attempted (abort on CriticalPaymentException) + +**Verification:** +[source,bash] +---- +# Check retry metrics +curl http://localhost:9080/metrics?scope=base | grep "ft_authorizePayment_retry" + +# Look for: +# - retry_callsSucceededNotRetried_total (first-attempt successes) +# - retry_callsSucceededRetried_total (succeeded after retry) +# - retry_retries_total (total number of retries executed) +---- + +=== Manual Testing: Circuit Breaker Pattern + +**What to test:** The health check endpoint opens the circuit after repeated failures, preventing cascade. + +**Trigger circuit breaker (50% simulated failure rate):** +[source,bash] +---- +# Send 10 health checks - circuit should open after ~4-5 requests +for i in {1..10}; do + echo "Request $i:" + curl http://localhost:9080/payment/api/health/gateway + echo "" + sleep 0.5 +done +---- + +**Expected state transitions:** +1. **CLOSED** (0): First few requests pass through normally +2. **OPEN** (1): After 2 failures out of 4 requests (50% threshold), circuit opens +3. Subsequent requests fail immediately with "Circuit breaker is OPEN" message +4. **HALF_OPEN** (2): After 5-second delay, circuit allows test requests +5. **CLOSED** (0): After 2 consecutive successes, circuit returns to normal + +**Wait for recovery:** +[source,bash] +---- +# Wait 5 seconds for circuit delay +sleep 5 + +# Send requests to test HALF_OPEN → CLOSED transition +curl http://localhost:9080/payment/api/health/gateway # Test 1 +curl http://localhost:9080/payment/api/health/gateway # Test 2 (should close circuit) +---- + +**Verification:** +[source,bash] +---- +# Check circuit breaker state +curl http://localhost:9080/metrics?scope=base | grep "ft_checkGatewayHealth_circuitbreaker" + +# Look for: +# - circuitbreaker_state_total{state="0"} = CLOSED count +# - circuitbreaker_state_total{state="1"} = OPEN count +# - circuitbreaker_state_total{state="2"} = HALF_OPEN count +# - circuitbreaker_opened_total = Number of times circuit opened +---- + +=== Manual Testing: Bulkhead Pattern + +**What to test:** The notification endpoint limits concurrent executions and rejects excess requests. + +**Send concurrent requests to test limits:** +[source,bash] +---- +# Send 15 concurrent notifications +# Expected: 3 concurrent + 2 queued = 5 accepted, 10 rejected +for i in {1..15}; do + curl -X POST "http://localhost:9080/payment/api/notify/PAY-$(printf "%05d" $i)" & +done +wait +---- + +**Expected behavior:** +* **Bulkhead capacity:** 3 concurrent + 2 waiting queue = 5 total +* Requests 1-3: Execute immediately (concurrent slots) +* Requests 4-5: Queued for execution (waiting queue) +* Requests 6-15: **Rejected** with BulkheadException (HTTP 503) + +**Verification:** +[source,bash] +---- +# Check bulkhead metrics +curl http://localhost:9080/metrics?scope=base | grep "ft_sendPaymentNotification_bulkhead" + +# Look for: +# - bulkhead_callsAccepted_total (accepted requests) +# - bulkhead_callsRejected_total (rejected requests) +# - bulkhead_executionDuration_* (execution time stats) +# - bulkhead_queuePopulation_current (current queue size) +---- + +=== Manual Testing: Asynchronous Pattern + +**What to test:** The notification endpoint returns immediately (non-blocking) despite 2-second processing time. + +**Send sequential async requests:** +[source,bash] +---- +# Time 5 sequential async requests +time for i in {1..5}; do + curl -X POST "http://localhost:9080/payment/api/notify/PAY-0000$i" +done + +# Expected: < 2.5 seconds total (if truly async) +# If blocking: ~10 seconds (5 requests × 2s each) +---- + +**Expected behavior:** +* Each request returns immediately (< 0.5s) +* Total time for 5 requests: < 2.5 seconds +* Processing continues in background +* Uses CompletionStage (not Future) for proper fault tolerance integration + +**Verification:** +* Check response time (should be < 500ms per request) +* Monitor server logs for async processing +* Verify bulkhead metrics show concurrent executions + +=== Observing Fault Tolerance in Server Logs + +Monitor the Liberty server logs to see fault tolerance in action: + +[source,bash] +---- +# Tail the server logs +tail -f target/liberty/wlp/usr/servers/defaultServer/logs/messages.log + +# Look for log messages indicating: +# - "Processing payment..." (method entry) +# - "Retry attempt X of 3" (retry behavior) +# - "Circuit breaker opened/closed" (state changes) +# - "Bulkhead rejected request" (resource limits) +# - "Fallback invoked" (degraded service) +---- + +=== Validating Configuration + +Test that fault tolerance configuration is loaded correctly: + +[source,bash] +---- +# Check all fault tolerance configuration properties +curl http://localhost:9080/payment/api/payment-config | grep -E "(retry|timeout|circuitbreaker|bulkhead)" +---- + +=== Complete Verification Checklist + +Use this checklist to verify all fault tolerance patterns: + +- [ ] **Retry**: Failed requests retry automatically (check logs and metrics) +- [ ] **Retry Abort**: Critical errors don't retry (test with amount=0) +- [ ] **Timeout**: Requests timeout after configured duration +- [ ] **Circuit Breaker**: Circuit opens after failure threshold +- [ ] **Circuit Recovery**: Circuit closes after successful probes +- [ ] **Bulkhead**: Concurrent requests limited (check rejected count) +- [ ] **Async**: Requests return immediately (< 500ms response time) +- [ ] **Fallback**: Degraded service provided on failures +- [ ] **Metrics**: All fault tolerance metrics exposed and updating + +**Success criteria:** All test scripts complete successfully, and metrics show expected fault tolerance activations. + +== Monitoring with MicroProfile Metrics + +Access fault tolerance metrics at the metrics endpoint: + +[source,bash] +---- +# View all base scope metrics (includes fault tolerance) +curl http://localhost:9080/metrics?scope=base + +# Filter for payment authorization retry metrics +curl http://localhost:9080/metrics?scope=base | grep "authorizePayment.*retry" + +# Check circuit breaker state (0=CLOSED, 1=OPEN, 2=HALF_OPEN) +curl http://localhost:9080/metrics?scope=base | grep "circuitbreaker.state" + +# View bulkhead metrics +curl http://localhost:9080/metrics?scope=base | grep "bulkhead" +---- + +**Key Metrics to Monitor:** + +[cols="2,3", options="header"] +|=== +| Metric | Description + +| `ft_authorizePayment_retry_callsSucceededNotRetried_total` +| Payments that succeeded without retry + +| `ft_authorizePayment_retry_callsSucceededRetried_total` +| Payments that succeeded after retrying + +| `ft_authorizePayment_fallback_calls_total` +| Number of times fallback was invoked + +| `ft_checkGatewayHealth_circuitbreaker_state_total` +| Circuit breaker state (0/1/2) + +| `ft_sendPaymentNotification_bulkhead_callsRejected_total` +| Notifications rejected by bulkhead +|=== + +== Building and Running + +=== Prerequisites + +* JDK 21 or higher +* Maven 3.13.x or higher + +=== Local Development + +[source,bash] +---- +# Navigate to payment service directory +cd code/chapter08/payment + +# Build the application +mvn clean package + +# Run with Liberty +mvn liberty:run + +# In another terminal, test the endpoints +curl -X POST "http://localhost:9080/payment/api/authorize?amount=100" +---- + +The server will start on: +* HTTP: `http://localhost:9080` +* HTTPS: `https://localhost:9081` + +=== Docker + +[source,bash] +---- +# Build and run with Docker +./run-docker.sh +---- + +== Project Structure + +[source] +---- +payment/ +├── src/ +│ ├── main/ +│ │ ├── java/ +│ │ │ └── io/microprofile/tutorial/store/payment/ +│ │ │ ├── PaymentRestApplication.java +│ │ │ ├── service/ +│ │ │ │ └── PaymentService.java # Fault tolerance patterns +│ │ │ ├── resource/ +│ │ │ │ ├── PaymentResource.java # REST endpoints +│ │ │ │ └── PaymentConfigResource.java +│ │ │ ├── entity/ +│ │ │ │ └── PaymentDetails.java +│ │ │ ├── exception/ +│ │ │ │ ├── PaymentProcessingException.java # Retryable errors +│ │ │ │ └── CriticalPaymentException.java # Non-retryable errors +│ │ │ └── config/ +│ │ │ ├── PaymentServiceConfigSource.java +│ │ │ └── PaymentConfig.java +│ │ ├── liberty/config/ +│ │ │ └── server.xml # mpFaultTolerance feature +│ │ └── resources/ +│ │ └── META-INF/ +│ │ └── microprofile-config.properties +│ └── test/ +│ └── java/ +├── pom.xml +└── README.adoc +---- + +== Key Concepts Demonstrated + +This chapter demonstrates: + +**1. Retry Pattern** +- Automatically retry transient failures +- Configure retry delays with jitter +- Distinguish retryable vs non-retryable exceptions + +**2. Circuit Breaker Pattern** +- Prevent cascading failures +- Automatic recovery detection +- State transitions (CLOSED → OPEN → HALF_OPEN → CLOSED) + +**3. Bulkhead Pattern ** +- Resource isolation +- Thread pool style with queuing +- Prevent resource exhaustion + +**4. Timeout Pattern** +- Prevent indefinite blocking +- Per-attempt timeout with retry +- Free resources for other operations + +**5. Fallback Pattern** +- Graceful degradation +- Alternative responses during failures +- Maintain service availability + +**6. Asynchronous Execution** +- Non-blocking operations +- CompletionStage for proper fault tolerance integration +- Improved scalability + +**7. Configuration Externalization** +- Runtime configuration via MicroProfile Config +- Environment-specific fault tolerance parameters +- No code changes required + +## Related Chapters + +- **Chapter 3**: Idempotency (safe retries) and Audit Logging +- **Chapter 5**: MicroProfile Config (externalize fault tolerance settings) +- **Chapter 7**: MicroProfile Metrics (monitor fault tolerance behavior) +- **Chapter 11**: Complete integrated e-commerce application + +== Further Reading + +- link:https://download.eclipse.org/microprofile/microprofile-fault-tolerance-4.1/microprofile-fault-tolerance-spec-4.1.html[MicroProfile Fault Tolerance 4.1 Specification] +- link:https://openliberty.io/docs/latest/fault-tolerance.html[Open Liberty Fault Tolerance Guide] +- link:https://openliberty.io/blog/2020/06/04/asynchronous-programming-microprofile-fault-tolerance.html[Asynchronous Programming with MicroProfile Fault Tolerance] == Building and Running the Service === Prerequisites -* JDK 17 or higher -* Maven 3.6.0 or higher +* JDK 21 or higher +* Maven 3.13.x or higher === Local Development diff --git a/code/chapter08/payment/pom.xml b/code/chapter08/payment/pom.xml index 12b8fada..ecc75ae2 100644 --- a/code/chapter08/payment/pom.xml +++ b/code/chapter08/payment/pom.xml @@ -12,8 +12,8 @@ UTF-8 - 17 - 17 + 21 + 21 UTF-8 UTF-8 @@ -33,7 +33,7 @@ org.projectlombok lombok - 1.18.26 + 1.18.36 provided @@ -49,7 +49,7 @@ org.eclipse.microprofile microprofile - 6.1 + 7.1 pom provided @@ -65,6 +65,16 @@ ${project.artifactId} + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 21 + + + io.openliberty.tools @@ -82,4 +92,4 @@ - \ No newline at end of file + diff --git a/code/chapter08/payment/src/main/java/io/microprofile/tutorial/store/payment/service/PaymentService.java b/code/chapter08/payment/src/main/java/io/microprofile/tutorial/store/payment/service/PaymentService.java index db3ba2a0..7dd1667e 100644 --- a/code/chapter08/payment/src/main/java/io/microprofile/tutorial/store/payment/service/PaymentService.java +++ b/code/chapter08/payment/src/main/java/io/microprofile/tutorial/store/payment/service/PaymentService.java @@ -16,72 +16,203 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; +import java.util.logging.Logger; +/** + * Payment service demonstrating MicroProfile Fault Tolerance patterns. + * + * Each method demonstrates specific fault tolerance strategies: + * - authorizePayment: Retry + Timeout + Fallback for transient failures + * - checkGatewayHealth: Circuit Breaker + Timeout to prevent hammering failed services + * - sendPaymentNotification: Asynchronous + Bulkhead + Fallback for resource isolation + * + * These patterns work together to prevent cascading failures and ensure resilient payment processing. + */ @ApplicationScoped public class PaymentService { - @ConfigProperty(name = "payment.gateway.endpoint", defaultValue = "https://defaultapi.paymentgateway.com") + private static final Logger logger = Logger.getLogger(PaymentService.class.getName()); + + @ConfigProperty(name = "payment.gateway.endpoint", defaultValue = "https://api.paymentgateway.com") private String endpoint; /** - * Process the payment request. - * - * @param paymentDetails details of the payment - * @return response message indicating success or failure - * @throws PaymentProcessingException if a transient issue occurs + * Authorize a payment transaction with fault tolerance. + * + * Fault Tolerance Strategy: + * - @Retry: Handles transient network failures (up to 3 retries with jitter) + * - @Timeout: Prevents indefinite waits (3 second limit per attempt) + * - @Fallback: Provides degraded service when gateway unavailable + * + * @param paymentDetails Payment details to authorize + * @return Authorization result JSON + * @throws CriticalPaymentException For non-retryable failures (invalid card, insufficient funds) */ - @Asynchronous + @Retry( + maxRetries = 3, + delay = 2000, + jitter = 500, + retryOn = PaymentProcessingException.class, + abortOn = CriticalPaymentException.class + ) @Timeout(3000) - @Retry(maxRetries = 3, - delay = 2000, - jitter = 500, - retryOn = PaymentProcessingException.class, - abortOn = CriticalPaymentException.class) - @Fallback(fallbackMethod = "fallbackProcessPayment") - @Bulkhead(value=5) + @Fallback(fallbackMethod = "fallbackAuthorizePayment") + public String authorizePayment(PaymentDetails paymentDetails) + throws PaymentProcessingException { + + logger.info("Calling payment gateway at: " + endpoint); + logger.info("Processing payment for amount: " + paymentDetails.getAmount()); + + // Simulate network latency + simulateDelay(1500); + + // Simulate transient failures (60% failure rate to demonstrate retries) + if (Math.random() > 0.4) { + throw new PaymentProcessingException( + "Temporary payment gateway failure - will retry" + ); + } + + return String.format( + "{\"status\":\"success\",\"message\":\"Payment authorized\",\"amount\":%s}", + paymentDetails.getAmount() + ); + } + + /** + * Fallback method for payment authorization. + * Returns a pending status and queues payment for offline processing. + * + * @param paymentDetails Payment details + * @return Fallback response indicating payment is queued + */ + public String fallbackAuthorizePayment(PaymentDetails paymentDetails) { + logger.warning("Payment gateway unavailable - using fallback for amount: " + + paymentDetails.getAmount()); + + // In production: queue for offline processing, use backup gateway, etc. + return String.format( + "{\"status\":\"pending\",\"message\":\"Payment queued for processing\",\"amount\":%s}", + paymentDetails.getAmount() + ); + } + + /** + * Check payment gateway health with circuit breaker protection. + * + * Fault Tolerance Strategy: + * - @CircuitBreaker: Prevents hammering failed gateway (opens at 50% failure rate) + * - @Timeout: Quick health check (2 second limit) + * + * Circuit breaker will OPEN after 2 failures out of 4 requests (50% failure ratio), + * wait 5 seconds in OPEN state, then transition to HALF_OPEN to test recovery. + * Two consecutive successes in HALF_OPEN state will CLOSE the circuit. + * + * @return true if gateway is healthy, false otherwise + */ @CircuitBreaker( failureRatio = 0.5, requestVolumeThreshold = 4, - delay = 3000 + delay = 5000, + successThreshold = 2 ) - public CompletionStage processPayment(PaymentDetails paymentDetails) throws PaymentProcessingException { + @Timeout(2000) + public boolean checkGatewayHealth() { + logger.info("Checking payment gateway health"); - // Example logic to call the payment gateway API - System.out.println("Calling payment gateway API at: " + endpoint); - - simulateDelay(); - - System.out.println("Processing payment for amount: " + paymentDetails.getAmount()); - - // Simulating a transient failure - if (Math.random() > 0.7) { - throw new PaymentProcessingException("Temporary payment processing failure"); + // Simulate health check call + simulateDelay(500); + + // Simulate intermittent gateway failures (60% success rate) + if (Math.random() > 0.6) { + throw new RuntimeException("Gateway health check failed"); } + + return true; + } - // Simulating successful processing - return CompletableFuture.completedFuture("{\"status\":\"success\", \"message\":\"Payment processed successfully.\"}"); + /** + * Send payment notification asynchronously with resource isolation. + * + * Fault Tolerance Strategy: + * - @Asynchronous: Non-blocking execution in separate thread + * - @Bulkhead: Limits concurrent notifications (max 10 concurrent, queue up to 20) + * - @Timeout: Prevents stuck notification threads (5 second limit) + * - @Fallback: Logs failed notifications for later retry + * + * Uses CompletionStage (not Future) to ensure fault tolerance annotations + * react properly to asynchronous failures. With CompletionStage, exceptions + * in exceptionally-completed stages trigger @Retry, @CircuitBreaker, and @Fallback. + * + * @param paymentId Payment identifier + * @param recipient Notification recipient email/phone + * @return CompletionStage that completes when notification is sent + */ + @Asynchronous + @Bulkhead(value = 10, waitingTaskQueue = 20) + @Timeout(5000) + @Fallback(fallbackMethod = "fallbackSendNotification") + public CompletionStage sendPaymentNotification( + String paymentId, + String recipient + ) { + logger.info("Notification queued for payment: " + paymentId + " to " + recipient); + + // Schedule actual notification work in background (fire-and-forget) + // This allows the HTTP response to return immediately + CompletableFuture.runAsync(() -> { + try { + // Simulate notification sending delay (e.g., calling external SMS/email service) + simulateDelay(2000); + + // Simulate notification failures (80% success rate) + if (Math.random() > 0.8) { + logger.warning("Notification service unavailable for payment: " + paymentId); + throw new RuntimeException("Notification service unavailable"); + } + + logger.info("Notification sent successfully for payment: " + paymentId); + } catch (Exception e) { + logger.severe("Failed to send notification for payment: " + paymentId + " - " + e.getMessage()); + } + }); + + // Return immediately - client gets instant response + // @Asynchronous annotation ensures this executes on a background thread, + // but the CompletionStage itself completes immediately + return CompletableFuture.completedFuture("Notification queued for processing"); } /** - * Fallback method when payment processing fails. - * - * @param paymentDetails details of the payment - * @return response message for fallback + * Fallback for notification - logs failure for later retry. + * + * @param paymentId Payment identifier + * @param recipient Notification recipient + * @return CompletionStage indicating notification was queued */ - public CompletionStage fallbackProcessPayment(PaymentDetails paymentDetails) { - System.out.println("Fallback invoked for payment of amount: " + paymentDetails.getAmount()); - return CompletableFuture.completedFuture("{\"status\":\"failed\", \"message\":\"Payment service is currently unavailable.\"}"); + public CompletionStage fallbackSendNotification( + String paymentId, + String recipient + ) { + logger.warning("Failed to send notification for payment: " + paymentId); + + // In production: queue notification for retry, use backup channel (SMS if email failed), etc. + return CompletableFuture.completedFuture( + "Notification queued for retry" + ); } /** - * Simulate a delay in processing to demonstrate timeout. + * Simulate network/processing delay. + * + * @param milliseconds Delay duration in milliseconds */ - private void simulateDelay() { + private void simulateDelay(long milliseconds) { try { - Thread.sleep(1500); // Simulated long-running task + Thread.sleep(milliseconds); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new RuntimeException("Processing interrupted"); + throw new RuntimeException("Processing interrupted", e); } } } diff --git a/code/chapter08/payment/src/main/java/io/microprofile/tutorial/store/payment/service/payment.http b/code/chapter08/payment/src/main/java/io/microprofile/tutorial/store/payment/service/payment.http index 98ae2e52..2977a452 100644 --- a/code/chapter08/payment/src/main/java/io/microprofile/tutorial/store/payment/service/payment.http +++ b/code/chapter08/payment/src/main/java/io/microprofile/tutorial/store/payment/service/payment.http @@ -1,4 +1,4 @@ -POST https://orange-zebra-r745vp6rjcxp67-9080.app.github.dev/payment/authorize +POST https://:/payment/authorize { "cardNumber": "4111111111111111", @@ -6,4 +6,4 @@ POST https://orange-zebra-r745vp6rjcxp67-9080.app.github.dev/payment/authorize "expiryDate": "12/25", "securityCode": "123", "amount": 100.00 -} \ No newline at end of file +} diff --git a/code/chapter08/payment/src/main/liberty/config/server.xml b/code/chapter08/payment/src/main/liberty/config/server.xml index 9d5f15b8..19577622 100644 --- a/code/chapter08/payment/src/main/liberty/config/server.xml +++ b/code/chapter08/payment/src/main/liberty/config/server.xml @@ -1,13 +1,12 @@ + microProfile-7.1 jakartaEE-10.0 - microProfile-6.1 restfulWS jsonp - jsonb cdi mpConfig - mpOpenAPI + mpOpenAPI-4.1 mpHealth mpMetrics mpFaultTolerance @@ -17,6 +16,6 @@ id="defaultHttpEndpoint" host="*" /> - + - \ No newline at end of file + diff --git a/code/chapter08/payment/test-payment-async.sh b/code/chapter08/payment/test-payment-async.sh index 2803b66e..eb15b28e 100755 --- a/code/chapter08/payment/test-payment-async.sh +++ b/code/chapter08/payment/test-payment-async.sh @@ -1,10 +1,7 @@ #!/bin/bash -# Enhanced test script for verifying asynchronous processing -# This script shows the asynchronous behavior by: -# 1. Checking for concurrent processing -# 2. Monitoring response times to verify non-blocking behavior -# 3. Analyzing the server logs to confirm retry patterns +# Test script for asynchronous payment notification processing +# Demonstrates @Asynchronous annotation behavior # Color definitions RED='\033[0;31m' @@ -15,107 +12,172 @@ PURPLE='\033[0;35m' CYAN='\033[0;36m' NC='\033[0m' # No Color -# Check if bc is installed and install it if not +# Check if bc is installed if ! command -v bc &> /dev/null; then - echo -e "${YELLOW}The 'bc' command is not found. Installing bc...${NC}" + echo -e "${YELLOW}Installing bc...${NC}" sudo apt-get update && sudo apt-get install -y bc - if [ $? -ne 0 ]; then - echo -e "${RED}Failed to install bc. Please install it manually.${NC}" - exit 1 - fi - echo -e "${GREEN}bc installed successfully.${NC}" fi -# Set payment endpoint URL -HOST="localhost" -PAYMENT_URL="http://${HOST}:9080/payment/api/authorize" +# Dynamically determine the base URL +if [ -n "$CODESPACE_NAME" ] && [ -n "$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN" ]; then + BASE_URL="http://localhost:9080/payment/api" + echo -e "${CYAN}Detected GitHub Codespaces environment${NC}" +elif [ -n "$GITPOD_WORKSPACE_URL" ]; then + GITPOD_HOST=$(echo $GITPOD_WORKSPACE_URL | sed 's|https://||' | sed 's|/||') + BASE_URL="https://9080-$GITPOD_HOST/payment/api" + echo -e "${CYAN}Detected Gitpod environment${NC}" +else + BASE_URL="http://localhost:9080/payment/api" + echo -e "${CYAN}Using local environment${NC}" +fi -echo -e "${BLUE}=== Enhanced Asynchronous Testing ====${NC}" -echo -e "${CYAN}This test will send a series of requests to demonstrate asynchronous processing${NC}" +echo "" +echo -e "${BLUE}=== Testing Asynchronous Payment Notifications ===${NC}" +echo -e "${CYAN}Endpoint: POST /notify/{paymentId}${NC}" +echo -e "${CYAN}Base URL: $BASE_URL${NC}" echo "" -# First, let's check server logs before test -echo -e "${YELLOW}Checking server logs before test...${NC}" -echo -e "${CYAN}(This establishes a baseline for comparison)${NC}" -cd /workspaces/liberty-rest-app/payment -MESSAGES_LOG="target/liberty/wlp/usr/servers/mpServer/logs/messages.log" - -if [ -f "$MESSAGES_LOG" ]; then - echo -e "${PURPLE}Server log file exists at: $MESSAGES_LOG${NC}" - # Count initial payment processing messages - INITIAL_PROCESSING_COUNT=$(grep -c "Processing payment for amount" "$MESSAGES_LOG") - INITIAL_FALLBACK_COUNT=$(grep -c "Fallback invoked for payment" "$MESSAGES_LOG") - - echo -e "${CYAN}Initial payment processing count: $INITIAL_PROCESSING_COUNT${NC}" - echo -e "${CYAN}Initial fallback count: $INITIAL_FALLBACK_COUNT${NC}" -else - echo -e "${RED}Server log file not found at: $MESSAGES_LOG${NC}" - INITIAL_PROCESSING_COUNT=0 - INITIAL_FALLBACK_COUNT=0 -fi +echo -e "${YELLOW}Asynchronous Configuration:${NC}" +echo " • Method: sendPaymentNotification()" +echo " • Return Type: CompletionStage" +echo " • Processing: Non-blocking" +echo " • Simulated delay: ~2 seconds" +echo "" +echo -e "${CYAN}🔍 Testing Goals:${NC}" +echo " 1. Verify requests return immediately (non-blocking)" +echo " 2. Demonstrate concurrent processing" +echo " 3. Show CompletionStage behavior" echo "" -echo -e "${YELLOW}Now sending 3 requests in rapid succession...${NC}" -# Function to send request and measure time -send_request() { +# Function to send notification and measure response time +send_async_notification() { local id=$1 - local amount=$2 - local start_time=$(date +%s.%N) + local payment_id=$2 - response=$(curl -s -X POST "${PAYMENT_URL}?amount=${amount}") + start_time=$(date +%s.%N) + response=$(curl -s -w "\nHTTP_STATUS:%{http_code}\nTIME_TOTAL:%{time_total}" \ + -X POST "${BASE_URL}/notify/${payment_id}" 2>/dev/null || echo "HTTP_STATUS:000") + end_time=$(date +%s.%N) - local end_time=$(date +%s.%N) - local duration=$(echo "$end_time - $start_time" | bc) + http_code=$(echo "$response" | grep "HTTP_STATUS:" | cut -d: -f2) + curl_time=$(echo "$response" | grep "TIME_TOTAL:" | cut -d: -f2) + body=$(echo "$response" | sed '/HTTP_STATUS:/d' | sed '/TIME_TOTAL:/d') - echo -e "${GREEN}[Request $id] Completed in ${duration}s${NC}" - echo -e "${CYAN}[Request $id] Response: $response${NC}" + wall_time=$(echo "$end_time - $start_time" | bc) + wall_time_formatted=$(printf "%.3f" $wall_time) - return 0 + if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then + # Check if response was truly async (should be fast, not 2 seconds) + is_async=$(echo "$curl_time < 0.5" | bc) + if [ "$is_async" -eq 1 ]; then + echo -e "${GREEN}[Request $id] ✓ ASYNC - Returned immediately (${wall_time_formatted}s)${NC}" + echo -e "${GREEN} Response: $body${NC}" + else + echo -e "${YELLOW}[Request $id] ⚠ SYNC? - Took ${wall_time_formatted}s (expected < 0.5s)${NC}" + echo -e "${YELLOW} Response: $body${NC}" + fi + else + echo -e "${RED}[Request $id] ✗ ERROR (HTTP $http_code, ${wall_time_formatted}s)${NC}" + echo -e "${RED} Response: $body${NC}" + fi + + echo "$wall_time_formatted" } -# Send 3 requests in rapid succession -for i in {1..3}; do - # Use a fixed amount for consistency - amount=25.99 - echo -e "${PURPLE}[Request $i] Sending request for \$$amount...${NC}" - send_request $i $amount & - # Sleep briefly to ensure log messages are distinguishable - sleep 0.1 +# Test 1: Single async request +echo -e "${BLUE}=== Test 1: Single Asynchronous Request ===${NC}" +echo -e "${CYAN}Sending single notification to verify async behavior...${NC}" +echo "" + +send_async_notification 1 "PAY-00001" + +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Test 2: Multiple sequential requests +echo -e "${BLUE}=== Test 2: Sequential Asynchronous Requests ===${NC}" +echo -e "${CYAN}Sending 5 sequential requests...${NC}" +echo -e "${YELLOW}Each should return immediately (< 0.5s) despite 2s processing time${NC}" +echo "" + +total_start=$(date +%s.%N) + +for i in {2..6}; do + payment_id=$(printf "PAY-%05d" $i) + send_async_notification $i $payment_id done -# Wait for all background processes to complete -wait +total_end=$(date +%s.%N) +total_time=$(echo "$total_end - $total_start" | bc) +total_time_formatted=$(printf "%.2f" $total_time) echo "" -echo -e "${YELLOW}Waiting 5 seconds for processing to complete...${NC}" -sleep 5 +echo -e "${PURPLE}Total time for 5 sequential requests: ${total_time_formatted}s${NC}" +echo -e "${CYAN}Expected: < 2.5s if async, ~10s if blocking${NC}" -# Check the server logs after test -echo -e "${YELLOW}Checking server logs after test...${NC}" +if (( $(echo "$total_time < 3" | bc -l) )); then + echo -e "${GREEN}✓ CONFIRMED: Requests are truly asynchronous!${NC}" +else + echo -e "${YELLOW}⚠ WARNING: Requests may be blocking (total time too high)${NC}" +fi -if [ -f "$MESSAGES_LOG" ]; then - # Count final payment processing messages - FINAL_PROCESSING_COUNT=$(grep -c "Processing payment for amount" "$MESSAGES_LOG") - FINAL_FALLBACK_COUNT=$(grep -c "Fallback invoked for payment" "$MESSAGES_LOG") - - NEW_PROCESSING=$(($FINAL_PROCESSING_COUNT - $INITIAL_PROCESSING_COUNT)) - NEW_FALLBACKS=$(($FINAL_FALLBACK_COUNT - $INITIAL_FALLBACK_COUNT)) - - echo -e "${CYAN}New payment processing events: $NEW_PROCESSING${NC}" - echo -e "${CYAN}New fallback events: $NEW_FALLBACKS${NC}" - - # Extract the latest log entries - echo "" - echo -e "${BLUE}Latest server log entries related to payment processing:${NC}" - grep "Processing payment for amount\|Fallback invoked for payment" "$MESSAGES_LOG" | tail -10 +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Test 3: Concurrent requests +echo -e "${BLUE}=== Test 3: Concurrent Asynchronous Requests ===${NC}" +echo -e "${CYAN}Sending 5 concurrent requests in parallel...${NC}" +echo -e "${YELLOW}All should complete quickly, demonstrating true async processing${NC}" +echo "" + +concurrent_start=$(date +%s.%N) + +# Launch background jobs +for i in {7..11}; do + payment_id=$(printf "PAY-%05d" $i) + (send_async_notification $i $payment_id) & + sleep 0.05 # Small stagger to make output readable +done + +# Wait for all background jobs +wait + +concurrent_end=$(date +%s.%N) +concurrent_time=$(echo "$concurrent_end - $concurrent_start" | bc) +concurrent_time_formatted=$(printf "%.2f" $concurrent_time) + +echo "" +echo -e "${PURPLE}Total time for 5 concurrent requests: ${concurrent_time_formatted}s${NC}" +echo -e "${CYAN}Expected: ~0.5-1.0s (all start immediately)${NC}" + +if (( $(echo "$concurrent_time < 2" | bc -l) )); then + echo -e "${GREEN}✓ EXCELLENT: Concurrent async processing confirmed!${NC}" else - echo -e "${RED}Server log file not found after test${NC}" + echo -e "${YELLOW}⚠ Note: Time higher than expected${NC}" fi echo "" -echo -e "${BLUE}=== Asynchronous Behavior Analysis ====${NC}" -echo -e "${CYAN}1. Rapid response times indicate non-blocking behavior${NC}" -echo -e "${CYAN}2. Multiple processing entries in logs show concurrent execution${NC}" -echo -e "${CYAN}3. Fallbacks demonstrate the fault tolerance mechanism${NC}" -echo -e "${CYAN}4. All @Asynchronous methods return quickly while processing continues in background${NC}" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Summary +echo -e "${BLUE}=== Asynchronous Processing Summary ===${NC}" +echo "" +echo -e "${CYAN}Key Observations:${NC}" +echo " • @Asynchronous methods return CompletionStage" +echo " • JAX-RS endpoint returns CompletionStage" +echo " • Client receives immediate response (non-blocking)" +echo " • Server processes work in background thread pool" +echo " • Multiple requests can execute concurrently" +echo "" + +echo -e "${CYAN}CompletionStage vs Future:${NC}" +echo " • CompletionStage: Fault tolerance applied, supports chaining" +echo " • Future: Fault tolerance NOT applied (avoid!)" +echo "" + +echo -e "${GREEN}=== Test Complete ===${NC}" +echo "" diff --git a/code/chapter08/payment/test-payment-basic.sh b/code/chapter08/payment/test-payment-basic.sh index 1eecb6ff..ab85d096 100755 --- a/code/chapter08/payment/test-payment-basic.sh +++ b/code/chapter08/payment/test-payment-basic.sh @@ -1,55 +1,120 @@ #!/bin/bash -# Simple test script for payment service +# Basic test script for payment service +# Tests all three fault-tolerant endpoints # Color definitions RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' NC='\033[0m' # No Color -# Check if bc is installed and install it if not +# Check if bc is installed if ! command -v bc &> /dev/null; then echo -e "${YELLOW}The 'bc' command is not found. Installing bc...${NC}" sudo apt-get update && sudo apt-get install -y bc - if [ $? -ne 0 ]; then - echo -e "${RED}Failed to install bc. Please install it manually.${NC}" - exit 1 - fi - echo -e "${GREEN}bc installed successfully.${NC}" fi -HOST="localhost" -PAYMENT_URL="http://${HOST}:9080/payment/api/authorize" +# Dynamically determine the base URL +if [ -n "$CODESPACE_NAME" ] && [ -n "$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN" ]; then + BASE_URL="http://localhost:9080/payment/api" + echo -e "${CYAN}Detected GitHub Codespaces environment${NC}" +elif [ -n "$GITPOD_WORKSPACE_URL" ]; then + GITPOD_HOST=$(echo $GITPOD_WORKSPACE_URL | sed 's|https://||' | sed 's|/||') + BASE_URL="https://9080-$GITPOD_HOST/payment/api" + echo -e "${CYAN}Detected Gitpod environment${NC}" +else + BASE_URL="http://localhost:9080/payment/api" + echo -e "${CYAN}Using local environment${NC}" +fi -# Hardcoded amount for testing -AMOUNT=25.99 +echo -e "${BLUE}=== Payment Service Basic Test ===${NC}" +echo -e "${CYAN}Base URL: $BASE_URL${NC}" +echo "" -echo -e "${YELLOW}Sending payment request for \$$AMOUNT...${NC}" +# Test 1: Payment Authorization +echo -e "${YELLOW}Test 1: Payment Authorization (Retry + Timeout + Fallback)${NC}" +echo -e "${CYAN}Endpoint: POST /authorize?amount=100${NC}" echo "" -# Capture start time start_time=$(date +%s.%N) +response=$(curl -s -X POST "${BASE_URL}/authorize?amount=100") +end_time=$(date +%s.%N) +duration=$(echo "$end_time - $start_time" | bc) -# Send request -response=$(curl -s -X POST "${PAYMENT_URL}?amount=${AMOUNT}") +echo -e "${GREEN}Response:${NC} $response" +echo -e "${GREEN}Duration:${NC} ${duration}s" +echo "" -# Capture end time -end_time=$(date +%s.%N) +if echo "$response" | grep -q "success"; then + echo -e "${GREEN}✓ SUCCESS: Payment authorized${NC}" +elif echo "$response" | grep -q "fallback"; then + echo -e "${YELLOW}⚠ FALLBACK: Used fallback response${NC}" +else + echo -e "${RED}✗ ERROR: Unexpected response${NC}" +fi -# Calculate duration +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Test 2: Gateway Health Check +echo -e "${YELLOW}Test 2: Gateway Health Check (Circuit Breaker + Timeout)${NC}" +echo -e "${CYAN}Endpoint: GET /health/gateway${NC}" +echo "" + +start_time=$(date +%s.%N) +response=$(curl -s -X GET "${BASE_URL}/health/gateway") +end_time=$(date +%s.%N) duration=$(echo "$end_time - $start_time" | bc) +echo -e "${GREEN}Response:${NC} $response" +echo -e "${GREEN}Duration:${NC} ${duration}s" echo "" -echo -e "${GREEN}✓ Request completed in ${duration} seconds${NC}" -echo -e "${YELLOW}Response:${NC} $response" + +if echo "$response" | grep -q "healthy"; then + echo -e "${GREEN}✓ SUCCESS: Gateway is healthy${NC}" +elif echo "$response" | grep -q "degraded"; then + echo -e "${YELLOW}⚠ WARNING: Gateway health degraded (Circuit Breaker may be OPEN)${NC}" +else + echo -e "${RED}✗ ERROR: Unexpected response${NC}" +fi + +echo "" +echo -e "${BLUE}----------------------------------------${NC}" echo "" -# Show if the response indicates success or fallback -if echo "$response" | grep -q "success"; then - echo -e "${GREEN}✓ SUCCESS: Payment processed successfully${NC}" -elif echo "$response" | grep -q "failed"; then - echo -e "${RED}✗ FALLBACK: Payment service unavailable${NC}" +# Test 3: Async Payment Notification +echo -e "${YELLOW}Test 3: Payment Notification (Async + Bulkhead + Timeout + Fallback)${NC}" +echo -e "${CYAN}Endpoint: POST /notify/PAY-12345${NC}" +echo "" + +start_time=$(date +%s.%N) +response=$(curl -s -X POST "${BASE_URL}/notify/PAY-12345") +end_time=$(date +%s.%N) +duration=$(echo "$end_time - $start_time" | bc) + +echo -e "${GREEN}Response:${NC} $response" +echo -e "${GREEN}Duration:${NC} ${duration}s" +echo "" + +if echo "$response" | grep -q "queued\|sent"; then + echo -e "${GREEN}✓ SUCCESS: Notification queued/sent${NC}" +elif echo "$response" | grep -q "fallback"; then + echo -e "${YELLOW}⚠ FALLBACK: Used fallback notification${NC}" else echo -e "${RED}✗ ERROR: Unexpected response${NC}" fi + +echo "" +echo -e "${BLUE}========================================${NC}" +echo -e "${GREEN}Basic tests complete!${NC}" +echo "" +echo -e "${CYAN}Next steps:${NC}" +echo -e " • Run ${YELLOW}test-payment-retry.sh${NC} to test retry behavior" +echo -e " • Run ${YELLOW}test-payment-circuit-breaker.sh${NC} to test circuit breaker states" +echo -e " • Run ${YELLOW}test-payment-bulkhead.sh${NC} to test bulkhead limits" +echo -e " • Check metrics at ${BLUE}http://localhost:9080/metrics?scope=base${NC}" +echo "" diff --git a/code/chapter08/payment/test-payment-bulkhead.sh b/code/chapter08/payment/test-payment-bulkhead.sh index 86199af5..2e26d6e4 100755 --- a/code/chapter08/payment/test-payment-bulkhead.sh +++ b/code/chapter08/payment/test-payment-bulkhead.sh @@ -1,8 +1,7 @@ #!/bin/bash # Test script for Payment Service Bulkhead functionality -# This script demonstrates the @Bulkhead annotation by sending many concurrent requests -# and observing how the service handles concurrent load up to its configured limit (5) +# Tests the @Bulkhead annotation on the sendPaymentNotification method # Color definitions RED='\033[0;31m' @@ -13,45 +12,188 @@ PURPLE='\033[0;35m' CYAN='\033[0;36m' NC='\033[0m' # No Color -# Check if bc is installed and install it if not +# Check if bc is installed if ! command -v bc &> /dev/null; then - echo -e "${YELLOW}The 'bc' command is not found. Installing bc...${NC}" + echo -e "${YELLOW}Installing bc for duration calculations...${NC}" sudo apt-get update && sudo apt-get install -y bc - if [ $? -ne 0 ]; then - echo -e "${RED}Failed to install bc. Please install it manually.${NC}" - exit 1 - fi - echo -e "${GREEN}bc installed successfully.${NC}" fi -# Header -echo -e "${BLUE}==============================================${NC}" -echo -e "${BLUE} Payment Service Bulkhead Test ${NC}" -echo -e "${BLUE}==============================================${NC}" -echo "" - # Dynamically determine the base URL if [ -n "$CODESPACE_NAME" ] && [ -n "$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN" ]; then - # GitHub Codespaces environment - use localhost for internal testing BASE_URL="http://localhost:9080/payment/api" - echo -e "${CYAN}Detected GitHub Codespaces environment (using localhost)${NC}" - echo -e "${CYAN}Note: External access would be https://$CODESPACE_NAME-9080.$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN/payment/api${NC}" + METRICS_URL="http://localhost:9080/metrics" + echo -e "${CYAN}Detected GitHub Codespaces environment${NC}" elif [ -n "$GITPOD_WORKSPACE_URL" ]; then - # Gitpod environment GITPOD_HOST=$(echo $GITPOD_WORKSPACE_URL | sed 's|https://||' | sed 's|/||') BASE_URL="https://9080-$GITPOD_HOST/payment/api" + METRICS_URL="https://9080-$GITPOD_HOST/metrics" echo -e "${CYAN}Detected Gitpod environment${NC}" else - # Local or other environment BASE_URL="http://localhost:9080/payment/api" + METRICS_URL="http://localhost:9080/metrics" echo -e "${CYAN}Using local environment${NC}" fi +echo "" +echo -e "${BLUE}=== Payment Notification Bulkhead Test ===${NC}" +echo -e "${CYAN}Endpoint: POST /notify/{paymentId}${NC}" echo -e "${CYAN}Base URL: $BASE_URL${NC}" echo "" -# Set up log monitoring -LOG_FILE="/workspaces/liberty-rest-app/payment/target/liberty/wlp/usr/servers/mpServer/logs/messages.log" +echo -e "${YELLOW}Bulkhead Configuration:${NC}" +echo " • Maximum Concurrent Requests: 3" +echo " • Waiting Queue Size: 2" +echo " • Total Capacity: 5 (3 concurrent + 2 queued)" +echo " • Asynchronous: Yes (@Asynchronous)" +echo " • Processing delay: ~2 seconds per notification" +echo "" + +echo -e "${CYAN}🔍 Expected Behavior:${NC}" +echo " • First 3 requests: Execute immediately (concurrent slots)" +echo " • Next 2 requests: Queue for execution (waiting queue)" +echo " • 6th+ requests: ${RED}REJECTED${NC} - Bulkhead full" +echo "" + +read -p "Press Enter to start the bulkhead test..." +echo "" + +# Function to send async notification request +send_notification() { + local id=$1 + local payment_id="PAY-$(printf "%05d" $id)" + + start_time=$(date +%s.%N) + response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" -X POST "${BASE_URL}/notify/${payment_id}" 2>/dev/null || echo "HTTP_STATUS:000") + end_time=$(date +%s.%N) + + duration=$(echo "$end_time - $start_time" | bc) + duration_formatted=$(printf "%.3f" $duration) + + http_code=$(echo "$response" | grep "HTTP_STATUS:" | cut -d: -f2) + body=$(echo "$response" | sed '/HTTP_STATUS:/d') + + if [ "$http_code" -eq 200 ] || [ "$http_code" -eq 202 ]; then + if echo "$body" | grep -q "queued\|sent"; then + echo -e "${GREEN}[Request $id] ✓ Accepted (${duration_formatted}s) - Response: $body${NC}" + elif echo "$body" | grep -q "fallback"; then + echo -e "${PURPLE}[Request $id] ⚡ Fallback (${duration_formatted}s) - Response: $body${NC}" + else + echo -e "${GREEN}[Request $id] ✓ Success (${duration_formatted}s) - Response: $body${NC}" + fi + elif echo "$body" | grep -q "BulkheadException\|Bulkhead"; then + echo -e "${RED}[Request $id] ✗ REJECTED: Bulkhead full (${duration_formatted}s)${NC}" + elif [ "$http_code" -eq 503 ]; then + echo -e "${YELLOW}[Request $id] ⚠ Service Unavailable (${duration_formatted}s)${NC}" + else + echo -e "${PURPLE}[Request $id] ? Unknown (HTTP $http_code, ${duration_formatted}s): $body${NC}" + fi +} + +# Phase 1: Single request baseline +echo -e "${BLUE}=== Phase 1: Single Request (Baseline) ===${NC}" +echo -e "${CYAN}Establishing baseline performance...${NC}" +echo "" + +send_notification 1 +sleep 3 + +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Phase 2: Test bulkhead capacity +echo -e "${BLUE}=== Phase 2: Testing Bulkhead Capacity (10 concurrent requests) ===${NC}" +echo -e "${CYAN}Sending 10 concurrent requests...${NC}" +echo -e "${YELLOW}Expected:${NC}" +echo -e " • Requests 1-3: ${GREEN}Execute immediately (concurrent slots)${NC}" +echo -e " • Requests 4-5: ${CYAN}Queued (waiting queue)${NC}" +echo -e " • Requests 6-10: ${RED}REJECTED (bulkhead full)${NC}" +echo "" + +# Send 10 requests concurrently in background +for i in {1..10}; do + send_notification $i & + sleep 0.1 # Small delay to stagger requests slightly +done + +# Wait for all background jobs to complete +wait + +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Phase 3: Check metrics +echo -e "${BLUE}=== Phase 3: Bulkhead Metrics ===${NC}" +echo -e "${CYAN}Fetching fault tolerance metrics...${NC}" +echo "" + +sleep 1 +metrics=$(curl -s "$METRICS_URL" 2>/dev/null | grep -i "bulkhead.*sendPaymentNotification") + +if [ -n "$metrics" ]; then + echo "$metrics" | while IFS= read -r line; do + if echo "$line" | grep -q "callsAccepted\|accepted"; then + echo -e "${GREEN}$line${NC}" + elif echo "$line" | grep -q "callsRejected\|rejected"; then + echo -e "${RED}$line${NC}" + elif echo "$line" | grep -q "executionDuration\|duration\|running"; then + echo -e "${CYAN}$line${NC}" + else + echo "$line" + fi + done +else + echo -e "${YELLOW}No bulkhead metrics found${NC}" + echo -e "${CYAN}Make sure the service is running and has processed some requests${NC}" + echo -e "${CYAN}Checking all available fault tolerance metrics...${NC}" + echo "" + all_ft_metrics=$(curl -s "$METRICS_URL" 2>/dev/null | grep -i "ft.*bulkhead" | head -20) + if [ -n "$all_ft_metrics" ]; then + echo -e "${BLUE}Available bulkhead metrics:${NC}" + echo "$all_ft_metrics" + else + echo -e "${YELLOW}No fault tolerance bulkhead metrics found at all${NC}" + fi +fi + +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Phase 4: Verify recovery after wait +echo -e "${BLUE}=== Phase 4: Testing Recovery ===${NC}" +echo -e "${CYAN}Waiting 3 seconds for in-progress notifications to complete...${NC}" +sleep 3 + +echo "" +echo -e "${CYAN}Sending new requests to verify bulkhead has freed up...${NC}" +echo "" + +for i in {11..13}; do + send_notification $i + sleep 0.5 +done + +echo "" +echo -e "${GREEN}=== Bulkhead Test Complete ===${NC}" +echo "" + +echo -e "${CYAN}Summary:${NC}" +echo " • Bulkhead limits concurrent async executions" +echo " • Requests beyond capacity are rejected immediately" +echo " • Protects system resources from exhaustion" +echo " • Works with @Asynchronous for non-blocking behavior" +echo "" + +echo -e "${CYAN}To view all bulkhead metrics:${NC}" +echo -e "${BLUE}curl $METRICS_URL | grep -i bulkhead${NC}" +echo "" +echo "" + +# Set up log monitoring - use relative path from script location +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LOG_FILE="$SCRIPT_DIR/target/liberty/wlp/usr/servers/mpServer/logs/messages.log" # Get initial log position for later analysis if [ -f "$LOG_FILE" ]; then diff --git a/code/chapter08/payment/test-payment-circuit-breaker.sh b/code/chapter08/payment/test-payment-circuit-breaker.sh new file mode 100644 index 00000000..e11a20de --- /dev/null +++ b/code/chapter08/payment/test-payment-circuit-breaker.sh @@ -0,0 +1,319 @@ +#!/bin/bash + +# Test script for Payment Service Circuit Breaker functionality +# Tests the @CircuitBreaker annotation on the checkGatewayHealth method + +# Color definitions +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +PURPLE='\033[0;35m' +NC='\033[0m' # No Color + +# Dynamically determine the base URL +if [ -n "$CODESPACE_NAME" ] && [ -n "$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN" ]; then + BASE_URL="http://localhost:9080/payment/api" + METRICS_URL="http://localhost:9080/metrics?scope=base" + echo -e "${CYAN}Detected GitHub Codespaces environment${NC}" +elif [ -n "$GITPOD_WORKSPACE_URL" ]; then + GITPOD_HOST=$(echo $GITPOD_WORKSPACE_URL | sed 's|https://||' | sed 's|/||') + BASE_URL="https://9080-$GITPOD_HOST/payment/api" + METRICS_URL="https://9080-$GITPOD_HOST/metrics?scope=base" + echo -e "${CYAN}Detected Gitpod environment${NC}" +else + BASE_URL="http://localhost:9080/payment/api" + METRICS_URL="http://localhost:9080/metrics?scope=base" + echo -e "${CYAN}Using local environment${NC}" +fi + +echo "" +echo -e "${BLUE}=== Payment Gateway Circuit Breaker Test ===${NC}" +echo -e "${CYAN}Endpoint: GET /health/gateway${NC}" +echo -e "${CYAN}Base URL: $BASE_URL${NC}" +echo "" + +echo -e "${YELLOW}Circuit Breaker Configuration:${NC}" +echo " • Request Volume Threshold: 4" +echo " • Failure Ratio: 0.5 (50%)" +echo " • Delay: 5000ms" +echo " • Success Threshold: 2" +echo " • Simulated failure rate: 50%" +echo "" + +echo -e "${CYAN}🔍 Circuit Breaker States:${NC}" +echo -e " • ${GREEN}CLOSED${NC} (0): Normal operation, requests pass through" +echo -e " • ${RED}OPEN${NC} (1): Too many failures, requests immediately fail" +echo -e " • ${YELLOW}HALF_OPEN${NC} (2): Testing if service recovered" +echo "" + +echo -e "${PURPLE}This test will:${NC}" +echo " 1. Send initial requests to establish baseline" +echo " 2. Trigger failures to trip the circuit breaker to OPEN" +echo " 3. Wait for delay period (5s)" +echo " 4. Test HALF_OPEN state (probing for recovery)" +echo " 5. Verify circuit returns to CLOSED on success" +echo "" + +read -p "Press Enter to start the circuit breaker test..." +echo "" + +# Function to check circuit breaker state from metrics +check_circuit_state() { + # Fetch all metrics + metrics=$(curl -s "$METRICS_URL" 2>/dev/null) + + # Try to find circuit breaker state metric (MicroProfile FT uses labels for method name) + # Look for ft_circuitbreaker_state_total_nanoseconds with checkGatewayHealth in method label + state_metrics=$(echo "$metrics" | grep "ft_circuitbreaker_state_total" | grep "checkGatewayHealth") + + if [ -z "$state_metrics" ]; then + # Check if we can reach the metrics endpoint at all + if [ -z "$metrics" ]; then + echo -e "${RED}✗ Unable to reach metrics endpoint${NC}" + echo -e "${YELLOW} Make sure the payment service is running on port 9080${NC}" + else + # Metrics endpoint works, but circuit breaker metrics not found + echo -e "${YELLOW}⚠ Circuit breaker metrics not yet available${NC}" + echo -e "${CYAN} (This is normal if no health check requests have been made yet)${NC}" + fi + return + fi + + # Determine current state by checking which state has recent activity + # The state with the highest time value is likely the current state + closed_time=$(echo "$state_metrics" | grep 'state="closed"' | grep -o '[0-9.E+]*$' || echo "0") + open_time=$(echo "$state_metrics" | grep 'state="open"' | grep -o '[0-9.E+]*$' || echo "0") + halfopen_time=$(echo "$state_metrics" | grep 'state="halfOpen"' | grep -o '[0-9.E+]*$' || echo "0") + + # Simple heuristic: Check if open or halfOpen have recent values + if echo "$state_metrics" | grep -q 'state="open"' && [ "$open_time" != "0" ]; then + # Check if we're in half-open by looking at recent halfOpen time + if echo "$state_metrics" | grep -q 'state="halfOpen"' && [ "$halfopen_time" != "0" ]; then + echo -e "${YELLOW}Circuit Breaker State: HALF_OPEN (2) - Testing recovery${NC}" + else + echo -e "${RED}Circuit Breaker State: OPEN (1) - Blocking requests${NC}" + fi + else + echo -e "${GREEN}Circuit Breaker State: CLOSED (0) - Normal operation${NC}" + fi +} + +# Function to make health check request +make_health_request() { + local request_num=$1 + + start_time=$(date +%s.%N) + response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" -X GET "${BASE_URL}/health/gateway" 2>/dev/null || echo "HTTP_STATUS:000") + end_time=$(date +%s.%N) + + duration=$(echo "$end_time - $start_time" | bc) + duration_formatted=$(printf "%.3f" $duration) + + http_code=$(echo "$response" | grep "HTTP_STATUS:" | cut -d: -f2) + body=$(echo "$response" | sed '/HTTP_STATUS:/d') + + if echo "$body" | grep -q "\"status\":\"healthy\""; then + echo -e "${GREEN}[Request $request_num] ✓ Gateway healthy (${duration_formatted}s)${NC}" + elif echo "$body" | grep -q "\"status\":\"unhealthy\""; then + echo -e "${YELLOW}[Request $request_num] ⚠ Gateway unhealthy (${duration_formatted}s)${NC}" + elif echo "$body" | grep -q "\"status\":\"circuit_open\""; then + echo -e "${RED}[Request $request_num] ✗ Circuit OPEN - Request blocked (${duration_formatted}s)${NC}" + else + echo -e "${PURPLE}[Request $request_num] ? Unknown status (${duration_formatted}s): $body${NC}" + fi +} + +# Phase 0: Ensure circuit is CLOSED before starting test +echo -e "${BLUE}=== Phase 0: Initialization ===${NC}" +echo -e "${CYAN}Checking current circuit breaker state...${NC}" +echo "" + +# Check initial state and wait if needed +metrics=$(curl -s "$METRICS_URL" 2>/dev/null) +state_metrics=$(echo "$metrics" | grep "ft_circuitbreaker_state_total" | grep "checkGatewayHealth") + +if [ -z "$state_metrics" ]; then + echo -e "${YELLOW}⚠ Circuit breaker metrics not yet available${NC}" + echo -e "${CYAN}Sending initial requests to initialize circuit breaker...${NC}" + # Send a few requests to initialize the circuit breaker + for i in {1..2}; do + curl -s "${BASE_URL}/health/gateway" > /dev/null 2>&1 + sleep 0.5 + done + sleep 1 + echo -e "${GREEN}✓ Circuit breaker initialized${NC}" +elif echo "$state_metrics" | grep -q 'state="open"'; then + # Check if there's significant open time (circuit is OPEN) + open_time=$(echo "$state_metrics" | grep 'state="open"' | grep -o '[0-9.E+]*$') + if [ -n "$open_time" ] && [ "$open_time" != "0" ]; then + echo -e "${YELLOW}⚠ Circuit is currently OPEN (from previous test run)${NC}" + echo -e "${CYAN}Waiting 6 seconds for circuit to transition to HALF_OPEN...${NC}" + for i in {6..1}; do + echo -ne "${CYAN} Waiting... $i seconds\r${NC}" + sleep 1 + done + echo "" + echo -e "${CYAN}Sending probe requests to close the circuit...${NC}" + # Send a few successful probes to close the circuit + for i in {1..3}; do + curl -s "${BASE_URL}/health/gateway" > /dev/null 2>&1 + sleep 0.5 + done + sleep 2 + echo -e "${GREEN}✓ Circuit should now be CLOSED${NC}" + fi +elif echo "$state_metrics" | grep -q 'state="halfOpen"'; then + halfopen_time=$(echo "$state_metrics" | grep 'state="halfOpen"' | grep -o '[0-9.E+]*$') + if [ -n "$halfopen_time" ] && [ "$halfopen_time" != "0" ]; then + echo -e "${YELLOW}⚠ Circuit is currently HALF_OPEN${NC}" + echo -e "${CYAN}Sending probe requests to close the circuit...${NC}" + for i in {1..3}; do + curl -s "${BASE_URL}/health/gateway" > /dev/null 2>&1 + sleep 0.5 + done + sleep 1 + echo -e "${GREEN}✓ Circuit should now be CLOSED${NC}" + fi +else + echo -e "${GREEN}✓ Circuit is already CLOSED${NC}" +fi + +echo "" +check_circuit_state +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Phase 1: Initial requests (circuit should be CLOSED) +echo -e "${BLUE}=== Phase 1: Initial Requests (Circuit: CLOSED) ===${NC}" +echo -e "${CYAN}Sending initial requests to establish baseline...${NC}" +echo "" + +for i in {1..3}; do + make_health_request $i + sleep 0.5 +done + +echo "" +check_circuit_state +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Phase 2: Trigger failures to OPEN the circuit +echo -e "${BLUE}=== Phase 2: Triggering Circuit Breaker (CLOSED → OPEN) ===${NC}" +echo -e "${CYAN}Sending requests to trigger failures...${NC}" +echo -e "${YELLOW}Given 50% failure rate, we expect some failures${NC}" +echo "" + +for i in {4..10}; do + make_health_request $i + sleep 0.3 +done + +echo "" +check_circuit_state +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Phase 3: Circuit should be OPEN, requests should fail immediately +echo -e "${BLUE}=== Phase 3: Verify Circuit OPEN (Blocking Requests) ===${NC}" +echo -e "${CYAN}These requests should fail immediately without hitting the service...${NC}" +echo "" + +for i in {11..13}; do + make_health_request $i + sleep 0.5 +done + +echo "" +check_circuit_state +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Phase 4: Wait for delay period to transition to HALF_OPEN +echo -e "${BLUE}=== Phase 4: Waiting for Recovery Period ===${NC}" +echo -e "${CYAN}Waiting 5 seconds for circuit breaker delay...${NC}" +echo -e "${YELLOW}Circuit should transition from OPEN → HALF_OPEN${NC}" +echo "" + +for i in {5..1}; do + echo -ne "${CYAN}Waiting... $i seconds remaining\r${NC}" + sleep 1 +done +echo "" + +echo "" +check_circuit_state +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Phase 5: Test HALF_OPEN state +echo -e "${BLUE}=== Phase 5: Testing HALF_OPEN State ===${NC}" +echo -e "${CYAN}Circuit should allow probe requests to test recovery...${NC}" +echo -e "${YELLOW}Need 2 consecutive successes to return to CLOSED${NC}" +echo "" + +for i in {14..18}; do + make_health_request $i + sleep 1 + check_circuit_state + echo "" +done + +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Final state check +echo -e "${BLUE}=== Final Circuit Breaker State ===${NC}" +check_circuit_state +echo "" + +# Show metrics +echo -e "${BLUE}=== Circuit Breaker Metrics ===${NC}" +echo -e "${CYAN}Fetching fault tolerance metrics...${NC}" +echo "" + +# Use flexible pattern matching for metrics +all_metrics=$(curl -s "$METRICS_URL" 2>/dev/null) +metrics=$(echo "$all_metrics" | grep -i "circuitbreaker" | grep -i "checkGatewayHealth") + +if [ -n "$metrics" ]; then + echo "$metrics" | while IFS= read -r line; do + if echo "$line" | grep -q "opened"; then + echo -e "${YELLOW}$line${NC}" + elif echo "$line" | grep -q "state"; then + echo -e "${GREEN}$line${NC}" + elif echo "$line" | grep -q "succeeded\|failed"; then + echo -e "${CYAN}$line${NC}" + else + echo "$line" + fi + done +else + # Try alternative metric patterns + alt_metrics=$(echo "$all_metrics" | grep -E "ft.*Health.*circuit|circuit.*Health") + if [ -n "$alt_metrics" ]; then + echo -e "${CYAN}Found alternative circuit breaker metrics:${NC}" + echo "$alt_metrics" + else + echo -e "${RED}✗ No circuit breaker metrics found${NC}" + echo -e "${YELLOW}This could mean:${NC}" + echo -e "${YELLOW} • The service is not running${NC}" + echo -e "${YELLOW} • Metrics are not enabled${NC}" + echo -e "${YELLOW} • No health check requests were processed successfully${NC}" + fi +fi + +echo "" +echo -e "${GREEN}=== Circuit Breaker Test Complete ===${NC}" +echo "" +echo -e "${CYAN}To view all fault tolerance metrics:${NC}" +echo -e "${BLUE}curl $METRICS_URL | grep -i circuit${NC}" +echo "" diff --git a/code/chapter08/payment/test-payment-concurrent-load.sh b/code/chapter08/payment/test-payment-concurrent-load.sh index 94452a42..70fabde6 100755 --- a/code/chapter08/payment/test-payment-concurrent-load.sh +++ b/code/chapter08/payment/test-payment-concurrent-load.sh @@ -1,10 +1,7 @@ #!/bin/bash -# Test script for asynchronous payment processing -# This script sends multiple concurrent requests to test: -# 1. Asynchronous processing (@Asynchronous) -# 2. Resource isolation (@Bulkhead) -# 3. Retry behavior (@Retry) +# Comprehensive load test for all fault tolerance patterns +# Tests retry, circuit breaker, bulkhead, and async under concurrent load # Color definitions RED='\033[0;31m' @@ -15,109 +12,335 @@ PURPLE='\033[0;35m' CYAN='\033[0;36m' NC='\033[0m' # No Color -# Check if bc is installed and install it if not +# Check if bc is installed if ! command -v bc &> /dev/null; then - echo -e "${YELLOW}The 'bc' command is not found. Installing bc...${NC}" + echo -e "${YELLOW}Installing bc...${NC}" sudo apt-get update && sudo apt-get install -y bc - if [ $? -ne 0 ]; then - echo -e "${RED}Failed to install bc. Please install it manually.${NC}" - exit 1 - fi - echo -e "${GREEN}bc installed successfully.${NC}" fi -# Set payment endpoint URL - automatically detect if running in GitHub Codespaces -if [ -n "$CODESPACES" ]; then - HOST="localhost" +# Dynamically determine the base URL +if [ -n "$CODESPACE_NAME" ] && [ -n "$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN" ]; then + BASE_URL="http://localhost:9080/payment/api" + METRICS_URL="http://localhost:9080/metrics" + echo -e "${CYAN}Detected GitHub Codespaces environment${NC}" +elif [ -n "$GITPOD_WORKSPACE_URL" ]; then + GITPOD_HOST=$(echo $GITPOD_WORKSPACE_URL | sed 's|https://||' | sed 's|/||') + BASE_URL="https://9080-$GITPOD_HOST/payment/api" + METRICS_URL="https://9080-$GITPOD_HOST/metrics" + echo -e "${CYAN}Detected Gitpod environment${NC}" else - HOST="localhost" + BASE_URL="http://localhost:9080/payment/api" + METRICS_URL="http://localhost:9080/metrics" + echo -e "${CYAN}Using local environment${NC}" fi -PAYMENT_URL="http://${HOST}:9080/payment/api/authorize" -NUM_REQUESTS=10 -CONCURRENCY=5 +echo "" +echo -e "${BLUE}╔════════════════════════════════════════════════╗${NC}" +echo -e "${BLUE}║ Payment Service Concurrent Load Test ║${NC}" +echo -e "${BLUE}║ Testing All Fault Tolerance Patterns ║${NC}" +echo -e "${BLUE}╚════════════════════════════════════════════════╝${NC}" +echo "" +echo -e "${CYAN}Base URL: $BASE_URL${NC}" +echo "" + +# Test configuration +NUM_AUTHORIZE_REQUESTS=20 +NUM_HEALTH_REQUESTS=15 +NUM_NOTIFY_REQUESTS=15 -echo -e "${BLUE}=== Testing Asynchronous Payment Processing ===${NC}" -echo -e "${CYAN}Endpoint: ${PAYMENT_URL}${NC}" -echo -e "${CYAN}Sending ${NUM_REQUESTS} requests with concurrency ${CONCURRENCY}${NC}" +echo -e "${YELLOW}Load Test Configuration:${NC}" +echo " • Authorization requests (Retry): $NUM_AUTHORIZE_REQUESTS" +echo " • Health check requests (Circuit Breaker): $NUM_HEALTH_REQUESTS" +echo " • Notification requests (Async + Bulkhead): $NUM_NOTIFY_REQUESTS" +echo "" +echo -e "${CYAN}Expected Behavior:${NC}" +echo -e "${CYAN} Phase 1 (Retry): Some failures will be retried and eventually succeed${NC}" +echo -e "${CYAN} Phase 2 (Circuit Breaker): Circuit will OPEN after 50% failures, then recover${NC}" +echo -e "${CYAN} Phase 3 (Bulkhead): Requests queued up to limit, excess rejected${NC}" echo "" -# Function to send a single request and capture timing -send_request() { +read -p "Press Enter to start the load test..." +echo "" + +# Counters +declare -A success_count=( ["authorize"]=0 ["health"]=0 ["notify"]=0 ) +declare -A failure_count=( ["authorize"]=0 ["health"]=0 ["notify"]=0 ) +declare -A fallback_count=( ["authorize"]=0 ["health"]=0 ["notify"]=0 ) + +# Function to send authorization request +send_authorize_request() { local id=$1 - local amount=$2 - local start_time=$(date +%s) + local amount=$((50 + RANDOM % 450)) - echo -e "${YELLOW}[Request $id] Sending payment request for \$$amount...${NC}" + response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" \ + -X POST "${BASE_URL}/authorize?amount=${amount}" 2>/dev/null || echo "HTTP_STATUS:000") - # Send request and capture response - response=$(curl -s -X POST "${PAYMENT_URL}?amount=${amount}") + http_code=$(echo "$response" | grep "HTTP_STATUS:" | cut -d: -f2) + body=$(echo "$response" | sed '/HTTP_STATUS:/d') - local end_time=$(date +%s) - local duration=$((end_time - start_time)) - - # Check if response contains "success" or "failed" - if echo "$response" | grep -q "success"; then - echo -e "${GREEN}[Request $id] SUCCESS: Payment processed in ${duration}s${NC}" - echo -e "${GREEN}[Request $id] Response: $response${NC}" - elif echo "$response" | grep -q "failed"; then - echo -e "${RED}[Request $id] FALLBACK: Service unavailable (took ${duration}s)${NC}" - echo -e "${RED}[Request $id] Response: $response${NC}" + if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then + if echo "$body" | grep -q "fallback\|pending"; then + echo -e "${YELLOW}[AUTH-$id] ⚡ Fallback${NC}" + echo "fallback" > /tmp/auth_result_$id.txt + else + echo -e "${GREEN}[AUTH-$id] ✓ Success${NC}" + echo "success" > /tmp/auth_result_$id.txt + fi else - echo -e "${RED}[Request $id] ERROR: Unexpected response (took ${duration}s)${NC}" - echo -e "${RED}[Request $id] Response: $response${NC}" + echo -e "${RED}[AUTH-$id] ✗ Failed${NC}" + echo "failed" > /tmp/auth_result_$id.txt fi - - # Return the duration for analysis - echo "$duration" } -# Run concurrent requests using GNU Parallel if available, or background processes if not -if command -v parallel > /dev/null; then - echo -e "${PURPLE}Using GNU Parallel for concurrent requests${NC}" - export -f send_request - export PAYMENT_URL RED GREEN YELLOW BLUE PURPLE CYAN NC +# Function to send health check request +send_health_request() { + local id=$1 - # Use predefined amounts instead of bc calculation - amounts=("15.99" "24.50" "19.95" "32.75" "12.99" "22.50" "18.75" "29.99" "14.50" "27.25") - for i in $(seq 1 $NUM_REQUESTS); do - amount_index=$((i-1 % 10)) - amount=${amounts[$amount_index]} - send_request $i $amount & - done -else - echo -e "${PURPLE}Running concurrent requests using background processes${NC}" - # Store PIDs - pids=() + response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" \ + -X GET "${BASE_URL}/health/gateway" 2>/dev/null || echo "HTTP_STATUS:000") - # Predefined amounts - amounts=("15.99" "24.50" "19.95" "32.75" "12.99" "22.50" "18.75" "29.99" "14.50" "27.25") + http_code=$(echo "$response" | grep "HTTP_STATUS:" | cut -d: -f2) + body=$(echo "$response" | sed '/HTTP_STATUS:/d') - # Launch requests in background - for i in $(seq 1 $NUM_REQUESTS); do - # Get amount from predefined list - amount_index=$((i-1 % 10)) - amount=${amounts[$amount_index]} - send_request $i $amount & - pids+=($!) - - # Control concurrency - if [ ${#pids[@]} -ge $CONCURRENCY ]; then - # Wait for one process to finish before starting more - wait "${pids[0]}" - pids=("${pids[@]:1}") + if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then + if echo "$body" | grep -q "healthy"; then + echo -e "${GREEN}[HEALTH-$id] ✓ Healthy${NC}" + echo "success" > /tmp/health_result_$id.txt + else + echo -e "${YELLOW}[HEALTH-$id] ⚠ Degraded${NC}" + echo "fallback" > /tmp/health_result_$id.txt fi - done + elif [ "$http_code" -eq 503 ]; then + if echo "$body" | grep -q "circuit_open"; then + echo -e "${PURPLE}[HEALTH-$id] ⚡ Circuit Breaker OPEN${NC}" + echo "failed" > /tmp/health_result_$id.txt + elif echo "$body" | grep -q "unhealthy"; then + echo -e "${YELLOW}[HEALTH-$id] ⚠ Gateway Unhealthy${NC}" + echo "fallback" > /tmp/health_result_$id.txt + else + echo -e "${RED}[HEALTH-$id] ✗ Service Unavailable${NC}" + echo "failed" > /tmp/health_result_$id.txt + fi + else + echo -e "${RED}[HEALTH-$id] ✗ Failed (HTTP $http_code)${NC}" + echo "failed" > /tmp/health_result_$id.txt + fi +} + +# Function to send notification request +send_notify_request() { + local id=$1 + local payment_id=$(printf "PAY-%05d" $id) + + response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" \ + -X POST "${BASE_URL}/notify/${payment_id}" 2>/dev/null || echo "HTTP_STATUS:000") + + http_code=$(echo "$response" | grep "HTTP_STATUS:" | cut -d: -f2) + body=$(echo "$response" | sed '/HTTP_STATUS:/d') - # Wait for remaining processes - for pid in "${pids[@]}"; do - wait $pid - done + if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then + if echo "$body" | grep -q "fallback"; then + echo -e "${YELLOW}[NOTIFY-$id] ⚡ Fallback${NC}" + echo "fallback" > /tmp/notify_result_$id.txt + else + echo -e "${GREEN}[NOTIFY-$id] ✓ Queued${NC}" + echo "success" > /tmp/notify_result_$id.txt + fi + else + if echo "$body" | grep -q "Bulkhead"; then + echo -e "${RED}[NOTIFY-$id] ✗ Rejected (Bulkhead)${NC}" + else + echo -e "${RED}[NOTIFY-$id] ✗ Failed${NC}" + fi + echo "failed" > /tmp/notify_result_$id.txt + fi +} + +# Phase 1: Concurrent authorization requests +echo -e "${BLUE}=== Phase 1: Authorization Load Test (Retry Pattern) ===${NC}" +echo -e "${CYAN}Sending $NUM_AUTHORIZE_REQUESTS concurrent authorization requests...${NC}" +echo "" + +start_time=$(date +%s.%N) + +for i in $(seq 1 $NUM_AUTHORIZE_REQUESTS); do + send_authorize_request $i & + sleep 0.05 +done + +wait + +end_time=$(date +%s.%N) +duration=$(echo "$end_time - $start_time" | bc) +duration_formatted=$(printf "%.2f" $duration) + +# Collect results from temp files +for i in $(seq 1 $NUM_AUTHORIZE_REQUESTS); do + if [ -f "/tmp/auth_result_$i.txt" ]; then + result=$(cat /tmp/auth_result_$i.txt) + case $result in + success) ((success_count["authorize"]++)) ;; + fallback) ((fallback_count["authorize"]++)) ;; + failed) ((failure_count["authorize"]++)) ;; + esac + rm /tmp/auth_result_$i.txt + fi +done + +echo "" +echo -e "${PURPLE}Authorization Phase Complete: ${duration_formatted}s${NC}" +echo -e "${GREEN}✓ Success: ${success_count["authorize"]}${NC}" +echo -e "${YELLOW}⚡ Fallback: ${fallback_count["authorize"]}${NC}" +echo -e "${RED}✗ Failed: ${failure_count["authorize"]}${NC}" +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +sleep 2 + +# Phase 2: Health check requests (circuit breaker) +echo -e "${BLUE}=== Phase 2: Health Check Load Test (Circuit Breaker) ===${NC}" +echo -e "${CYAN}Sending $NUM_HEALTH_REQUESTS health check requests (sequential to observe circuit breaker)...${NC}" +echo -e "${CYAN}Watch for circuit breaker to OPEN after failures, then potentially CLOSE after recovery${NC}" +echo "" + +start_time=$(date +%s.%N) + +# Send requests sequentially to better observe circuit breaker behavior +for i in $(seq 1 $NUM_HEALTH_REQUESTS); do + send_health_request $i + sleep 0.5 # Small delay to observe circuit breaker state changes +done + +end_time=$(date +%s.%N) +duration=$(echo "$end_time - $start_time" | bc) +duration_formatted=$(printf "%.2f" $duration) + +# Collect results from temp files +for i in $(seq 1 $NUM_HEALTH_REQUESTS); do + if [ -f "/tmp/health_result_$i.txt" ]; then + result=$(cat /tmp/health_result_$i.txt) + case $result in + success) ((success_count["health"]++)) ;; + fallback) ((fallback_count["health"]++)) ;; + failed) ((failure_count["health"]++)) ;; + esac + rm /tmp/health_result_$i.txt + fi +done + +echo "" +echo -e "${PURPLE}Health Check Phase Complete: ${duration_formatted}s${NC}" +echo -e "${GREEN}✓ Healthy: ${success_count["health"]}${NC}" +echo -e "${YELLOW}⚠ Degraded/Unhealthy: ${fallback_count["health"]}${NC}" +echo -e "${RED}✗ Failed/Circuit Open: ${failure_count["health"]}${NC}" +echo "" +echo -e "${CYAN}Circuit Breaker Behavior:${NC}" +echo -e "${CYAN}• Opens at 50% failure rate (after 4 requests)${NC}" +echo -e "${CYAN}• Simulated gateway has 60% failure rate${NC}" +echo -e "${CYAN}• Circuit stays open for 5 seconds, then tries to recover${NC}" +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +sleep 2 + +# Phase 3: Notification requests (bulkhead + async) +echo -e "${BLUE}=== Phase 3: Notification Load Test (Bulkhead + Async) ===${NC}" +echo -e "${CYAN}Sending $NUM_NOTIFY_REQUESTS concurrent notification requests...${NC}" +echo "" + +start_time=$(date +%s.%N) + +for i in $(seq 1 $NUM_NOTIFY_REQUESTS); do + send_notify_request $i & + sleep 0.05 +done + +wait + +end_time=$(date +%s.%N) +duration=$(echo "$end_time - $start_time" | bc) +duration_formatted=$(printf "%.2f" $duration) + +# Collect results from temp files +for i in $(seq 1 $NUM_NOTIFY_REQUESTS); do + if [ -f "/tmp/notify_result_$i.txt" ]; then + result=$(cat /tmp/notify_result_$i.txt) + case $result in + success) ((success_count["notify"]++)) ;; + fallback) ((fallback_count["notify"]++)) ;; + failed) ((failure_count["notify"]++)) ;; + esac + rm /tmp/notify_result_$i.txt + fi +done + +echo "" +echo -e "${PURPLE}Notification Phase Complete: ${duration_formatted}s${NC}" +echo -e "${GREEN}✓ Queued: ${success_count["notify"]}${NC}" +echo -e "${YELLOW}⚡ Fallback: ${fallback_count["notify"]}${NC}" +echo -e "${RED}✗ Rejected: ${failure_count["notify"]}${NC}" +echo "" +echo -e "${BLUE}----------------------------------------${NC}" +echo "" + +# Summary +total_requests=$((NUM_AUTHORIZE_REQUESTS + NUM_HEALTH_REQUESTS + NUM_NOTIFY_REQUESTS)) +total_success=$((success_count["authorize"] + success_count["health"] + success_count["notify"])) +total_fallback=$((fallback_count["authorize"] + fallback_count["health"] + fallback_count["notify"])) +total_failure=$((failure_count["authorize"] + failure_count["health"] + failure_count["notify"])) + +echo -e "${BLUE}╔════════════════════════════════════════════════╗${NC}" +echo -e "${BLUE}║ Load Test Summary ║${NC}" +echo -e "${BLUE}╚════════════════════════════════════════════════╝${NC}" +echo "" +echo -e "${CYAN}Total Requests: ${total_requests}${NC}" +echo -e "${GREEN}✓ Successful: ${total_success}${NC}" +echo -e "${YELLOW}⚡ Fallback: ${total_fallback}${NC}" +echo -e "${RED}✗ Failed: ${total_failure}${NC}" +echo "" + +success_rate=$(echo "scale=2; ($total_success * 100) / $total_requests" | bc) +echo -e "${CYAN}Success Rate: ${success_rate}%${NC}" +echo "" + +# Fetch and display metrics +echo -e "${BLUE}=== Fault Tolerance Metrics Summary ===${NC}" +echo "" + +echo -e "${YELLOW}Retry Metrics (authorizePayment):${NC}" +retry_metrics=$(curl -s "$METRICS_URL" 2>/dev/null | grep -i "retry.*authorizePayment\|authorizePayment.*retry") +if [ -n "$retry_metrics" ]; then + echo "$retry_metrics" | head -10 +else + echo -e "${CYAN}No retry metrics found. Checking alternative patterns...${NC}" + curl -s "$METRICS_URL" 2>/dev/null | grep -iE "ft.*retry|retry.*total|retry.*calls" | head -5 fi echo "" -echo -e "${BLUE}=== Test Complete ===${NC}" -echo -e "${CYAN}Analyze the responses to verify:${NC}" -echo -e "${CYAN}1. Asynchronous processing (@Asynchronous)${NC}" -echo -e "${CYAN}2. Resource isolation (@Bulkhead)${NC}" -echo -e "${CYAN}3. Retry behavior on failures (@Retry)${NC}" +echo -e "${YELLOW}Circuit Breaker Metrics (checkGatewayHealth):${NC}" +cb_metrics=$(curl -s "$METRICS_URL" 2>/dev/null | grep -i "circuitbreaker.*checkGatewayHealth\|checkGatewayHealth.*circuit") +if [ -n "$cb_metrics" ]; then + echo "$cb_metrics" | head -10 +else + echo -e "${CYAN}No circuit breaker metrics found. Checking alternative patterns...${NC}" + curl -s "$METRICS_URL" 2>/dev/null | grep -iE "ft.*circuit|circuit.*state|circuit.*calls" | head -5 +fi + +echo "" +echo -e "${YELLOW}Bulkhead Metrics (sendPaymentNotification):${NC}" +bulkhead_metrics=$(curl -s "$METRICS_URL" 2>/dev/null | grep -i "bulkhead.*sendPaymentNotification\|sendPaymentNotification.*bulkhead") +if [ -n "$bulkhead_metrics" ]; then + echo "$bulkhead_metrics" | head -10 +else + echo -e "${CYAN}No bulkhead metrics found. Checking alternative patterns...${NC}" + curl -s "$METRICS_URL" 2>/dev/null | grep -iE "ft.*bulkhead|bulkhead.*accept|bulkhead.*reject" | head -5 +fi + +echo "" +echo -e "${GREEN}=== Load Test Complete ===${NC}" +echo "" +echo -e "${CYAN}To view all fault tolerance metrics:${NC}" +echo -e "${BLUE}curl $METRICS_URL | grep -i ft${NC}" +echo "" diff --git a/code/chapter08/payment/test-payment-retry.sh b/code/chapter08/payment/test-payment-retry.sh index dcbb62c3..85707bb0 100755 --- a/code/chapter08/payment/test-payment-retry.sh +++ b/code/chapter08/payment/test-payment-retry.sh @@ -1,180 +1,126 @@ #!/bin/bash # Test script for Payment Service Retry functionality -# Tests the @Retry annotation on the processPayment method +# Tests the @Retry annotation on the authorizePayment method -set -e - -echo "=== Payment Service Retry Test ===" -echo "" +# Color definitions +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color # Dynamically determine the base URL if [ -n "$CODESPACE_NAME" ] && [ -n "$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN" ]; then - # GitHub Codespaces environment - use localhost for internal testing BASE_URL="http://localhost:9080/payment/api" - echo "Detected GitHub Codespaces environment (using localhost)" - echo "Note: External access would be https://$CODESPACE_NAME-9080.$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN/payment/api" + echo -e "${CYAN}Detected GitHub Codespaces environment${NC}" elif [ -n "$GITPOD_WORKSPACE_URL" ]; then - # Gitpod environment GITPOD_HOST=$(echo $GITPOD_WORKSPACE_URL | sed 's|https://||' | sed 's|/||') BASE_URL="https://9080-$GITPOD_HOST/payment/api" - echo "Detected Gitpod environment" + echo -e "${CYAN}Detected Gitpod environment${NC}" else - # Local or other environment BASE_URL="http://localhost:9080/payment/api" - echo "Using local environment" + echo -e "${CYAN}Using local environment${NC}" fi -echo "Base URL: $BASE_URL" +echo "" +echo -e "${BLUE}=== Payment Service Retry Test ===${NC}" +echo -e "${CYAN}Base URL: $BASE_URL${NC}" echo "" -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color +echo -e "${YELLOW}Retry Configuration:${NC}" +echo " • Max Retries: 3" +echo " • Delay: 2000ms" +echo " • Jitter: 500ms" +echo " • Retry on: PaymentProcessingException" +echo " • Abort on: CriticalPaymentException" +echo " • Simulated failure rate: 60%" +echo " • Processing delay: 1500ms per attempt" +echo "" + +echo -e "${CYAN}🔍 How to identify retry behavior:${NC}" +echo " • ⚡ Fast (~1.5s) = Success on 1st attempt" +echo " • 🔄 Medium (~4s) = Succeeded after 1 retry" +echo " • 🔄🔄 Slow (~6.5s) = Succeeded after 2 retries" +echo " • 🔄🔄🔄 Very slow (~9-12s) = Needed all 3 retries or fallback" +echo "" # Function to make HTTP requests and display results make_request() { - local method=$1 - local url=$2 - local description=$3 + local amount=$1 + local description=$2 echo -e "${BLUE}Testing: $description${NC}" - echo "Request: $method $url" - echo "" + echo -e "${CYAN}Amount: \$$amount${NC}" - # Capture start time - start_time=$(date +%s%3N) + start_time=$(date +%s.%N) + response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" -X POST "${BASE_URL}/authorize?amount=${amount}" 2>/dev/null || echo "HTTP_STATUS:000") + end_time=$(date +%s.%N) - response=$(curl -s -w "\nHTTP_STATUS:%{http_code}\nTIME_TOTAL:%{time_total}" -X $method "$url" 2>/dev/null || echo "HTTP_STATUS:000") - - # Capture end time - end_time=$(date +%s%3N) - total_time=$((end_time - start_time)) + duration=$(echo "$end_time - $start_time" | bc) + duration_formatted=$(printf "%.2f" $duration) http_code=$(echo "$response" | grep "HTTP_STATUS:" | cut -d: -f2) - curl_time=$(echo "$response" | grep "TIME_TOTAL:" | cut -d: -f2) - body=$(echo "$response" | sed '/HTTP_STATUS:/d' | sed '/TIME_TOTAL:/d') + body=$(echo "$response" | sed '/HTTP_STATUS:/d') + + # Determine retry behavior based on duration + duration_int=$(echo "$duration * 10" | bc | cut -d. -f1) if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then - # Analyze timing to determine retry behavior - # Convert curl_time to integer for comparison (multiply by 10 to handle decimals) - curl_time_int=$(echo "$curl_time" | awk '{printf "%.0f", $1 * 10}') - - if [ "$curl_time_int" -lt 20 ]; then # < 2.0 seconds + if [ "$duration_int" -lt 20 ]; then echo -e "${GREEN}✓ Success (HTTP $http_code) - First attempt! ⚡${NC}" - elif [ "$curl_time_int" -lt 55 ]; then # < 5.5 seconds + elif [ "$duration_int" -lt 55 ]; then echo -e "${GREEN}✓ Success (HTTP $http_code) - After 1 retry 🔄${NC}" - elif [ "$curl_time_int" -lt 80 ]; then # < 8.0 seconds + elif [ "$duration_int" -lt 80 ]; then echo -e "${GREEN}✓ Success (HTTP $http_code) - After 2 retries 🔄🔄${NC}" else - echo -e "${GREEN}✓ Success (HTTP $http_code) - After 3 retries 🔄🔄🔄${NC}" + echo -e "${GREEN}✓ Success (HTTP $http_code) - After 3 retries or fallback 🔄🔄🔄${NC}" fi - elif [ "$http_code" -ge 400 ] && [ "$http_code" -lt 500 ]; then - echo -e "${YELLOW}⚠ Client Error (HTTP $http_code)${NC}" + echo -e "${GREEN}Response: $body${NC}" + elif [ "$http_code" -eq 400 ]; then + echo -e "${YELLOW}⚠ Client Error (HTTP $http_code) - No retries (abort condition)${NC}" + echo -e "${YELLOW}Response: $body${NC}" else echo -e "${RED}✗ Server Error (HTTP $http_code)${NC}" + echo -e "${RED}Response: $body${NC}" fi - echo "Response: $body" - echo "Total time: ${total_time}ms (curl: ${curl_time}s)" + echo -e "${CYAN}Duration: ${duration_formatted}s${NC}" echo "" echo "----------------------------------------" echo "" } -echo "Starting Payment Service Retry Tests..." -echo "" -echo "Your PaymentService has these retry settings:" -echo "• Max Retries: 3" -echo "• Delay: 2000ms" -echo "• Jitter: 500ms" -echo "• Retry on: PaymentProcessingException" -echo "• Abort on: CriticalPaymentException" -echo "• Simulated failure rate: 30% (Math.random() > 0.7)" -echo "• Processing delay: 1500ms per attempt" -echo "" -echo "🔍 HOW TO IDENTIFY RETRY BEHAVIOR:" -echo "• ⚡ Fast response (~1.5s) = Succeeded on 1st attempt" -echo "• 🔄 Medium response (~4s) = Needed 1 retry" -echo "• 🔄🔄 Slow response (~6.5s) = Needed 2 retries" -echo "• 🔄🔄🔄 Very slow response (~9-12s) = Needed 3 retries" -echo "" - -echo "Make sure the Payment Service is running on port 9080" -echo "You can start it with: cd payment && mvn liberty:run" -echo "" -read -p "Press Enter to continue..." -echo "" - -# Test 1: Valid payment (should succeed, may need retries due to random failures) -echo -e "${BLUE}=== Test 1: Valid Payment Authorization ===${NC}" -echo "This test uses a valid amount and may succeed immediately or after retries" -echo "Expected: Success after 1-4 attempts (due to 30% failure simulation)" -echo "" -make_request "POST" "$BASE_URL/authorize?amount=100.50" \ - "Valid payment amount (100.50) - may trigger retries due to random failures" +# Run multiple tests +echo -e "${BLUE}=== Test 1: Valid Payment (Will Likely Trigger Retries) ===${NC}" +make_request "100.50" "Valid payment - 60% failure rate per attempt" -# Test 2: Another valid payment to see retry behavior echo -e "${BLUE}=== Test 2: Another Valid Payment ===${NC}" -echo "Running another test to demonstrate retry variability" -echo "" -make_request "POST" "$BASE_URL/authorize?amount=250.00" \ - "Valid payment amount (250.00) - testing retry behavior" +make_request "250.00" "Valid payment - observe retry behavior" -# Test 3: Invalid payment amount (should abort immediately) -echo -e "${BLUE}=== Test 3: Invalid Payment Amount (Abort Condition) ===${NC}" -echo "This test uses an invalid amount which should trigger CriticalPaymentException" -echo "Expected: Immediate failure with no retries" -echo "" -make_request "POST" "$BASE_URL/authorize?amount=0" \ - "Invalid payment amount (0) - should abort immediately with CriticalPaymentException" +echo -e "${BLUE}=== Test 3: Large Payment ===${NC}" +make_request "999.99" "Large payment - testing retry behavior" -# Test 4: Negative amount (should abort immediately) -echo -e "${BLUE}=== Test 4: Negative Payment Amount ===${NC}" -echo "Expected: Immediate failure with no retries" -echo "" -make_request "POST" "$BASE_URL/authorize?amount=-50" \ - "Negative payment amount (-50) - should abort immediately" +echo -e "${BLUE}=== Test 4: Invalid Amount (Abort Condition) ===${NC}" +make_request "0" "Invalid amount - should abort immediately (no retries)" -# Test 5: No amount parameter (should abort immediately) -echo -e "${BLUE}=== Test 5: Missing Payment Amount ===${NC}" -echo "Expected: Immediate failure with no retries" -echo "" -make_request "POST" "$BASE_URL/authorize" \ - "Missing payment amount - should abort immediately" +echo -e "${BLUE}=== Test 5: Negative Amount (Abort Condition) ===${NC}" +make_request "-50" "Negative amount - should abort immediately (no retries)" -# Test 6: Multiple requests to observe retry patterns -echo -e "${BLUE}=== Test 6: Multiple Requests to Observe Retry Patterns ===${NC}" -echo "Running 5 valid payment requests to observe retry behavior patterns" +echo -e "${BLUE}=== Test 6: Multiple Requests to Observe Patterns ===${NC}" +echo -e "${CYAN}Sending 5 requests to observe retry patterns...${NC}" echo "" for i in {1..5}; do - echo "Request $i/5:" - amount=$((100 + i * 25)) - make_request "POST" "$BASE_URL/authorize?amount=$amount" \ - "Payment request $i with amount $amount" - - # Small delay between requests - sleep 2 + amount=$((50 + RANDOM % 150)) + echo -e "${YELLOW}Request $i:${NC}" + make_request "$amount" "Random amount test #$i" + sleep 1 done -echo -e "${GREEN}=== Retry Testing Complete ===${NC}" -echo "" -echo "Key observations to look for:" -echo -e "• ${GREEN}Successful requests:${NC} Should complete in ~1.5-12 seconds depending on retries" -echo -e "• ${YELLOW}Retry behavior:${NC} Failed attempts will retry up to 3 times with 2-2.5 second delays" -echo -e "• ${RED}Abort conditions:${NC} Invalid amounts should fail immediately (~1.5 seconds)" -echo -e "• ${BLUE}Random failures:${NC} ~30% of valid requests may need retries" +echo -e "${GREEN}=== Retry Test Complete ===${NC}" echo "" -echo "To see detailed retry logs, monitor the server logs:" -echo "tail -f target/liberty/wlp/usr/servers/mpServer/logs/messages.log" +echo -e "${CYAN}Check fault tolerance metrics:${NC}" +echo -e "${BLUE}curl http://localhost:9080/metrics?scope=base | grep retry${NC}" echo "" -echo "Expected timing patterns:" -echo "• Success on 1st attempt: ~1.5 seconds" -echo "• Success on 2nd attempt: ~4-4.5 seconds" -echo "• Success on 3rd attempt: ~6.5-7.5 seconds" -echo "• Success on 4th attempt: ~9-10.5 seconds" -echo "• Abort conditions: ~1.5 seconds (no retries)"