From f7313bdfce2b7e6506ebecca59b80a75c1f59677 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Wed, 29 Jul 2026 19:25:13 +0200 Subject: [PATCH] WARC writer: WARC-Protocol header to follow WARC field proposals (fixes #1998) OkHttp protocol: add protocol response header key `_cipher_suites_` to hold the SSL/TLS Cipher suite separate from `_protocol_versions_`. WARC writer: 1. add WARC header `WARC-Cipher-Suite` 2. split multiple values in `WARC-Protocol` header and repeat header --- .../stormcrawler/protocol/ProtocolResponse.java | 6 ++++++ .../stormcrawler/protocol/okhttp/HttpProtocol.java | 4 +++- .../apache/stormcrawler/warc/WARCRecordFormat.java | 9 ++++++++- .../apache/stormcrawler/warc/WARCHdfsBoltTest.java | 11 ++++++++++- .../stormcrawler/warc/WARCRecordFormatTest.java | 14 ++++++++++---- 5 files changed, 37 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java index c465ff000..31e78cd9c 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java @@ -45,6 +45,12 @@ public class ProtocolResponse { */ public static final String PROTOCOL_VERSIONS_KEY = "_protocol_versions_"; + /** + * Key which holds the SSL/TLS cipher suites. For requests sent over http:// the value may be + * null. + */ + public static final String CIPHER_SUITES_KEY = "_cipher_suites_"; + /** * Metadata key which holds a boolean value in metadata whether the response content is trimmed * or not. diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index ba60d7f99..4ecb18e16 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -669,10 +669,11 @@ public Response intercept(Interceptor.Chain chain) throws IOException { .getBytes(StandardCharsets.ISO_8859_1)); final StringBuilder protocols = new StringBuilder(response.protocol().toString()); + String cipherSuite = null; final Handshake handshake = connection.handshake(); if (handshake != null) { protocols.append(',').append(handshake.tlsVersion()); - protocols.append(',').append(handshake.cipherSuite()); + cipherSuite = handshake.cipherSuite().toString(); } // returns a modified version of the response @@ -686,6 +687,7 @@ public Response intercept(Interceptor.Chain chain) throws IOException { .header(ProtocolResponse.RESPONSE_IP_KEY, ipAddress) .header(ProtocolResponse.REQUEST_TIME_KEY, Long.toString(startFetchTime)) .header(ProtocolResponse.PROTOCOL_VERSIONS_KEY, protocols.toString()) + .header(ProtocolResponse.CIPHER_SUITES_KEY, cipherSuite) .build(); } } diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java index cee46c8fb..3a4b7d84c 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java @@ -468,7 +468,14 @@ public byte[] format(Tuple tuple) { metadata.getFirstValue( ProtocolResponse.PROTOCOL_VERSIONS_KEY, this.protocolMDprefix); if (protocolVersions != null) { - buffer.append("WARC-Protocol: ").append(protocolVersions).append(CRLF); + for (String val : StringUtils.split(protocolVersions, ',')) { + buffer.append("WARC-Protocol: ").append(val).append(CRLF); + } + } + final String cipherSuites = + metadata.getFirstValue(ProtocolResponse.CIPHER_SUITES_KEY, this.protocolMDprefix); + if (cipherSuites != null) { + buffer.append("WARC-Cipher-Suite: ").append(cipherSuites).append(CRLF); } buffer.append("WARC-Payload-Digest").append(": ").append(payloadDigest).append(CRLF); diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java index 662355122..dcc21c43e 100644 --- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java @@ -130,6 +130,13 @@ void testHttp2() throws IOException { assertTrue( response.headers().first("WARC-Protocol").isPresent(), "WARC response record is expected to include WARC header \"WARC-Protocol\""); + assertEquals( + 2, + response.headers().all("WARC-Protocol").size(), + "WARC response record is expected to include WARC header \"WARC-Protocol\""); + assertTrue( + response.headers().first("WARC-Cipher-Suite").isPresent(), + "WARC response record is expected to include WARC header \"WARC-Cipher-Suite\""); assertTrue( response.headers().first("WARC-IP-Address").isPresent(), "WARC response record is expected to include WARC header \"WARC-IP-Address\""); @@ -211,7 +218,9 @@ private Tuple getPage(String httpVersionString) { + "Connection: close\r\n\r\n"); metadata.addValue( protocolMDprefix + ProtocolResponse.PROTOCOL_VERSIONS_KEY, - httpVersionString + ",TLS_1_3,TLS_AES_256_GCM_SHA384"); + httpVersionString + ",TLS_1_3"); + metadata.addValue( + protocolMDprefix + ProtocolResponse.CIPHER_SUITES_KEY, "TLS_AES_256_GCM_SHA384"); metadata.addValue(protocolMDprefix + ProtocolResponse.RESPONSE_IP_KEY, "123.123.123.123"); Tuple tuple = mock(Tuple.class); when(tuple.getBinaryByField("content")).thenReturn(content); diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java index 045579a89..c794d2a98 100644 --- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java @@ -225,9 +225,9 @@ void testReplaceHttpVersion() { + "Content-Encoding: gzip\r\n" + "Content-Length: 26\r\n" + "Connection: close"); + metadata.addValue(protocolMDprefix + ProtocolResponse.PROTOCOL_VERSIONS_KEY, "h2,TLS_1_3"); metadata.addValue( - protocolMDprefix + ProtocolResponse.PROTOCOL_VERSIONS_KEY, - "h2,TLS_1_3,TLS_AES_256_GCM_SHA384"); + protocolMDprefix + ProtocolResponse.CIPHER_SUITES_KEY, "TLS_AES_256_GCM_SHA384"); metadata.addValue(protocolMDprefix + ProtocolResponse.RESPONSE_IP_KEY, "123.123.123.123"); Tuple tuple = mock(Tuple.class); when(tuple.getBinaryByField("content")).thenReturn(content); @@ -246,8 +246,14 @@ void testReplaceHttpVersion() { statusLine.matches("^HTTP/1\\.[01] .*"), "WARC response record: HTTP status line must start with HTTP/1.1 or HTTP/1.0"); assertTrue( - headersPayload[0].contains("\r\nWARC-Protocol: "), - "WARC response record is expected to include WARC header \"WARC-Protocol\""); + headersPayload[0].contains("\r\nWARC-Protocol: h2\r\n"), + "WARC response record is expected to include a WARC header \"WARC-Protocol: h2\""); + assertTrue( + headersPayload[0].contains("\r\nWARC-Protocol: TLS_1_3\r\n"), + "WARC response record is expected to include a WARC header \"WARC-Protocol: TLS_1_3\""); + assertTrue( + headersPayload[0].contains("\r\nWARC-Cipher-Suite: "), + "WARC response record is expected to include WARC header \"WARC-Cipher-Suite\""); assertTrue( headersPayload[0].contains("\r\nWARC-IP-Address: "), "WARC response record is expected to include WARC header \"WARC-IP-Address\"");