Add ChunkDecodingStream and ChunkDecodingClient

Closes bblanchon/ArduinoJson#2105

Author: bblanchon

.github/workflows/ci.yml

@@ -54,52 +54,61 @@ jobs:
           - core: arduino:avr
             board: arduino:avr:leonardo
             eeprom: true
+            httpclient: false
             softwareserial: true
 
           - core: esp8266:esp8266
             board: esp8266:esp8266:huzzah
             eeprom: true
+            httpclient: true
             softwareserial: true
             index_url: https://arduino.esp8266.com/stable/package_esp8266com_index.json
 
           - core: esp32:esp32
             board: esp32:esp32:esp32
             eeprom: true
+            httpclient: true
             softwareserial: false
             index_url: https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_index.json
 
           - core: arduino:samd
             board: arduino:samd:mkr1000
             eeprom: false
+            httpclient: false
             softwareserial: false
 
           - core: STMicroelectronics:stm32
             board: STMicroelectronics:stm32:Nucleo_32
             eeprom: true
+            httpclient: false
             softwareserial: true
             index_url: https://github.com/stm32duino/BoardManagerFiles/raw/main/package_stmicroelectronics_index.json
 
           - core: stm32duino:STM32F1
             board: stm32duino:STM32F1:mapleMini
             eeprom: false
+            httpclient: false
             softwareserial: false
             index_url: http://dan.drown.org/stm32duino/package_STM32duino_index.json
 
           - core: stm32duino:STM32F4
             board: stm32duino:STM32F4:blackpill_f401
             eeprom: false
+            httpclient: false
             softwareserial: false
             index_url: http://dan.drown.org/stm32duino/package_STM32duino_index.json
 
           - core: rp2040:rp2040
             board: rp2040:rp2040:rpipico
             eeprom: true
+            httpclient: false
             softwareserial: true
             index_url: https://github.com/earlephilhower/arduino-pico/releases/download/global/package_rp2040_index.json
 
           - core: arduino:mbed_rp2040
             board: arduino:mbed_rp2040:pico
             eeprom: false
+            httpclient: false
             softwareserial: false
 
     steps:
@@ -115,6 +124,10 @@ jobs:
       - name: Install core
         run: arduino-cli core install --additional-urls "${{ matrix.index_url }}" ${{ matrix.core }}
 
+      - name: Build ChunkDecoding
+        if: matrix.httpclient
+        run: arduino-cli compile --library . --warnings all -b ${{ matrix.board }} "examples/ChunkDecoding/ChunkDecoding.ino"
+
       - name: Build EepromRead
         if: matrix.eeprom
         run: arduino-cli compile --library . --warnings all -b ${{ matrix.board }} "examples/EepromRead/EepromRead.ino"

README.md

@@ -27,6 +27,7 @@ For example, with this library, you can:
 * debug your program more easily by logging what it sends to a Web service
 * send large data with the [Wire library](https://www.arduino.cc/en/reference/wire)
 * use a `String`, EEPROM, or `PROGMEM` with a stream interface
+* decode HTTP chunks
 
 Read on to see how StreamUtils can help you!
 
@@ -37,11 +38,11 @@ How to add buffering to a Stream?
 ### Buffering read operations
 
 Sometimes, you can significantly improve performance by reading many bytes at once. 
-For example, [according to SPIFFS's wiki](https://github.com/pellepl/spiffs/wiki/Performance-and-Optimizing#reading-files), reading read files in chunks of 64 bytes is much faster than reading them one byte at a time.
+For example, [according to SPIFFS's wiki](https://github.com/pellepl/spiffs/wiki/Performance-and-Optimizing#reading-files), reading files in chunks of 64 bytes is much faster than reading them one byte at a time.
 
 ![ReadBufferingStream](https://github.com/bblanchon/ArduinoStreamUtils/raw/master/extras/images/ReadBuffer.svg)
 
-To buffer the input, decorate the original `Stream` with `ReadBufferingStream`. For example, suppose your program reads a JSON document from SPIFFS like that:
+To buffer the input, decorate the original `Stream` with `ReadBufferingStream`. For example, suppose your program reads a JSON document from SPIFFS like this:
 
 ```c++
 File file = SPIFFS.open("example.json", "r");
@@ -61,7 +62,7 @@ Unfortunately, this optimization is only possible if:
 1. `Stream.readBytes()` is declared `virtual` in your Arduino Code (as it's the case for ESP8266), and
 2. the derived class has an optimized implementation of `readBytes()` (as it's the case for SPIFFS' `File`).
 
-When possible, prefer `ReadBufferingClient` to `ReadBufferingStream` because `Client` defines a `read()` method similar to `readBytes()`, except that this one is `virtual` on all platforms.
+When possible, prefer `ReadBufferingClient` to `ReadBufferingStream` because `Client` defines a `read()` method similar to `readBytes()`, except this one is `virtual` on all platforms.
 
 If memory allocation fails, `ReadBufferingStream` behaves as if no buffer was used: it forwards all calls to the upstream `Stream`.
 
@@ -74,7 +75,7 @@ For example, writing to `WiFiClient` one byte at a time is very slow; it's much
 
 ![WriteBufferingStream](https://github.com/bblanchon/ArduinoStreamUtils/raw/master/extras/images/WriteBuffer.svg)
 
-To add a buffer, decorate the original `Stream` with  `WriteBufferingStream`. For example, if your program sends a JSON document via `WiFiClient`, like that:
+To add a buffer, decorate the original `Stream` with  `WriteBufferingStream`. For example, if your program sends a JSON document via `WiFiClient` like this:
 
 ```c++
 serializeJson(doc, wifiClient);
@@ -166,7 +167,7 @@ char response[256];
 client.readBytes(response, 256);
 ```
 
-Then decorate `client` and replace the calls:
+Then, decorate `client` and replace the calls:
 
 ```c++
 LoggingStream loggingClient(client, Serial);
@@ -187,7 +188,7 @@ These extra bits increase the amount of traffic but allow correcting any one-bit
 If you use this encoding on an 8-bit channel, it effectively doubles the amount of traffic. However, if you use an [`HardwareSerial`](https://www.arduino.cc/reference/en/language/functions/communication/serial/) instance (like `Serial`, `Serial1`...), you can slightly reduce the overhead by configuring the ports as a 7-bit channel, like so:
 
 ```c++
-// Initialize serial port with 9600 bauds, 7-bits of data, no parity, and one stop bit
+// Initialize serial port with 9600 bauds, 7 bits of data, no parity, and one stop bit
 Serial1.begin(9600, SERIAL_7N1);
 ```
 
@@ -346,6 +347,49 @@ Serial.println(stream.readString());
 
 `ProgmemStream`'s constructor also supports `const __FlashStringHelper*` (the type returned by the `F()` macro) and an optional second argument to specify the size of the buffer.
 
+How to decode HTTP chunks?
+--------------------------
+
+HTTP servers can send their response in multiple parts using [Chunked Transfer Encoding](https://en.wikipedia.org/wiki/Chunked_transfer_encoding). Clients using HTTP 1.1 must support this encoding as it's not optional and is dictated by the server.
+
+`ChunkDecodingStream` and `ChunkDecodingClient` are decorators that decode the chunks and make the response available as a regular stream.
+
+![ChunkDecodingStream](https://github.com/bblanchon/ArduinoStreamUtils/raw/master/extras/images/ChunkDecodingStream.svg)
+
+Here is an example using `HTTPClient`:
+
+```c++
+// Initialize HTTPClient
+HTTPClient http;
+http.begin(client, url);
+
+// Tell HTTPClient to collect the Transfer-Encoding header
+// (by default HTTPClient discards the response headers)
+const char *keys[] = {"Transfer-Encoding"};
+http.collectHeaders(keys, 1);
+
+// Send the request
+int status = http.GET();
+if (status != 200) return;
+
+// Create the raw and decoded stream
+Stream& rawStream = http.getStream();
+ChunkDecodingStream decodedStream(http.getStream());
+
+// Choose the stream based on the Transfer-Encoding header
+Stream& response = http.header("Transfer-Encoding") == "chunked" ? decodedStream : rawStream;
+
+// Read the response
+JsonDocument doc;
+deserializeJson(doc, response);
+
+// Close the connection
+http.end();
+```
+
+Note that `HTTPClient` already performs chunk decoding **if** you use `getString()`, but you might want to use `getStream()` to avoid buffering the entire response in memory.
+
+Also, you can avoid chunked transfer encoding by downgrading the HTTP version to 1.0. `HTTPClient` allows you to do that by calling `useHTTP10(true)` before sending the request.
 
 Summary
 -------
@@ -367,6 +411,7 @@ See the equivalence table below.
 | Error correction (decode only)     | `HammingDecodingClient` | `HammingDecodingStream` |                  |
 | Error correction (encode only)     | `HammingEncodingClient` | `HammingEncodingStream` | `HammingPrint`   |
 | Error correction (encode & decode) | `HammingClient`         | `HammingStream`         |                  |
+| Decode HTTP chunks                 | `ChunkDecodingClient`   | `ChunkDecodingStream`   |                  |
 
 Prefer `XxxClient` to `XxxStream` because, unlike `Stream::readBytes()`, `Client::read()` is virtual on all cores and therefore allows optimized implementations.
 

examples/ChunkDecoding/ChunkDecoding.ino

@@ -0,0 +1,73 @@
+#if defined(ARDUINO_ARCH_ESP8266)
+#include <ESP8266HTTPClient.h>
+#include <ESP8266WiFi.h>
+#elif defined(ARDUINO_ARCH_ESP32)
+#include <HTTPClient.h>
+#include <WiFi.h>
+#include <WiFiClientSecure.h>
+#else
+#error Unsuported platform
+#endif
+
+#include <StreamUtils.h>
+
+// WiFi network configuration.
+const char* WIFI_SSID = "*****EDIT*****";
+const char* WIFI_PASSWORD = "*****EDIT*****";
+
+void setup() {
+  // Initialize Serial Port
+  Serial.begin(115200);
+  while (!Serial)
+    continue;
+
+  // Connect to the WLAN
+  WiFi.mode(WIFI_STA);
+  WiFi.begin(WIFI_SSID, WIFI_PASSWORD);
+  while (WiFi.status() != WL_CONNECTED) {
+    Serial.println(F("Connecting to Wifi..."));
+    delay(500);
+  }
+
+  // Initialize the SSL library
+  WiFiClientSecure client;
+  client.setInsecure();  // ignore server's certificate
+
+  // Send the request
+  HTTPClient http;
+  http.begin(client, F("https://jigsaw.w3.org/HTTP/ChunkedScript"));
+
+  // Ask HTTPClient to collect the Transfer-Encoding header
+  // (by default it discards all headers)
+  const char* keys[] = {"Transfer-Encoding"};
+  http.collectHeaders(keys, 1);
+
+  Serial.println(F("Sending request..."));
+  int status = http.GET();
+  if (status != 200) {
+    Serial.print(F("Unexpected HTTP status: "));
+    Serial.println(status);
+    return;
+  }
+
+  // Get a reference to the stream
+  Stream& rawStream = http.getStream();
+  ChunkDecodingStream decodedStream(http.getStream());
+  Stream& response =
+      http.header("Transfer-Encoding") == "chunked" ? decodedStream : rawStream;
+
+  // Read and print the response
+  char buffer[256];
+  size_t n = 0;
+  do {
+    n = response.readBytes(buffer, sizeof(buffer));
+    Serial.write(buffer, n);
+  } while (n == sizeof(buffer));
+
+  // Disconnect
+  http.end();
+
+  Serial.println(F("Done!"));
+}
+
+void loop() {}

extras/images/ChunkDecodingStream.svg

@@ -0,0 +1,83 @@
+// StreamUtils - github.com/bblanchon/ArduinoStreamUtils
+// Copyright Benoit Blanchon 2019-2024
+// MIT License
+
+#include "StreamUtils/Clients/ChunkDecodingClient.hpp"
+#include "StreamUtils/Clients/MemoryClient.hpp"
+#include "StreamUtils/Clients/SpyingClient.hpp"
+#include "StreamUtils/Prints/StringPrint.hpp"
+
+#include "doctest.h"
+
+using namespace StreamUtils;
+
+TEST_CASE("ChunkDecodingClient") {
+  MemoryClient upstream(128);
+  StringPrint log;
+  SpyingClient spy{upstream, log};
+  ChunkDecodingClient client{spy};
+
+  // Most of the tests are in ChunkDecodingStreamTest.cpp
+  // This file only tests the client-specific methods
+
+  SUBCASE("read() merge chunks") {
+    uint8_t buffer[32];
+
+    upstream.print(
+        "4\r\n"
+        "XXXX\r\n"
+        "4\r\n"
+        "YYYY\r\n"
+        "0\r\n"
+        "\r\n");
+    REQUIRE(client.read(buffer, 32) == 8);
+    REQUIRE(buffer[0] == 'X');
+    REQUIRE(buffer[4] == 'Y');
+  }
+
+  SUBCASE("read() waits timeout") {
+    uint8_t buffer[32];
+
+    upstream.print(
+        "4\r\n"
+        "XXXX\r\n");
+    REQUIRE(client.read(buffer, 32) == 4);
+    REQUIRE(buffer[0] == 'X');
+
+    REQUIRE(log.str() ==
+            "read(1) -> 1"  // 1
+            "read(1) -> 1"  // \r
+            "read(1) -> 1"  // \n
+            "read(4) -> 4"  // XXXX
+            "read(1) -> 1"  // \r
+            "read(1) -> 1"  // \n
+            "read(1) -> 0 [timeout]");
+  }
+
+  SUBCASE("read() doen't wait if final chunk received") {
+    uint8_t buffer[32];
+
+    upstream.print(
+        "4\r\n"
+        "XXXX\r\n"
+        "0\r\n"
+        "\r\n");
+    REQUIRE(client.read(buffer, 32) == 4);
+    REQUIRE(buffer[0] == 'X');
+    REQUIRE(client.ended() == true);
+
+    REQUIRE(log.str() ==
+            "read(1) -> 1"  // 1
+            "read(1) -> 1"  // \r
+            "read(1) -> 1"  // \n
+            "read(4) -> 4"  // XXXX
+            "read(1) -> 1"  // \r
+            "read(1) -> 1"  // \n
+            "read(1) -> 1"  // 0
+            "read(1) -> 1"  // \r
+            "read(1) -> 1"  // \n
+            "read(1) -> 1"  // \r
+            "read(1) -> 1"  // \n
+    );
+  }
+}