Merge pull request #5462 from willmmiles/16_x-unlimited-fxdata-size

Serialize fxdata without ArduinoJSON

Author: netmindz

wled00/fcn_declare.h

@@ -176,7 +176,6 @@ void serializeSegment(const JsonObject& root, const Segment& seg, byte id, bool
 void serializeState(JsonObject root, bool forPreset = false, bool includeBri = true, bool segmentBounds = true, bool selectedSegmentsOnly = false);
 void serializeInfo(JsonObject root);
 void serializeModeNames(JsonArray arr);
-void serializeModeData(JsonArray fxdata);
 void serializePins(JsonObject root);
 void serveJson(AsyncWebServerRequest* request);
 #ifdef WLED_ENABLE_JSONLIVE

wled00/json.cpp

@@ -1160,21 +1160,6 @@ void serializePins(JsonObject root)
   }
 }
 
-// deserializes mode data string into JsonArray
-void serializeModeData(JsonArray fxdata)
-{
-  char lineBuffer[256];
-  for (size_t i = 0; i < strip.getModeCount(); i++) {
-    strncpy_P(lineBuffer, strip.getModeData(i), sizeof(lineBuffer)/sizeof(char)-1);
-    lineBuffer[sizeof(lineBuffer)/sizeof(char)-1] = '\0'; // terminate string
-    if (lineBuffer[0] != 0) {
-      char* dataPtr = strchr(lineBuffer,'@');
-      if (dataPtr) fxdata.add(dataPtr+1);
-      else         fxdata.add("");
-    }
-  }
-}
-
 // deserializes mode names string into JsonArray
 // also removes effect data extensions (@...) from deserialised names
 void serializeModeNames(JsonArray arr)
@@ -1191,6 +1176,78 @@ void serializeModeNames(JsonArray arr)
   }
 }
 
+// Writes a JSON-escaped string (with surrounding quotes) into dest[0..maxLen-1].
+// Returns bytes written, or 0 if the buffer was too small.
+static size_t writeJSONString(uint8_t* dest, size_t maxLen, const char* src) {
+  size_t pos = 0;
+
+  auto emit = [&](char c) -> bool {
+    if (pos >= maxLen) return false;
+    dest[pos++] = (uint8_t)c;
+    return true;
+  };
+
+  if (!emit('"')) return 0;
+
+  for (const char* p = src; *p; ++p) {
+    char esc = ARDUINOJSON_NAMESPACE::EscapeSequence::escapeChar(*p);
+    if (esc) {
+      if (!emit('\\') || !emit(esc)) return 0;
+    } else {
+      if (!emit(*p)) return 0;
+    }
+  }
+
+  if (!emit('"')) return 0;
+  return pos;
+}
+
+// Writes ,"<escaped_src>" into dest[0..maxLen-1] (no null terminator).
+// Returns bytes written, or 0 if the buffer was too small.
+static size_t writeJSONStringElement(uint8_t* dest, size_t maxLen, const char* src) {
+  if (maxLen == 0) return 0;
+  dest[0] = ',';
+  size_t n = writeJSONString(dest + 1, maxLen - 1, src);
+  if (n == 0) return 0;
+  return 1 + n;
+}
+
+// Generate a streamed JSON response for the mode data
+// This uses sendChunked to send the reply in blocks based on how much fit in the outbound
+// packet buffer, minimizing the required state (ie. just the next index to send).  This
+// allows us to send an arbitrarily large response without using any significant amount of
+// memory (so no worries about buffer limits).
+void respondModeData(AsyncWebServerRequest* request) {
+  size_t fx_index = 0;
+  request->sendChunked(FPSTR(CONTENT_TYPE_JSON),
+    [fx_index](uint8_t* data, size_t len, size_t) mutable {
+      size_t bytes_written = 0;
+      char lineBuffer[256];
+      while (fx_index < strip.getModeCount()) {
+        strncpy_P(lineBuffer, strip.getModeData(fx_index), sizeof(lineBuffer)-1); // Copy to stack buffer for strchr
+        if (lineBuffer[0] != 0) {
+          lineBuffer[sizeof(lineBuffer)-1] = '\0'; // terminate string (only needed if strncpy filled the buffer)
+          const char* dataPtr = strchr(lineBuffer,'@'); // Find '@', if there is one
+          size_t mode_bytes = writeJSONStringElement(data, len, dataPtr ? dataPtr + 1 : "");
+          if (mode_bytes == 0) break;  // didn't fit; break loop and try again next packet
+          if (fx_index == 0) *data = '[';
+          data += mode_bytes;
+          len -= mode_bytes;
+          bytes_written += mode_bytes;
+        }
+        ++fx_index;        
+      }
+
+      if ((fx_index == strip.getModeCount()) && (len >= 1)) {
+        *data = ']';
+        ++bytes_written;
+        ++fx_index; // we're really done
+      }
+
+      return bytes_written;
+  });
+}
+
 // Global buffer locking response helper class (to make sure lock is released when AsyncJsonResponse is destroyed)
 class LockedJsonResponse: public AsyncJsonResponse {
   bool _holding_lock;
@@ -1218,7 +1275,7 @@ class LockedJsonResponse: public AsyncJsonResponse {
 void serveJson(AsyncWebServerRequest* request)
 {
   enum class json_target {
-    all, state, info, state_info, nodes, effects, palettes, fxdata, networks, config, pins
+    all, state, info, state_info, nodes, effects, palettes, networks, config, pins
   };
   json_target subJson = json_target::all;
 
@@ -1229,7 +1286,7 @@ void serveJson(AsyncWebServerRequest* request)
   else if (url.indexOf(F("nodes")) > 0) subJson = json_target::nodes;
   else if (url.indexOf(F("eff"))   > 0) subJson = json_target::effects;
   else if (url.indexOf(F("palx"))  > 0) subJson = json_target::palettes;
-  else if (url.indexOf(F("fxda"))  > 0) subJson = json_target::fxdata;
+  else if (url.indexOf(F("fxda"))  > 0) { respondModeData(request); return; }
   else if (url.indexOf(F("net"))   > 0) subJson = json_target::networks;
   else if (url.indexOf(F("cfg"))   > 0) subJson = json_target::config;
   else if (url.indexOf(F("pins"))  > 0) subJson = json_target::pins;
@@ -1254,7 +1311,7 @@ void serveJson(AsyncWebServerRequest* request)
   }
   // releaseJSONBufferLock() will be called when "response" is destroyed (from AsyncWebServer)
   // make sure you delete "response" if no "request->send(response);" is made
-  LockedJsonResponse *response = new LockedJsonResponse(pDoc, subJson==json_target::fxdata || subJson==json_target::effects); // will clear and convert JsonDocument into JsonArray if necessary
+  LockedJsonResponse *response = new LockedJsonResponse(pDoc, subJson==json_target::effects); // will clear and convert JsonDocument into JsonArray if necessary
 
   JsonVariant lDoc = response->getRoot();
 
@@ -1270,8 +1327,6 @@ void serveJson(AsyncWebServerRequest* request)
       serializePalettes(lDoc, request->hasParam(F("page")) ? request->getParam(F("page"))->value().toInt() : 0); break;
     case json_target::effects:
       serializeModeNames(lDoc); break;
-    case json_target::fxdata:
-      serializeModeData(lDoc); break;
     case json_target::networks:
       serializeNetworks(lDoc); break;
     case json_target::config: