Merge pull request #67 from TysonAndre/depth-ignore

Ignore pathologically large depths longer than strlen

Author: crazyxman

package.xml

@@ -18,10 +18,10 @@
   <active>yes</active>
  </lead>
  -->
- <date>2022-08-30</date>
+ <date>2022-09-30</date>
  <version>
-  <release>2.0.4dev</release>
-  <api>2.0.4dev</api>
+  <release>2.0.4</release>
+  <api>2.0.4</api>
  </version>
  <stability>
   <release>stable</release>
@@ -30,6 +30,8 @@
  <license uri="https://www.apache.org/licenses/LICENSE-2.0.html">Apache 2.0</license>
  <notes>
 * Add `-fvisibility=hidden` to compiler options to reduce compiled extension size by avoiding exporting symbols by default.
+* If the requested json parsing $depth is excessively large when reallocating larger buffers for the C simdjson parser,
+  then internally use a smaller $depth that would behave identically with lower memory usage. (#66)
  </notes>
  <contents>
   <dir name="/">
@@ -55,6 +57,7 @@
     <file name="decode_integer_overflow.phpt" role="test"/>
     <file name="decode_invalid_property.phpt" role="test"/>
     <file name="decode_max_depth.phpt" role="test"/>
+    <file name="decode_max_depth_memory_reduction.phpt" role="test"/>
     <file name="decode_result.phpt" role="test"/>
     <file name="decode_strict_types.phpt" role="test"/>
     <file name="decode_types.phpt" role="test"/>

php_simdjson.h

@@ -17,7 +17,7 @@
 extern zend_module_entry simdjson_module_entry;
 #define phpext_simdjson_ptr &simdjson_module_entry
 
-#define PHP_SIMDJSON_VERSION                  "2.0.4dev"
+#define PHP_SIMDJSON_VERSION                  "2.0.4"
 #define SIMDJSON_SUPPORT_URL                  "https://github.com/crazyxman/simdjson_php"
 #define SIMDJSON_PARSE_FAIL                   0
 #define SIMDJSON_PARSE_SUCCESS                1

src/bindings.cpp

@@ -26,6 +26,8 @@ extern "C" {
 #define zend_string_release_ex(s, persistent) zend_string_release((s))
 #endif
 
+#define SIMDJSON_DEPTH_CHECK_THRESHOLD 100000
+
 static inline simdjson::simdjson_result<simdjson::dom::element>
 get_key_with_optional_prefix(simdjson::dom::element &doc, std::string_view json_pointer)
 {
@@ -36,6 +38,25 @@ get_key_with_optional_prefix(simdjson::dom::element &doc, std::string_view json_
 static simdjson::error_code
 build_parsed_json_cust(simdjson::dom::parser& parser, simdjson::dom::element &doc, const char *buf, size_t len, bool realloc_if_needed,
                        size_t depth = simdjson::DEFAULT_MAX_DEPTH) {
+    if (UNEXPECTED(depth > SIMDJSON_DEPTH_CHECK_THRESHOLD) && depth > len && depth > parser.max_depth()) {
+        /*
+         * Choose the depth in a way that both avoids frequent reallocations
+         * and avoids excessive amounts of wasted memory beyond multiples of the largest string ever decoded.
+         *
+         * If the depth is already sufficient to parse a string of length `len`,
+         * then use the parser's previous depth.
+         *
+         * Precondition: depth > len
+         * Postcondition: depth <= original_depth && depth > len
+         */
+        if (len < SIMDJSON_DEPTH_CHECK_THRESHOLD) {
+            depth = SIMDJSON_DEPTH_CHECK_THRESHOLD;
+        } else if (depth > len * 2) {
+            // In callers, simdjson_validate_depth ensures depth <= SIMDJSON_MAX_DEPTH (which is <= SIZE_MAX/8),
+            // so len * 2 is even smaller than the previous depth and won't overflow.
+            depth = len * 2;
+        }
+    }
     auto error = parser.allocate(len, depth);
 
     if (error) {

tests/decode_max_depth_memory_reduction.phpt

@@ -0,0 +1,35 @@
+--TEST--
+simdjson_decode uses smaller depth than max_depth when safe to do so
+--FILE--
+<?php
+declare(strict_types=1);
+ini_set('error_reporting', (string)E_ALL);
+ini_set('display_errors', 'stderr');
+
+// This should only be allocating a few megabytes of memory, not gigabytes,
+// due to internally choosing a smaller depth that behaves equivalently for excessively large requested depths.
+foreach ([1024, 1 << 27] as $depth) {
+    echo "Test depth=$depth:\n";
+    $value = simdjson_decode('[]', true, $depth);
+    var_dump($value);
+    $value = simdjson_key_count('{"a":"b"}', 'a', $depth);
+    var_dump($value);
+    try {
+        simdjson_decode(str_repeat('[', 200000) . str_repeat(']', 199999), true, $depth);
+        echo "should be invalid\n";
+    } catch (Exception $e) {
+        printf("Caught %s: %s\n", get_class($e), $e->getMessage());
+    }
+}
+?>
+--EXPECT--
+Test depth=1024:
+array(0) {
+}
+int(0)
+Caught RuntimeException: The JSON document was too deep (too many nested objects and arrays)
+Test depth=134217728:
+array(0) {
+}
+int(0)
+Caught RuntimeException: The JSON document has an improper structure: missing or superfluous commas, braces, missing keys, etc.