From 2f8786b4dc15f6fdd0ab6a2188309b10d740cb8d Mon Sep 17 00:00:00 2001 From: Sean Tymon Date: Wed, 24 Jun 2026 08:46:52 +0100 Subject: [PATCH 01/12] feat: Improve perf and add conveniences --- docs/json-repair/configuration.mdx | 70 ++++++ src/Concerns/InputSanitization.php | 74 ++++++ src/Concerns/OutputTracking.php | 76 ++++++ src/Concerns/RepairLogging.php | 29 ++- src/Concerns/StateMachine.php | 206 ++++++++++------ src/DuplicateKeyPolicy.php | 11 + src/JsonRepairer.php | 372 +++++++++++++++++++++-------- src/RepairResult.php | 17 ++ src/StreamingJsonRepairer.php | 54 +++++ src/functions.php | 4 +- tests/Datasets/JsonRepair.php | 36 +++ tests/Unit/JsonRepairerTest.php | 115 +++++++++ 12 files changed, 892 insertions(+), 172 deletions(-) create mode 100644 src/Concerns/OutputTracking.php create mode 100644 src/DuplicateKeyPolicy.php create mode 100644 src/RepairResult.php create mode 100644 src/StreamingJsonRepairer.php diff --git a/docs/json-repair/configuration.mdx b/docs/json-repair/configuration.mdx index dc9260d..7b4b6f1 100644 --- a/docs/json-repair/configuration.mdx +++ b/docs/json-repair/configuration.mdx @@ -64,6 +64,21 @@ json_repair('{"complete": "value", "incomplete": "partial', omitIncompleteString // {"complete": "value"} ``` +## duplicateKeyPolicy + +When repairing malformed JSON that repeats the same object key, choose which value to keep. Pass `null` (default) to leave duplicate keys as-is in the repaired string. + +```php +use Cortex\JsonRepair\DuplicateKeyPolicy; +use Cortex\JsonRepair\JsonRepairer; + +// keep-first: {"a": 1} +new JsonRepairer('{a: 1, a: 2}', duplicateKeyPolicy: DuplicateKeyPolicy::KeepFirst)->repair(); + +// keep-last: {"a": 2} +new JsonRepairer('{a: 1, a: 2}', duplicateKeyPolicy: DuplicateKeyPolicy::KeepLast)->repair(); +``` + ## Combined Options Use both options together for streaming LLM output: @@ -92,3 +107,58 @@ $repairer = new JsonRepairer( $repaired = $repairer->repair(); // {"key1": "v1"} ``` + +## Advanced APIs + +### Structured repair report + +`repairWithDetails()` returns a `RepairResult` with the repaired JSON, whether input was already valid, and a list of fixes applied (same messages as debug logging): + +```php +use Cortex\JsonRepair\JsonRepairer; + +$result = (new JsonRepairer("{'key': 'value'}"))->repairWithDetails(); +// $result->json, $result->wasAlreadyValid, $result->fixes +``` + +### Multiple values / NDJSON + +`repairAll()` repairs each top-level JSON value separately: + +```php +$results = (new JsonRepairer("{'a':1}\n{'b':2}"))->repairAll(); +// ['{"a": 1}', '{"b": 2}'] +``` + +### Streaming repair + +`StreamingJsonRepairer` accumulates chunks and returns the best-effort repair so far: + +```php +use Cortex\JsonRepair\StreamingJsonRepairer; + +$stream = new StreamingJsonRepairer(); +$stream->feed('{"name": "'); +$partial = $stream->current(); // closes incomplete string/braces +``` + +### Scalar decode + +`decode()` returns `mixed` — top-level scalars (strings, numbers, booleans, null) decode directly without being cast to arrays: + +```php +(new JsonRepairer('true'))->decode(); // true +json_repair_decode('null'); // null +``` + +## Invalid numbers and non-finite values + +The repairer normalizes common invalid numeric literals: + +| Input | Repaired | +|-------|----------| +| `+123` | `123` | +| `007` | `7` | +| `.5` | `0.5` | +| `5.` | `5` | +| `Infinity`, `-Infinity`, `NaN` | `null` | diff --git a/src/Concerns/InputSanitization.php b/src/Concerns/InputSanitization.php index a12b8cd..efe7fde 100644 --- a/src/Concerns/InputSanitization.php +++ b/src/Concerns/InputSanitization.php @@ -21,6 +21,10 @@ trait InputSanitization */ private function extractJsonFromMarkdown(string $input): string { + if (! str_contains($input, '```')) { + return $input; + } + $matchCount = preg_match_all('/```json\s*([\s\S]*?)\s*```/', $input, $matches); if ($matchCount > 0) { @@ -352,4 +356,74 @@ private function extractFirstValidJson(string $input): string return $bestMatch ?? $input; } + + /** + * Extract all top-level JSON objects or arrays from the input. + * + * @return list + */ + private function extractAllTopLevelJson(string $input): array + { + if (json_validate($input)) { + return [$input]; + } + + $length = strlen($input); + $results = []; + $depth = 0; + $start = -1; + $inString = false; + $stringDelimiter = ''; + $escapeNext = false; + + for ($i = 0; $i < $length; $i++) { + $char = $input[$i]; + + if ($escapeNext) { + $escapeNext = false; + continue; + } + + if ($inString) { + if ($char === '\\') { + $escapeNext = true; + continue; + } + + if ($char === $stringDelimiter) { + $inString = false; + $stringDelimiter = ''; + } + + continue; + } + + if ($char === '"' || $char === "'") { + $inString = true; + $stringDelimiter = $char; + continue; + } + + if ($char === '{' || $char === '[') { + if ($depth === 0) { + $start = $i; + } + + $depth++; + } elseif ($char === '}' || $char === ']') { + $depth--; + + if ($depth === 0 && $start !== -1) { + $results[] = substr($input, $start, $i - $start + 1); + $start = -1; + } + } + } + + if ($results === []) { + return [$input]; + } + + return $results; + } } diff --git a/src/Concerns/OutputTracking.php b/src/Concerns/OutputTracking.php new file mode 100644 index 0000000..508cb5a --- /dev/null +++ b/src/Concerns/OutputTracking.php @@ -0,0 +1,76 @@ +lastNonWhitespaceChar = null; + $this->outputSyncedLength = 0; + } + + private function syncOutputTail(): void + { + $length = strlen($this->output); + + for ($j = $this->outputSyncedLength; $j < $length; $j++) { + $char = $this->output[$j]; + + if (! in_array($char, [' ', "\t", "\n", "\r", "\f", "\v"], true)) { + $this->lastNonWhitespaceChar = $char; + } + } + + $this->outputSyncedLength = $length; + } + + private function recalculateLastNonWhitespaceChar(): void + { + $trimmed = rtrim($this->output); + $this->lastNonWhitespaceChar = $trimmed === '' ? null : $trimmed[strlen($trimmed) - 1]; + $this->outputSyncedLength = strlen($this->output); + } + + private function outputEndsWithNonWhitespace(string $char): bool + { + $this->syncOutputTail(); + + return $this->lastNonWhitespaceChar === $char; + } + + private function trimOutputTrailingWhitespace(): void + { + if ($this->output === '') { + return; + } + + $lastChar = $this->output[strlen($this->output) - 1]; + + if (in_array($lastChar, [' ', "\t", "\n", "\r", "\f", "\v"], true)) { + $this->output = rtrim($this->output); + $this->recalculateLastNonWhitespaceChar(); + } + } + + private function truncateOutput(int $length): void + { + $this->output = substr($this->output, 0, $length); + $this->recalculateLastNonWhitespaceChar(); + } + + private function setOutput(string $output): void + { + $this->output = $output; + $this->recalculateLastNonWhitespaceChar(); + } +} diff --git a/src/Concerns/RepairLogging.php b/src/Concerns/RepairLogging.php index 54d2880..3fb62ea 100644 --- a/src/Concerns/RepairLogging.php +++ b/src/Concerns/RepairLogging.php @@ -10,19 +10,44 @@ trait RepairLogging { /** - * Log a repair action with context. - * + * @var list + */ + private array $collectedFixes = []; + + private bool $collectFixes = false; + + /** * @param string $message Description of the repair action * @param array $context Additional context data */ private function log(string $message, array $context = []): void { + if ($this->collectFixes) { + $this->collectedFixes[] = $message; + } + $this->logger?->debug($message, array_merge([ 'position' => $this->pos, 'context' => $this->getContextSnippet(), ], $context)); } + private function beginFixCollection(): void + { + $this->collectFixes = true; + $this->collectedFixes = []; + } + + /** + * @return list + */ + private function endFixCollection(): array + { + $this->collectFixes = false; + + return $this->collectedFixes; + } + /** * Get a snippet of the JSON around the current position for logging context. */ diff --git a/src/Concerns/StateMachine.php b/src/Concerns/StateMachine.php index e25e530..aa57cbd 100644 --- a/src/Concerns/StateMachine.php +++ b/src/Concerns/StateMachine.php @@ -9,6 +9,42 @@ */ trait StateMachine { + /** + * Handle an escape sequence within a string. + * + * Processes escape sequences like \", \\, \/, \b, \f, \n, \r, \t, and + * unicode escapes (\uXXXX). Invalid or incomplete escapes are treated + * as escaped backslash followed by the character. + * + * @return int Number of extra characters consumed beyond the escape character itself + */ + /** + * @var string + */ + private const VALID_ESCAPES = '"\\/bfnrt'; + + /** + * @var string + */ + private const UNQUOTED_VALUE_STOP_CHARS = ",}]\"'"; + + /** + * @var list + */ + private const KEYWORD_MATCH_SPECS = [ + ['infinity', 8, 'null'], + ['true', 4, 'true'], + ['false', 5, 'false'], + ['nan', 3, 'null'], + ['null', 4, 'null'], + ['none', 4, 'null'], + ]; + + /** + * @var string + */ + private const KEYWORD_FIRST_CHARS = 'tTfFnNiI'; + /** * Handle the starting state of parsing. * @@ -34,6 +70,7 @@ private function tryOpenObjectOrArray(string $json, int $i, bool $resetKeyTracki if ($char === '{') { $this->output .= '{'; $this->stack[] = '}'; + $this->pushObjectKeyScope(); $this->state = self::STATE_IN_OBJECT_KEY; if ($resetKeyTracking) { @@ -90,6 +127,11 @@ private function handleObjectKey(string $json, int $i): int $this->removeTrailingComma(); $this->output .= '}'; array_pop($this->stack); + + if ($this->objectKeysStack !== []) { + array_pop($this->objectKeysStack); + } + $this->state = $this->stack === [] ? self::STATE_START : self::STATE_EXPECTING_COMMA_OR_END; return $i + 1; @@ -216,6 +258,12 @@ private function handleExpectingColon(string $json, int $i): int $length = strlen($json); if ($char === ':') { + if ($this->handleDuplicateKey($this->extractCompletedKeyName())) { + $this->state = self::STATE_IN_OBJECT_VALUE; + + return $this->skipValueAt($json, $i + 1); + } + $this->output .= ':'; $this->state = self::STATE_IN_OBJECT_VALUE; @@ -231,6 +279,12 @@ private function handleExpectingColon(string $json, int $i): int // Missing colon, insert it if (! ctype_space($char)) { + if ($this->handleDuplicateKey($this->extractCompletedKeyName())) { + $this->state = self::STATE_IN_OBJECT_VALUE; + + return $this->skipValueAt($json, $i); + } + $this->log('Inserting missing colon after key'); $this->output .= ':'; $this->state = self::STATE_IN_OBJECT_VALUE; @@ -284,11 +338,8 @@ private function handleObjectValue(string $json, int $i): int if ($char === '}') { // Check for missing value - output ends with colon (possibly followed by space) - $trimmedOutput = rtrim($this->output); - - if (str_ends_with($trimmedOutput, ':')) { - // Remove trailing space(s) after colon before adding empty value - $this->output = $trimmedOutput; + if ($this->outputEndsWithNonWhitespace(':')) { + $this->trimOutputTrailingWhitespace(); if ($this->omitEmptyValues) { $this->log('Removing key with missing value (omitEmptyValues enabled)'); @@ -302,6 +353,11 @@ private function handleObjectValue(string $json, int $i): int $this->removeTrailingComma(); $this->output .= '}'; array_pop($this->stack); + + if ($this->objectKeysStack !== []) { + array_pop($this->objectKeysStack); + } + $this->state = $this->stack === [] ? self::STATE_START : self::STATE_EXPECTING_COMMA_OR_END; return $i + 1; @@ -311,7 +367,8 @@ private function handleObjectValue(string $json, int $i): int $keywordMatch = $this->tryMatchKeyword($json, $i, $length); if ($keywordMatch !== null) { - [$normalized, $klen] = $keywordMatch; + $normalized = $keywordMatch[0]; + $klen = $keywordMatch[1]; $original = substr($json, $i, $klen); if ($original !== $normalized) { @@ -330,7 +387,7 @@ private function handleObjectValue(string $json, int $i): int } // Handle numbers - if (ctype_digit($char) || $char === '-' || $char === '+') { + if (ctype_digit($char) || $char === '-' || $char === '+' || $char === '.') { $this->state = self::STATE_IN_NUMBER; return $i; @@ -419,7 +476,8 @@ private function handleArrayValue(string $json, int $i): int $keywordMatch = $this->tryMatchKeyword($json, $i, $length); if ($keywordMatch !== null) { - [$normalized, $klen] = $keywordMatch; + $normalized = $keywordMatch[0]; + $klen = $keywordMatch[1]; $this->output .= $normalized; $this->state = self::STATE_EXPECTING_COMMA_OR_END; @@ -427,7 +485,7 @@ private function handleArrayValue(string $json, int $i): int } // Handle numbers - if (ctype_digit($char) || $char === '-' || $char === '+') { + if (ctype_digit($char) || $char === '-' || $char === '+' || $char === '.') { $this->state = self::STATE_IN_NUMBER; return $i; @@ -456,6 +514,11 @@ private function handleExpectingCommaOrEnd(string $json, int $i): int $this->removeTrailingComma(); $this->output .= $top; array_pop($this->stack); + + if ($top === '}' && $this->objectKeysStack !== []) { + array_pop($this->objectKeysStack); + } + $this->state = $this->stack === [] ? self::STATE_START : self::STATE_EXPECTING_COMMA_OR_END; return $i + 1; @@ -504,25 +567,44 @@ private function handleNumber(string $json, int $i): int { $length = strlen($json); - // Handle sign - if ($i < $length && ($json[$i] === '-' || $json[$i] === '+')) { - $this->output .= $json[$i]; + if ($i < $length && $json[$i] === '+') { + $i++; + } elseif ($i < $length && $json[$i] === '-') { + $this->output .= '-'; $i++; } - // Handle integer part — batch all digits into one substr() append - $start = $i; - while ($i < $length && ctype_digit($json[$i])) { + if ($i < $length && $json[$i] === '.') { + $this->output .= '0.'; $i++; - } + $start = $i; + while ($i < $length && ctype_digit($json[$i])) { + $i++; + } - if ($i > $start) { - $this->output .= substr($json, $start, $i - $start); + if ($i > $start) { + $this->output .= substr($json, $start, $i - $start); + } + } else { + $start = $i; + while ($i < $length && ctype_digit($json[$i])) { + $i++; + } + + if ($i > $start) { + $intPart = substr($json, $start, $i - $start); + $intPart = ltrim($intPart, '0'); + + if ($intPart === '') { + $intPart = '0'; + } + + $this->output .= $intPart; + } } - // Handle decimal point if ($i < $length && $json[$i] === '.') { - $this->output .= '.'; + $dotPos = $i; $i++; $start = $i; while ($i < $length && ctype_digit($json[$i])) { @@ -530,22 +612,26 @@ private function handleNumber(string $json, int $i): int } if ($i > $start) { - $this->output .= substr($json, $start, $i - $start); + $this->output .= '.' . substr($json, $start, $i - $start); + } else { + $i = $dotPos + 1; } } - // Handle exponent if ($i < $length && ($json[$i] === 'e' || $json[$i] === 'E')) { $exponentStart = $i; $this->output .= $json[$i]; $i++; if ($i < $length && ($json[$i] === '-' || $json[$i] === '+')) { - $this->output .= $json[$i]; - $i++; + if ($json[$i] === '+') { + $i++; + } else { + $this->output .= '-'; + $i++; + } } - // Batch exponent digits; track where they start to detect empty exponent $digitStart = $i; while ($i < $length && ctype_digit($json[$i])) { $i++; @@ -554,32 +640,19 @@ private function handleNumber(string $json, int $i): int if ($i > $digitStart) { $this->output .= substr($json, $digitStart, $i - $digitStart); } else { - // No digits after 'e'/'E' — remove the incomplete exponent (letter + optional sign) - $this->output = substr($this->output, 0, -($digitStart - $exponentStart)); + $this->truncateOutput(strlen($this->output) - ($digitStart - $exponentStart)); } } $this->state = self::STATE_EXPECTING_COMMA_OR_END; - // Reset key tracking after successfully completing a number value $this->currentKeyStart = -1; return $i; } - /** - * Handle an escape sequence within a string. - * - * Processes escape sequences like \", \\, \/, \b, \f, \n, \r, \t, and - * unicode escapes (\uXXXX). Invalid or incomplete escapes are treated - * as escaped backslash followed by the character. - * - * @return int Number of extra characters consumed beyond the escape character itself - */ private function handleEscapeSequence(string $char, string $json): int { - $validEscapes = ['"', '\\', '/', 'b', 'f', 'n', 'r', 't']; - - if (in_array($char, $validEscapes, true)) { + if (str_contains(self::VALID_ESCAPES, $char)) { $this->output .= '\\' . $char; return 0; @@ -620,12 +693,13 @@ private function getNextStateAfterString(): int */ private function removeTrailingComma(): void { - $trimmed = rtrim($this->output); - - if (str_ends_with($trimmed, ',')) { - $this->log('Removing trailing comma'); - $this->output = substr($trimmed, 0, -1); + if (! $this->outputEndsWithNonWhitespace(',')) { + return; } + + $this->log('Removing trailing comma'); + $this->trimOutputTrailingWhitespace(); + $this->truncateOutput(strlen($this->output) - 1); } /** @@ -665,7 +739,7 @@ private function removeCurrentKey(): void $beforeKey = rtrim(substr($beforeKey, 0, -1)); } - $this->output = $beforeKey; + $this->setOutput($beforeKey); $this->currentKeyStart = -1; } @@ -690,7 +764,7 @@ private function handleUnquotedStringValue(string $json, int $i): int $char = $json[$i]; // Stop at structural characters or quotes - if (in_array($char, [',', '}', ']', '"', "'"], true)) { + if (strpbrk($char, self::UNQUOTED_VALUE_STOP_CHARS) !== false) { break; } @@ -760,44 +834,28 @@ private function escapeStringValue(string $value): string } /** - * Boolean / null keyword literals for {@see tryMatchKeyword()} (substr_compare + word boundary). - * - * @return list Each tuple: keyword text, byte length, normalized JSON token. - */ - private static function keywordMatchSpecs(): array - { - return [ - ['true', 4, 'true'], - ['false', 5, 'false'], - ['null', 4, 'null'], - ['none', 4, 'null'], - ]; - } - - /** - * Try to match a boolean or null keyword at the given position without regex. - * - * Uses substr_compare to avoid creating a substring and bypasses the regex - * engine entirely. Checks a word boundary after the match. - * - * @param int $length Pre-computed strlen($json) - * - * @return array{string, int}|null [normalized_value, keyword_length] or null + * @return array{0: string, 1: int}|null */ private function tryMatchKeyword(string $json, int $i, int $length): ?array { $c = $json[$i]; - // Quick first-char gate before doing heavier work - if (! in_array($c, ['t', 'T', 'f', 'F', 'n', 'N'], true)) { + if ($c === '-' && $i + 8 < $length && substr_compare($json, 'infinity', $i + 1, 8, true) === 0) { + $afterPos = $i + 9; + + if ($afterPos >= $length || (! ctype_alnum($json[$afterPos]) && $json[$afterPos] !== '_')) { + return ['null', 9]; + } + } + + if (! str_contains(self::KEYWORD_FIRST_CHARS, $c)) { return null; } - foreach (self::keywordMatchSpecs() as [$keyword, $klen, $normalized]) { + foreach (self::KEYWORD_MATCH_SPECS as [$keyword, $klen, $normalized]) { if ($length - $i >= $klen && substr_compare($json, $keyword, $i, $klen, true) === 0) { $afterPos = $i + $klen; - // Word boundary: next char must not be alphanumeric or underscore if ($afterPos >= $length || (! ctype_alnum($json[$afterPos]) && $json[$afterPos] !== '_')) { return [$normalized, $klen]; } diff --git a/src/DuplicateKeyPolicy.php b/src/DuplicateKeyPolicy.php new file mode 100644 index 0000000..c6e6632 --- /dev/null +++ b/src/DuplicateKeyPolicy.php @@ -0,0 +1,11 @@ +> + */ + private array $objectKeysStack = []; + + private bool $skipNextValue = false; + /** * @param string $json The JSON string to repair * @param bool $ensureAscii Whether to escape non-ASCII characters (default: true) * @param bool $omitEmptyValues Whether to remove keys with missing values instead of adding empty strings (default: false) * @param bool $omitIncompleteStrings Whether to remove keys with incomplete string values instead of closing them (default: false) + * @param DuplicateKeyPolicy|null $duplicateKeyPolicy How to handle duplicate object keys (default: null — no deduplication) */ public function __construct( private readonly string $json, private readonly bool $ensureAscii = true, private readonly bool $omitEmptyValues = false, private readonly bool $omitIncompleteStrings = false, + private readonly ?DuplicateKeyPolicy $duplicateKeyPolicy = null, ) {} - /** - * Repair the JSON string and return the corrected version. - * - * This method attempts to fix various common JSON errors including: - * - Missing quotes around keys and values - * - Missing commas between elements - * - Trailing commas - * - Unclosed brackets, braces, and strings - * - Single quotes instead of double quotes - * - Non-standard boolean/null values (True, False, None) - * - Incomplete escape sequences - * - Missing colons in key-value pairs - * - * @return string The repaired JSON string - * - * @throws \Cortex\JsonRepair\Exceptions\JsonRepairException If the repaired JSON is still invalid - */ public function repair(): string + { + return $this->repairWithDetails()->json; + } + + public function repairWithDetails(): RepairResult { if (json_validate($this->json)) { + $this->beginFixCollection(); $this->log('JSON is already valid, returning as-is'); + $fixes = $this->endFixCollection(); - return $this->json; + return new RepairResult($this->json, true, $fixes); } + $this->beginFixCollection(); $this->log('Starting JSON repair'); + $repaired = $this->repairInternal(extractFirstOnly: true); + $fixes = $this->endFixCollection(); + + return new RepairResult($repaired, false, $fixes); + } + + /** + * Repair each top-level JSON value in the input (NDJSON / concatenated objects). + * + * @return list + */ + public function repairAll(): array + { + if (json_validate($this->json)) { + return [$this->json]; + } + + $json = $this->extractJsonFromMarkdown($this->json); + $json = $this->removeComments($json); + + $segments = $this->extractAllTopLevelJson($json); + + return array_map( + $this->repairSegment(...), + $segments, + ); + } + + /** + * @param int<1, max> $depth + */ + public function decode( + int $depth = 512, + int $flags = JSON_THROW_ON_ERROR, + ): mixed { + $repaired = $this->repair(); + + return json_decode($repaired, true, $depth, $flags); + } + + private function repairSegment(string $json): string + { + $repairer = new self( + $json, + $this->ensureAscii, + $this->omitEmptyValues, + $this->omitIncompleteStrings, + $this->duplicateKeyPolicy, + ); + + if ($this->logger instanceof LoggerInterface) { + $repairer->setLogger($this->logger); + } - // Extract JSON from markdown code blocks if present + return $repairer->repair(); + } + + private function repairInternal(bool $extractFirstOnly): string + { $json = $this->extractJsonFromMarkdown($this->json); if ($json !== $this->json) { @@ -111,10 +171,10 @@ public function repair(): string $json = $jsonWithoutComments; } - // Handle multiple JSON objects - $json = $this->extractFirstValidJson($json); + if ($extractFirstOnly) { + $json = $this->extractFirstValidJson($json); + } - // Reset state $this->state = self::STATE_START; $this->pos = 0; $this->output = ''; @@ -123,6 +183,9 @@ public function repair(): string $this->stringDelimiter = ''; $this->stateBeforeString = self::STATE_START; $this->currentKeyStart = -1; + $this->objectKeysStack = []; + $this->skipNextValue = false; + $this->resetOutputTracking(); $length = strlen($json); $i = 0; @@ -131,11 +194,7 @@ public function repair(): string $char = $json[$i]; $this->pos = $i; - // Handle escape sequences in strings - // @phpstan-ignore identical.alwaysFalse (state changes in loop iterations) if ($this->state === self::STATE_IN_STRING_ESCAPE) { - // If we're at the end of the string and in escape state, the escape is incomplete - // Just drop the incomplete escape (backslash wasn't added to output yet) if ($i >= strlen($json)) { $this->state = self::STATE_IN_STRING; break; @@ -147,14 +206,9 @@ public function repair(): string continue; } - // Handle characters inside strings - // @phpstan-ignore identical.alwaysFalse (state changes in loop iterations) if ($this->state === self::STATE_IN_STRING) { - // Check for smart quotes as closing delimiter - $smartQuoteLength = $this->getSmartQuoteLength($json, $i); + $smartQuoteLength = $char === "\xE2" ? $this->getSmartQuoteLength($json, $i) : 0; - // Handle double quote inside single-quoted string - must escape it - // @phpstan-ignore booleanAnd.alwaysFalse, identical.alwaysFalse (delimiter set when entering string state and can be single quote) if ($char === '"' && $this->stringDelimiter === "'") { $this->log('Escaping double quote inside single-quoted string'); $this->output .= '\\"'; @@ -162,16 +216,11 @@ public function repair(): string continue; } - // @phpstan-ignore identical.alwaysFalse (delimiter set when entering string state) if ($char === $this->stringDelimiter || $smartQuoteLength > 0) { - // Check if this quote should be escaped (it's inside the string value) - // @phpstan-ignore identical.alwaysFalse (smartQuoteLength can be 0 when char matches delimiter) $isRegularQuote = $smartQuoteLength === 0; - // @phpstan-ignore booleanOr.alwaysFalse - $isInValue = $this->stateBeforeString === self::STATE_IN_OBJECT_VALUE // @phpstan-ignore identical.alwaysFalse - || $this->stateBeforeString === self::STATE_IN_ARRAY; // @phpstan-ignore identical.alwaysFalse + $isInValue = $this->stateBeforeString === self::STATE_IN_OBJECT_VALUE + || $this->stateBeforeString === self::STATE_IN_ARRAY; - // @phpstan-ignore booleanAnd.leftAlwaysFalse, booleanAnd.rightAlwaysFalse, booleanAnd.alwaysFalse (variables can be true at runtime) if ($isRegularQuote && $isInValue && $this->shouldEscapeQuoteInValue($json, $i)) { $this->log('Escaping embedded quote inside string value'); $this->output .= '\\"'; @@ -179,47 +228,47 @@ public function repair(): string continue; } - // Always close with double quote, even if opened with single quote $this->output .= '"'; $this->inString = false; $this->stringDelimiter = ''; $this->state = $this->getNextStateAfterString(); - // Reset key tracking after successfully completing a string value if ($this->state === self::STATE_EXPECTING_COMMA_OR_END) { $this->currentKeyStart = -1; } - // @phpstan-ignore greater.alwaysTrue (smartQuoteLength can be 0 when char matches delimiter) $i += $smartQuoteLength > 0 ? $smartQuoteLength : 1; continue; } if ($char === '\\') { - // Don't output the backslash yet - let handleEscapeSequence decide $this->state = self::STATE_IN_STRING_ESCAPE; $i++; continue; } - // Check if this is a structural character that should close an unclosed string - // This handles cases like {"key": "value with no closing quote} if (($char === '}' || $char === ']') && $this->shouldCloseStringAtStructuralChar($json, $i)) { $this->log('Closing unclosed string at structural character', [ 'char' => $char, ]); - // Close the string and let the structural character be processed $this->output .= '"'; $this->inString = false; $this->stringDelimiter = ''; $this->state = $this->getNextStateAfterString(); - // Reset key tracking if ($this->state === self::STATE_EXPECTING_COMMA_OR_END) { $this->currentKeyStart = -1; } - // Don't increment i - let the structural char be processed in the next iteration + continue; + } + + $stopChars = '\\' . $this->stringDelimiter . "\"}\xE2"; + $runLength = strcspn($json, $stopChars, $i); + + if ($runLength > 0) { + $this->output .= substr($json, $i, $runLength); + $i += $runLength; continue; } @@ -228,14 +277,19 @@ public function repair(): string continue; } - // Skip whitespace if (ctype_space($char)) { $i++; continue; } + if ($this->skipNextValue && ($this->state === self::STATE_IN_OBJECT_VALUE || $this->state === self::STATE_IN_NUMBER)) { + $i = $this->skipValueAt($json, $i); + $this->skipNextValue = false; + $this->state = self::STATE_EXPECTING_COMMA_OR_END; + continue; + } + $i = match ($this->state) { - // @phpstan-ignore match.alwaysTrue (first iteration starts at STATE_START, then changes) self::STATE_START => $this->handleStart($json, $i), self::STATE_IN_OBJECT_KEY => $this->handleObjectKey($json, $i), self::STATE_EXPECTING_COLON => $this->handleExpectingColon($json, $i), @@ -247,35 +301,21 @@ public function repair(): string }; } - // Close any unclosed strings - // @phpstan-ignore if.alwaysFalse (can be true if string wasn't closed in loop) if ($this->inString) { - // Check if we should remove incomplete string values - // @phpstan-ignore booleanAnd.alwaysFalse, identical.alwaysFalse (stateBeforeString is set when entering string state and can be STATE_IN_OBJECT_VALUE) if ($this->omitIncompleteStrings && $this->stateBeforeString === self::STATE_IN_OBJECT_VALUE) { $this->log('Removing incomplete string value (omitIncompleteStrings enabled)'); $this->removeCurrentKey(); - // Update state after removing key $this->state = self::STATE_EXPECTING_COMMA_OR_END; } else { $this->log('Adding missing closing quote for unclosed string'); $this->output .= '"'; - - // Note: If we were in escape state, the incomplete escape backslash - // was never added to output (we defer adding it to handleEscapeSequence) - - // Update state after closing string $this->state = $this->getNextStateAfterString(); } $this->inString = false; } - // Handle incomplete key (key without colon/value) - // Check if we're expecting a colon (just finished a key) but don't have one - // @phpstan-ignore identical.alwaysFalse (state set to STATE_EXPECTING_COLON after closing string key) if ($this->state === self::STATE_EXPECTING_COLON) { - // We have a key but no colon/value - add colon and empty value if ($this->omitEmptyValues) { $this->log('Removing key without value (omitEmptyValues enabled)'); $this->removeCurrentKey(); @@ -285,10 +325,7 @@ public function repair(): string } $this->state = self::STATE_EXPECTING_COMMA_OR_END; - // @phpstan-ignore identical.alwaysFalse (state can be STATE_IN_OBJECT_KEY for unquoted keys) } elseif ($this->state === self::STATE_IN_OBJECT_KEY) { - // We're still in key state - might have an incomplete unquoted key - // If output ends with a quote, we have a complete key, add colon and empty value if (str_ends_with($this->output, '"') && ! str_ends_with($this->output, ':""')) { if ($this->omitEmptyValues) { $this->removeCurrentKey(); @@ -298,12 +335,8 @@ public function repair(): string } } - // If we're in OBJECT_VALUE state and output ends with ':' (possibly with trailing space), add empty string - $trimmedForCheck = rtrim($this->output); - - // @phpstan-ignore booleanAnd.alwaysFalse, identical.alwaysFalse (state can change during loop) - if ($this->state === self::STATE_IN_OBJECT_VALUE && str_ends_with($trimmedForCheck, ':')) { - $this->output = $trimmedForCheck; + if ($this->state === self::STATE_IN_OBJECT_VALUE && $this->outputEndsWithNonWhitespace(':')) { + $this->trimOutputTrailingWhitespace(); if ($this->omitEmptyValues) { $this->removeCurrentKey(); @@ -314,20 +347,16 @@ public function repair(): string $this->state = self::STATE_EXPECTING_COMMA_OR_END; } - // Close any unclosed brackets/braces while ($this->stack !== []) { $expected = array_pop($this->stack); $this->log('Adding missing closing bracket/brace', [ 'char' => $expected, ]); - // Remove trailing comma before closing $this->removeTrailingComma(); - $trimmedForBrace = rtrim($this->output); - - if ($expected === '}' && str_ends_with($trimmedForBrace, ':')) { - $this->output = $trimmedForBrace; + if ($expected === '}' && $this->outputEndsWithNonWhitespace(':')) { + $this->trimOutputTrailingWhitespace(); if ($this->omitEmptyValues) { $this->removeCurrentKey(); @@ -337,39 +366,194 @@ public function repair(): string } $this->output .= $expected; + + if ($expected === '}') { + array_pop($this->objectKeysStack); + } } - if (! $this->ensureAscii) { + return $this->finalizeOutput(); + } + + private function finalizeOutput(): string + { + $encodedViaRoundTrip = false; + + if ($this->ensureAscii && preg_match('/[^\x00-\x7F]/', $this->output) === 1) { + $decoded = json_decode($this->output, true); + + if (json_last_error() === JSON_ERROR_NONE) { + $encoded = json_encode($decoded, JSON_UNESCAPED_SLASHES); + + if ($encoded !== false) { + $this->output = $encoded; + $encodedViaRoundTrip = true; + } + } + } elseif (! $this->ensureAscii && $this->output !== '') { $decoded = json_decode($this->output, true); - if ($decoded !== null) { + if (json_last_error() === JSON_ERROR_NONE) { $encoded = json_encode($decoded, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); if ($encoded !== false) { $this->output = $encoded; + $encodedViaRoundTrip = true; + } + } + } + + if ($this->duplicateKeyPolicy === DuplicateKeyPolicy::KeepLast && $this->output !== '') { + $decoded = json_decode($this->output, true); + + if (json_last_error() === JSON_ERROR_NONE) { + $flags = JSON_UNESCAPED_SLASHES; + + if (! $this->ensureAscii) { + $flags |= JSON_UNESCAPED_UNICODE; + } + + $encoded = json_encode($decoded, $flags); + + if ($encoded !== false) { + $this->output = $encoded; + $encodedViaRoundTrip = true; } } } - if ($this->output !== '' && ! json_validate($this->output)) { + if (! $encodedViaRoundTrip && $this->output !== '' && ! json_validate($this->output)) { throw JsonRepairException::invalidJsonAfterRepair($this->output); } return $this->output; } - /** - * @param int<1, max> $depth - * - * @return array|object - */ - public function decode( - int $depth = 512, - int $flags = JSON_THROW_ON_ERROR, - ): array|object { - $repaired = $this->repair(); - $decoded = json_decode($repaired, true, $depth, $flags); + private function pushObjectKeyScope(): void + { + $this->objectKeysStack[] = []; + } + + private function handleDuplicateKey(string $keyName): bool + { + if (! $this->duplicateKeyPolicy instanceof DuplicateKeyPolicy || $keyName === '' || $this->objectKeysStack === []) { + return false; + } + + $depth = count($this->objectKeysStack) - 1; + + if (! isset($this->objectKeysStack[$depth][$keyName])) { + $this->objectKeysStack[$depth][$keyName] = $this->currentKeyStart; + + return false; + } + + if ($this->duplicateKeyPolicy === DuplicateKeyPolicy::KeepFirst) { + $this->log('Skipping duplicate key (keep-first policy)', [ + 'key' => $keyName, + ]); + $this->removeCurrentKey(); + $this->skipNextValue = true; + + return true; + } + + $this->log('Replacing duplicate key (keep-last policy)', [ + 'key' => $keyName, + ]); + $previousStart = $this->objectKeysStack[$depth][$keyName]; + $this->removeKeyValueRegion($previousStart, $this->currentKeyStart); + $this->objectKeysStack[$depth][$keyName] = $this->currentKeyStart; + + return false; + } + + private function removeKeyValueRegion(int $keyStart, int $nextKeyStart): void + { + $before = substr($this->output, 0, $keyStart); + $before = rtrim($before); + + if (str_ends_with($before, ',')) { + $before = rtrim(substr($before, 0, -1)); + } + + $after = substr($this->output, $nextKeyStart); + $this->setOutput($before . $after); + } + + private function extractCompletedKeyName(): string + { + if ($this->currentKeyStart < 0) { + return ''; + } + + $segment = substr($this->output, $this->currentKeyStart); + + if (preg_match('/^"((?:[^"\\\\]|\\\\.)*)"/', $segment, $matches) !== 1) { + return ''; + } + + $decoded = json_decode('"' . $matches[1] . '"'); + + return is_string($decoded) ? $decoded : $matches[1]; + } + + private function skipValueAt(string $json, int $i): int + { + $length = strlen($json); + + while ($i < $length && ctype_space($json[$i])) { + $i++; + } + + if ($i >= $length) { + return $i; + } + + $char = $json[$i]; + + if ($char === '"' || $char === "'") { + $delimiter = $char; + $i++; + + while ($i < $length) { + if ($json[$i] === '\\') { + $i += 2; + continue; + } + + if ($json[$i] === $delimiter) { + return $i + 1; + } + + $i++; + } + + return $i; + } + + if ($char === '{' || $char === '[') { + $close = $char === '{' ? '}' : ']'; + $depth = 1; + $i++; + + while ($i < $length && $depth > 0) { + if ($json[$i] === $char) { + $depth++; + } elseif ($json[$i] === $close) { + $depth--; + } + + $i++; + } + + return $i; + } + + while ($i < $length && ! in_array($json[$i], [',', '}', ']'], true)) { + $i++; + } - return is_array($decoded) ? $decoded : (object) $decoded; + return $i; } } diff --git a/src/RepairResult.php b/src/RepairResult.php new file mode 100644 index 0000000..89aae0d --- /dev/null +++ b/src/RepairResult.php @@ -0,0 +1,17 @@ + $fixes + */ + public function __construct( + public string $json, + public bool $wasAlreadyValid, + public array $fixes, + ) {} +} diff --git a/src/StreamingJsonRepairer.php b/src/StreamingJsonRepairer.php new file mode 100644 index 0000000..4878b2c --- /dev/null +++ b/src/StreamingJsonRepairer.php @@ -0,0 +1,54 @@ +buffer .= $chunk; + + return $this; + } + + public function current(): string + { + $jsonRepairer = new JsonRepairer( + $this->buffer, + $this->ensureAscii, + $this->omitEmptyValues, + $this->omitIncompleteStrings, + $this->duplicateKeyPolicy, + ); + + if ($this->logger instanceof LoggerInterface) { + $jsonRepairer->setLogger($this->logger); + } + + return $jsonRepairer->repair(); + } + + public function reset(): self + { + $this->buffer = ''; + + return $this; + } +} diff --git a/src/functions.php b/src/functions.php index 54d8751..004ce71 100644 --- a/src/functions.php +++ b/src/functions.php @@ -44,7 +44,7 @@ function json_repair( * @param bool $omitIncompleteStrings Whether to remove keys with incomplete string values instead of closing them (default: false) * @param \Psr\Log\LoggerInterface|null $logger Optional PSR-3 logger for debugging repair actions * - * @return array|object The decoded JSON data + * @return mixed The decoded JSON data */ function json_repair_decode( string $json, @@ -54,7 +54,7 @@ function json_repair_decode( bool $omitEmptyValues = false, bool $omitIncompleteStrings = false, ?LoggerInterface $logger = null, -): array|object { +): mixed { $repairer = new JsonRepairer($json, $ensureAscii, $omitEmptyValues, $omitIncompleteStrings); if ($logger instanceof LoggerInterface) { diff --git a/tests/Datasets/JsonRepair.php b/tests/Datasets/JsonRepair.php index 5f5571a..be88c66 100644 --- a/tests/Datasets/JsonRepair.php +++ b/tests/Datasets/JsonRepair.php @@ -443,6 +443,9 @@ 'JSON false' => ['{"key": false}', '{"key": false}'], 'JSON null' => ['{"key": null}', '{"key": null}'], 'array with capitalized booleans' => ['[True, False, None]', '[true, false, null]'], + 'Infinity' => ['{"key": Infinity}', '{"key": null}'], + 'negative Infinity' => ['{"key": -Infinity}', '{"key": null}'], + 'NaN' => ['{"key": NaN}', '{"key": null}'], ]); dataset('standalone_booleans_null', [ @@ -462,6 +465,39 @@ 'large integer' => ['{"key": 12345678901234567890}', 'validate_only'], ]); +dataset('invalid_numbers', [ + 'leading plus sign' => [ + '{"key": +123}', + '{"key": 123}', + 123, + ], + 'leading zeros' => [ + '{"key": 007}', + '{"key": 7}', + 7, + ], + 'bare decimal' => [ + '{"key": .5}', + '{"key": 0.5}', + 0.5, + ], + 'trailing decimal point' => [ + '{"key": 5.}', + '{"key": 5}', + 5, + ], + 'leading zeros with decimal' => [ + '{"key": 007.5}', + '{"key": 7.5}', + 7.5, + ], + 'zero decimal preserved' => [ + '{"key": 0.5}', + '{"key": 0.5}', + 0.5, + ], +]); + dataset('empty_strings', [ 'incomplete empty string' => [ '{"key": ""', diff --git a/tests/Unit/JsonRepairerTest.php b/tests/Unit/JsonRepairerTest.php index ff50ab4..d7a991d 100644 --- a/tests/Unit/JsonRepairerTest.php +++ b/tests/Unit/JsonRepairerTest.php @@ -6,6 +6,8 @@ use Cortex\JsonRepair\JsonRepairer; use ColinODell\PsrTestLogger\TestLogger; +use Cortex\JsonRepair\DuplicateKeyPolicy; +use Cortex\JsonRepair\StreamingJsonRepairer; use function Cortex\JsonRepair\json_repair; use function Cortex\JsonRepair\json_repair_decode; @@ -164,6 +166,13 @@ expect(json_decode($result, true)['key'])->toBe($expected); })->with('numbers'); + it('repairs invalid numbers', function (string $input, string $expectedJson, int|float $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expectedJson); + expect(json_decode($result, true)['key'])->toBe($expected); + })->with('invalid_numbers'); + it( 'handles strings with special characters', function (string $input, string $expectedKey, string $expectedValue): void { @@ -192,6 +201,15 @@ function (string $input, string $expectedKey, string $expectedValue): void { expect($decoded)->toBe(json_decode($expected, true)); })->with('advanced_escaping'); + it('escapes unicode characters when ensureAscii is true', function (): void { + $result = json_repair("{'city':'上海'}"); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe('{"city":"\\u4e0a\\u6d77"}'); + + $decoded = json_decode($result, true); + expect($decoded['city'])->toBe('上海'); + }); + it('handles unicode characters when ensureAscii is false', function (): void { $input = "{'test_中国人_ascii':'统一码'}"; $result = json_repair($input, ensureAscii: false); @@ -604,3 +622,100 @@ function (string $input, string $expected): void { expect($result)->toBe('{"key": "value"}'); }); }); + +describe('RepairResult', function (): void { + it('returns structured repair details', function (): void { + $repairer = new JsonRepairer("{'key': 'value'}"); + $repairResult = $repairer->repairWithDetails(); + + expect($repairResult->wasAlreadyValid)->toBeFalse(); + expect($repairResult->json)->toBe('{"key": "value"}'); + expect($repairResult->fixes)->toContain('Starting JSON repair'); + }); + + it('marks valid JSON as already valid', function (): void { + $repairResult = (new JsonRepairer('{"key": "value"}'))->repairWithDetails(); + + expect($repairResult->wasAlreadyValid)->toBeTrue(); + expect($repairResult->json)->toBe('{"key": "value"}'); + expect($repairResult->fixes)->toContain('JSON is already valid, returning as-is'); + }); +}); + +describe('repairAll', function (): void { + it('repairs multiple top-level JSON values', function (): void { + $repairer = new JsonRepairer('{"a":1}{"b":2}'); + $results = $repairer->repairAll(); + + expect($results)->toHaveCount(2); + expect(json_decode($results[0], true))->toBe([ + 'a' => 1, + ]); + expect(json_decode($results[1], true))->toBe([ + 'b' => 2, + ]); + }); + + it('repairs NDJSON lines', function (): void { + $repairer = new JsonRepairer("{'a':1}\n{'b':2}"); + $results = $repairer->repairAll(); + + expect($results)->toHaveCount(2); + expect(json_decode($results[0], true))->toBe([ + 'a' => 1, + ]); + expect(json_decode($results[1], true))->toBe([ + 'b' => 2, + ]); + }); +}); + +describe('StreamingJsonRepairer', function (): void { + it('repairs incrementally fed chunks', function (): void { + $stream = new StreamingJsonRepairer(); + $stream->feed('{"key": '); + $stream->feed('"val'); + + $result = $stream->current(); + + expect(json_validate($result))->toBeTrue(); + expect(json_decode($result, true))->toBe([ + 'key' => 'val', + ]); + }); +}); + +describe('Duplicate key policy', function (): void { + it('keeps first duplicate key when configured', function (): void { + $repairer = new JsonRepairer('{a: 1, a: 2}', duplicateKeyPolicy: DuplicateKeyPolicy::KeepFirst); + $result = $repairer->repair(); + + expect(json_decode($result, true))->toBe([ + 'a' => 1, + ]); + }); + + it('keeps last duplicate key when configured', function (): void { + $repairer = new JsonRepairer('{a: 1, a: 2}', duplicateKeyPolicy: DuplicateKeyPolicy::KeepLast); + $result = $repairer->repair(); + + expect(json_decode($result, true))->toBe([ + 'a' => 2, + ]); + }); +}); + +describe('Scalar decode', function (): void { + it('decodes top-level scalar values', function (): void { + $repairer = new JsonRepairer('true'); + expect($repairer->decode())->toBeTrue(); + }); + + it('decodes top-level null via json_repair_decode', function (): void { + expect(json_repair_decode('null'))->toBeNull(); + }); + + it('decodes top-level string scalars', function (): void { + expect((new JsonRepairer('"hello"'))->decode())->toBe('hello'); + }); +}); From f1589c5c6a7d255d0cf9770c3914c93f743b1fbd Mon Sep 17 00:00:00 2001 From: Sean Tymon Date: Wed, 24 Jun 2026 09:02:58 +0100 Subject: [PATCH 02/12] tweaks --- src/Concerns/StateMachine.php | 74 +++++++++++++++--------------- src/Concerns/StringHeuristics.php | 4 +- src/JsonRepairer.php | 76 ++++++++++++------------------- src/ParserState.php | 32 +++++++++++++ 4 files changed, 102 insertions(+), 84 deletions(-) create mode 100644 src/ParserState.php diff --git a/src/Concerns/StateMachine.php b/src/Concerns/StateMachine.php index aa57cbd..d27f623 100644 --- a/src/Concerns/StateMachine.php +++ b/src/Concerns/StateMachine.php @@ -4,6 +4,8 @@ namespace Cortex\JsonRepair\Concerns; +use Cortex\JsonRepair\ParserState; + /** * @mixin \Cortex\JsonRepair\JsonRepairer */ @@ -71,7 +73,7 @@ private function tryOpenObjectOrArray(string $json, int $i, bool $resetKeyTracki $this->output .= '{'; $this->stack[] = '}'; $this->pushObjectKeyScope(); - $this->state = self::STATE_IN_OBJECT_KEY; + $this->state = ParserState::STATE_IN_OBJECT_KEY; if ($resetKeyTracking) { $this->currentKeyStart = -1; @@ -83,7 +85,7 @@ private function tryOpenObjectOrArray(string $json, int $i, bool $resetKeyTracki if ($char === '[') { $this->output .= '['; $this->stack[] = ']'; - $this->state = self::STATE_IN_ARRAY; + $this->state = ParserState::STATE_IN_ARRAY; if ($resetKeyTracking) { $this->currentKeyStart = -1; @@ -132,7 +134,7 @@ private function handleObjectKey(string $json, int $i): int array_pop($this->objectKeysStack); } - $this->state = $this->stack === [] ? self::STATE_START : self::STATE_EXPECTING_COMMA_OR_END; + $this->state = $this->stack === [] ? ParserState::STATE_START : ParserState::STATE_EXPECTING_COMMA_OR_END; return $i + 1; } @@ -175,7 +177,7 @@ private function handleObjectKey(string $json, int $i): int } $this->output .= '"'; - $this->state = self::STATE_EXPECTING_COLON; + $this->state = ParserState::STATE_EXPECTING_COLON; // Skip past the closing "" if present if ($keyEnd + 1 < $length && ($json[$keyEnd] === '"' || $json[$keyEnd] === "'") && $json[$keyEnd + 1] === $json[$keyEnd]) { @@ -200,8 +202,8 @@ private function handleObjectKey(string $json, int $i): int $this->output .= '"'; $this->inString = true; $this->stringDelimiter = $char; - $this->stateBeforeString = self::STATE_IN_OBJECT_KEY; - $this->state = self::STATE_IN_STRING; + $this->stateBeforeString = ParserState::STATE_IN_OBJECT_KEY; + $this->state = ParserState::STATE_IN_STRING; return $i + 1; } @@ -215,8 +217,8 @@ private function handleObjectKey(string $json, int $i): int $this->output .= '"'; $this->inString = true; $this->stringDelimiter = '"'; // Normalize to regular quote - $this->stateBeforeString = self::STATE_IN_OBJECT_KEY; - $this->state = self::STATE_IN_STRING; + $this->stateBeforeString = ParserState::STATE_IN_OBJECT_KEY; + $this->state = ParserState::STATE_IN_STRING; return $i + $smartQuoteLength; } @@ -233,7 +235,7 @@ private function handleObjectKey(string $json, int $i): int $this->output .= substr($json, $keyStart, $i - $keyStart); $this->output .= '"'; - $this->state = self::STATE_EXPECTING_COLON; + $this->state = ParserState::STATE_EXPECTING_COLON; return $i; } @@ -259,13 +261,13 @@ private function handleExpectingColon(string $json, int $i): int if ($char === ':') { if ($this->handleDuplicateKey($this->extractCompletedKeyName())) { - $this->state = self::STATE_IN_OBJECT_VALUE; + $this->state = ParserState::STATE_IN_OBJECT_VALUE; return $this->skipValueAt($json, $i + 1); } $this->output .= ':'; - $this->state = self::STATE_IN_OBJECT_VALUE; + $this->state = ParserState::STATE_IN_OBJECT_VALUE; // Preserve whitespace after colon $nextI = $i + 1; @@ -280,14 +282,14 @@ private function handleExpectingColon(string $json, int $i): int // Missing colon, insert it if (! ctype_space($char)) { if ($this->handleDuplicateKey($this->extractCompletedKeyName())) { - $this->state = self::STATE_IN_OBJECT_VALUE; + $this->state = ParserState::STATE_IN_OBJECT_VALUE; return $this->skipValueAt($json, $i); } $this->log('Inserting missing colon after key'); $this->output .= ':'; - $this->state = self::STATE_IN_OBJECT_VALUE; + $this->state = ParserState::STATE_IN_OBJECT_VALUE; return $i; } @@ -330,8 +332,8 @@ private function handleObjectValue(string $json, int $i): int $this->output .= '"'; $this->inString = true; $this->stringDelimiter = $char; - $this->stateBeforeString = self::STATE_IN_OBJECT_VALUE; - $this->state = self::STATE_IN_STRING; + $this->stateBeforeString = ParserState::STATE_IN_OBJECT_VALUE; + $this->state = ParserState::STATE_IN_STRING; return $i + 1; } @@ -358,7 +360,7 @@ private function handleObjectValue(string $json, int $i): int array_pop($this->objectKeysStack); } - $this->state = $this->stack === [] ? self::STATE_START : self::STATE_EXPECTING_COMMA_OR_END; + $this->state = $this->stack === [] ? ParserState::STATE_START : ParserState::STATE_EXPECTING_COMMA_OR_END; return $i + 1; } @@ -379,7 +381,7 @@ private function handleObjectValue(string $json, int $i): int } $this->output .= $normalized; - $this->state = self::STATE_EXPECTING_COMMA_OR_END; + $this->state = ParserState::STATE_EXPECTING_COMMA_OR_END; // Reset key tracking after successfully completing a boolean/null value $this->currentKeyStart = -1; @@ -388,7 +390,7 @@ private function handleObjectValue(string $json, int $i): int // Handle numbers if (ctype_digit($char) || $char === '-' || $char === '+' || $char === '.') { - $this->state = self::STATE_IN_NUMBER; + $this->state = ParserState::STATE_IN_NUMBER; return $i; } @@ -403,7 +405,7 @@ private function handleObjectValue(string $json, int $i): int $this->output .= '""'; } - $this->state = self::STATE_EXPECTING_COMMA_OR_END; + $this->state = ParserState::STATE_EXPECTING_COMMA_OR_END; return $i; } @@ -415,8 +417,8 @@ private function handleObjectValue(string $json, int $i): int $this->output .= '"'; $this->inString = true; $this->stringDelimiter = '"'; - $this->stateBeforeString = self::STATE_IN_OBJECT_VALUE; - $this->state = self::STATE_IN_STRING; + $this->stateBeforeString = ParserState::STATE_IN_OBJECT_VALUE; + $this->state = ParserState::STATE_IN_STRING; return $i + $smartQuoteLength; } @@ -451,7 +453,7 @@ private function handleArrayValue(string $json, int $i): int $this->removeTrailingComma(); $this->output .= ']'; array_pop($this->stack); - $this->state = $this->stack === [] ? self::STATE_START : self::STATE_EXPECTING_COMMA_OR_END; + $this->state = $this->stack === [] ? ParserState::STATE_START : ParserState::STATE_EXPECTING_COMMA_OR_END; return $i + 1; } @@ -466,8 +468,8 @@ private function handleArrayValue(string $json, int $i): int $this->output .= '"'; $this->inString = true; $this->stringDelimiter = $char; - $this->stateBeforeString = self::STATE_IN_ARRAY; - $this->state = self::STATE_IN_STRING; + $this->stateBeforeString = ParserState::STATE_IN_ARRAY; + $this->state = ParserState::STATE_IN_STRING; return $i + 1; } @@ -479,14 +481,14 @@ private function handleArrayValue(string $json, int $i): int $normalized = $keywordMatch[0]; $klen = $keywordMatch[1]; $this->output .= $normalized; - $this->state = self::STATE_EXPECTING_COMMA_OR_END; + $this->state = ParserState::STATE_EXPECTING_COMMA_OR_END; return $i + $klen; } // Handle numbers if (ctype_digit($char) || $char === '-' || $char === '+' || $char === '.') { - $this->state = self::STATE_IN_NUMBER; + $this->state = ParserState::STATE_IN_NUMBER; return $i; } @@ -519,14 +521,14 @@ private function handleExpectingCommaOrEnd(string $json, int $i): int array_pop($this->objectKeysStack); } - $this->state = $this->stack === [] ? self::STATE_START : self::STATE_EXPECTING_COMMA_OR_END; + $this->state = $this->stack === [] ? ParserState::STATE_START : ParserState::STATE_EXPECTING_COMMA_OR_END; return $i + 1; } if ($char === ',') { $this->output .= ','; - $this->state = $top === '}' ? self::STATE_IN_OBJECT_KEY : self::STATE_IN_ARRAY; + $this->state = $top === '}' ? ParserState::STATE_IN_OBJECT_KEY : ParserState::STATE_IN_ARRAY; // Preserve whitespace after comma $nextI = $i + 1; @@ -543,7 +545,7 @@ private function handleExpectingCommaOrEnd(string $json, int $i): int if (! ctype_space($char) && $char !== $top) { $this->log('Inserting missing comma'); $this->output .= ','; - $this->state = $top === '}' ? self::STATE_IN_OBJECT_KEY : self::STATE_IN_ARRAY; + $this->state = $top === '}' ? ParserState::STATE_IN_OBJECT_KEY : ParserState::STATE_IN_ARRAY; return $i; } @@ -644,7 +646,7 @@ private function handleNumber(string $json, int $i): int } } - $this->state = self::STATE_EXPECTING_COMMA_OR_END; + $this->state = ParserState::STATE_EXPECTING_COMMA_OR_END; $this->currentKeyStart = -1; return $i; @@ -681,11 +683,11 @@ private function handleEscapeSequence(string $char, string $json): int */ private function getNextStateAfterString(): int { - if ($this->stateBeforeString === self::STATE_IN_OBJECT_KEY) { - return self::STATE_EXPECTING_COLON; + if ($this->stateBeforeString === ParserState::STATE_IN_OBJECT_KEY) { + return ParserState::STATE_EXPECTING_COLON; } - return self::STATE_EXPECTING_COMMA_OR_END; + return ParserState::STATE_EXPECTING_COMMA_OR_END; } /** @@ -794,7 +796,7 @@ private function handleUnquotedStringValue(string $json, int $i): int $this->output .= '""'; } - $this->state = self::STATE_EXPECTING_COMMA_OR_END; + $this->state = ParserState::STATE_EXPECTING_COMMA_OR_END; $this->currentKeyStart = -1; return $i; @@ -810,7 +812,7 @@ private function handleUnquotedStringValue(string $json, int $i): int $this->currentKeyStart = -1; // Insert a comma before the new key and set state to expect the key $this->output .= ', '; - $this->state = self::STATE_IN_OBJECT_KEY; + $this->state = ParserState::STATE_IN_OBJECT_KEY; return $i; } @@ -818,7 +820,7 @@ private function handleUnquotedStringValue(string $json, int $i): int // Output the unquoted value as a quoted string if ($value !== '') { $this->output .= '"' . $this->escapeStringValue($value) . '"'; - $this->state = self::STATE_EXPECTING_COMMA_OR_END; + $this->state = ParserState::STATE_EXPECTING_COMMA_OR_END; $this->currentKeyStart = -1; } diff --git a/src/Concerns/StringHeuristics.php b/src/Concerns/StringHeuristics.php index bbca5ce..3be21df 100644 --- a/src/Concerns/StringHeuristics.php +++ b/src/Concerns/StringHeuristics.php @@ -4,6 +4,8 @@ namespace Cortex\JsonRepair\Concerns; +use Cortex\JsonRepair\ParserState; + /** * @mixin \Cortex\JsonRepair\JsonRepairer */ @@ -62,7 +64,7 @@ private function shouldEscapeQuoteInValue(string $json, int $quotePos): bool { // Only apply quote escaping logic for object values, not arrays // In arrays, quotes typically delimit separate values - if ($this->stateBeforeString === self::STATE_IN_ARRAY) { + if ($this->stateBeforeString === ParserState::STATE_IN_ARRAY) { return false; } diff --git a/src/JsonRepairer.php b/src/JsonRepairer.php index ce4aa28..d0f7aeb 100644 --- a/src/JsonRepairer.php +++ b/src/JsonRepairer.php @@ -23,25 +23,7 @@ class JsonRepairer implements LoggerAwareInterface use InputSanitization; use OutputTracking; - private const int STATE_START = 0; - - private const int STATE_IN_STRING = 1; - - private const int STATE_IN_STRING_ESCAPE = 2; - - private const int STATE_IN_NUMBER = 3; - - private const int STATE_IN_OBJECT_KEY = 4; - - private const int STATE_IN_OBJECT_VALUE = 5; - - private const int STATE_IN_ARRAY = 6; - - private const int STATE_EXPECTING_COLON = 7; - - private const int STATE_EXPECTING_COMMA_OR_END = 8; - - private int $state = self::STATE_START; + private int $state = ParserState::STATE_START; private int $pos = 0; @@ -56,7 +38,7 @@ class JsonRepairer implements LoggerAwareInterface private string $stringDelimiter = ''; - private int $stateBeforeString = self::STATE_START; + private int $stateBeforeString = ParserState::STATE_START; private int $currentKeyStart = -1; @@ -175,13 +157,13 @@ private function repairInternal(bool $extractFirstOnly): string $json = $this->extractFirstValidJson($json); } - $this->state = self::STATE_START; + $this->state = ParserState::STATE_START; $this->pos = 0; $this->output = ''; $this->stack = []; $this->inString = false; $this->stringDelimiter = ''; - $this->stateBeforeString = self::STATE_START; + $this->stateBeforeString = ParserState::STATE_START; $this->currentKeyStart = -1; $this->objectKeysStack = []; $this->skipNextValue = false; @@ -194,19 +176,19 @@ private function repairInternal(bool $extractFirstOnly): string $char = $json[$i]; $this->pos = $i; - if ($this->state === self::STATE_IN_STRING_ESCAPE) { + if ($this->state === ParserState::STATE_IN_STRING_ESCAPE) { if ($i >= strlen($json)) { - $this->state = self::STATE_IN_STRING; + $this->state = ParserState::STATE_IN_STRING; break; } $extraCharsConsumed = $this->handleEscapeSequence($char, $json); - $this->state = self::STATE_IN_STRING; + $this->state = ParserState::STATE_IN_STRING; $i += 1 + $extraCharsConsumed; continue; } - if ($this->state === self::STATE_IN_STRING) { + if ($this->state === ParserState::STATE_IN_STRING) { $smartQuoteLength = $char === "\xE2" ? $this->getSmartQuoteLength($json, $i) : 0; if ($char === '"' && $this->stringDelimiter === "'") { @@ -218,8 +200,8 @@ private function repairInternal(bool $extractFirstOnly): string if ($char === $this->stringDelimiter || $smartQuoteLength > 0) { $isRegularQuote = $smartQuoteLength === 0; - $isInValue = $this->stateBeforeString === self::STATE_IN_OBJECT_VALUE - || $this->stateBeforeString === self::STATE_IN_ARRAY; + $isInValue = $this->stateBeforeString === ParserState::STATE_IN_OBJECT_VALUE + || $this->stateBeforeString === ParserState::STATE_IN_ARRAY; if ($isRegularQuote && $isInValue && $this->shouldEscapeQuoteInValue($json, $i)) { $this->log('Escaping embedded quote inside string value'); @@ -233,7 +215,7 @@ private function repairInternal(bool $extractFirstOnly): string $this->stringDelimiter = ''; $this->state = $this->getNextStateAfterString(); - if ($this->state === self::STATE_EXPECTING_COMMA_OR_END) { + if ($this->state === ParserState::STATE_EXPECTING_COMMA_OR_END) { $this->currentKeyStart = -1; } @@ -242,7 +224,7 @@ private function repairInternal(bool $extractFirstOnly): string } if ($char === '\\') { - $this->state = self::STATE_IN_STRING_ESCAPE; + $this->state = ParserState::STATE_IN_STRING_ESCAPE; $i++; continue; } @@ -256,7 +238,7 @@ private function repairInternal(bool $extractFirstOnly): string $this->stringDelimiter = ''; $this->state = $this->getNextStateAfterString(); - if ($this->state === self::STATE_EXPECTING_COMMA_OR_END) { + if ($this->state === ParserState::STATE_EXPECTING_COMMA_OR_END) { $this->currentKeyStart = -1; } @@ -282,30 +264,30 @@ private function repairInternal(bool $extractFirstOnly): string continue; } - if ($this->skipNextValue && ($this->state === self::STATE_IN_OBJECT_VALUE || $this->state === self::STATE_IN_NUMBER)) { + if ($this->skipNextValue && ($this->state === ParserState::STATE_IN_OBJECT_VALUE || $this->state === ParserState::STATE_IN_NUMBER)) { $i = $this->skipValueAt($json, $i); $this->skipNextValue = false; - $this->state = self::STATE_EXPECTING_COMMA_OR_END; + $this->state = ParserState::STATE_EXPECTING_COMMA_OR_END; continue; } $i = match ($this->state) { - self::STATE_START => $this->handleStart($json, $i), - self::STATE_IN_OBJECT_KEY => $this->handleObjectKey($json, $i), - self::STATE_EXPECTING_COLON => $this->handleExpectingColon($json, $i), - self::STATE_IN_OBJECT_VALUE => $this->handleObjectValue($json, $i), - self::STATE_IN_ARRAY => $this->handleArrayValue($json, $i), - self::STATE_EXPECTING_COMMA_OR_END => $this->handleExpectingCommaOrEnd($json, $i), - self::STATE_IN_NUMBER => $this->handleNumber($json, $i), + ParserState::STATE_START => $this->handleStart($json, $i), + ParserState::STATE_IN_OBJECT_KEY => $this->handleObjectKey($json, $i), + ParserState::STATE_EXPECTING_COLON => $this->handleExpectingColon($json, $i), + ParserState::STATE_IN_OBJECT_VALUE => $this->handleObjectValue($json, $i), + ParserState::STATE_IN_ARRAY => $this->handleArrayValue($json, $i), + ParserState::STATE_EXPECTING_COMMA_OR_END => $this->handleExpectingCommaOrEnd($json, $i), + ParserState::STATE_IN_NUMBER => $this->handleNumber($json, $i), default => $i + 1, }; } if ($this->inString) { - if ($this->omitIncompleteStrings && $this->stateBeforeString === self::STATE_IN_OBJECT_VALUE) { + if ($this->omitIncompleteStrings && $this->stateBeforeString === ParserState::STATE_IN_OBJECT_VALUE) { $this->log('Removing incomplete string value (omitIncompleteStrings enabled)'); $this->removeCurrentKey(); - $this->state = self::STATE_EXPECTING_COMMA_OR_END; + $this->state = ParserState::STATE_EXPECTING_COMMA_OR_END; } else { $this->log('Adding missing closing quote for unclosed string'); $this->output .= '"'; @@ -315,7 +297,7 @@ private function repairInternal(bool $extractFirstOnly): string $this->inString = false; } - if ($this->state === self::STATE_EXPECTING_COLON) { + if ($this->state === ParserState::STATE_EXPECTING_COLON) { if ($this->omitEmptyValues) { $this->log('Removing key without value (omitEmptyValues enabled)'); $this->removeCurrentKey(); @@ -324,8 +306,8 @@ private function repairInternal(bool $extractFirstOnly): string $this->output .= ':""'; } - $this->state = self::STATE_EXPECTING_COMMA_OR_END; - } elseif ($this->state === self::STATE_IN_OBJECT_KEY) { + $this->state = ParserState::STATE_EXPECTING_COMMA_OR_END; + } elseif ($this->state === ParserState::STATE_IN_OBJECT_KEY) { if (str_ends_with($this->output, '"') && ! str_ends_with($this->output, ':""')) { if ($this->omitEmptyValues) { $this->removeCurrentKey(); @@ -335,7 +317,7 @@ private function repairInternal(bool $extractFirstOnly): string } } - if ($this->state === self::STATE_IN_OBJECT_VALUE && $this->outputEndsWithNonWhitespace(':')) { + if ($this->state === ParserState::STATE_IN_OBJECT_VALUE && $this->outputEndsWithNonWhitespace(':')) { $this->trimOutputTrailingWhitespace(); if ($this->omitEmptyValues) { @@ -344,7 +326,7 @@ private function repairInternal(bool $extractFirstOnly): string $this->output .= '""'; } - $this->state = self::STATE_EXPECTING_COMMA_OR_END; + $this->state = ParserState::STATE_EXPECTING_COMMA_OR_END; } while ($this->stack !== []) { diff --git a/src/ParserState.php b/src/ParserState.php new file mode 100644 index 0000000..bff3503 --- /dev/null +++ b/src/ParserState.php @@ -0,0 +1,32 @@ + Date: Wed, 24 Jun 2026 09:05:22 +0100 Subject: [PATCH 03/12] add param --- src/functions.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/functions.php b/src/functions.php index 004ce71..0055331 100644 --- a/src/functions.php +++ b/src/functions.php @@ -13,6 +13,7 @@ * @param bool $ensureAscii Whether to escape non-ASCII characters (default: true) * @param bool $omitEmptyValues Whether to remove keys with missing values instead of adding empty strings (default: false) * @param bool $omitIncompleteStrings Whether to remove keys with incomplete string values instead of closing them (default: false) + * @param \Cortex\JsonRepair\DuplicateKeyPolicy|null $duplicateKeyPolicy How to handle duplicate object keys (default: null — no deduplication) * @param \Psr\Log\LoggerInterface|null $logger Optional PSR-3 logger for debugging repair actions * * @return string The repaired JSON string @@ -22,9 +23,10 @@ function json_repair( bool $ensureAscii = true, bool $omitEmptyValues = false, bool $omitIncompleteStrings = false, + ?DuplicateKeyPolicy $duplicateKeyPolicy = null, ?LoggerInterface $logger = null, ): string { - $repairer = new JsonRepairer($json, $ensureAscii, $omitEmptyValues, $omitIncompleteStrings); + $repairer = new JsonRepairer($json, $ensureAscii, $omitEmptyValues, $omitIncompleteStrings, $duplicateKeyPolicy); if ($logger instanceof LoggerInterface) { $repairer->setLogger($logger); @@ -42,6 +44,7 @@ function json_repair( * @param bool $ensureAscii Whether to escape non-ASCII characters (default: true) * @param bool $omitEmptyValues Whether to remove keys with missing values instead of adding empty strings (default: false) * @param bool $omitIncompleteStrings Whether to remove keys with incomplete string values instead of closing them (default: false) + * @param \Cortex\JsonRepair\DuplicateKeyPolicy|null $duplicateKeyPolicy How to handle duplicate object keys (default: null — no deduplication) * @param \Psr\Log\LoggerInterface|null $logger Optional PSR-3 logger for debugging repair actions * * @return mixed The decoded JSON data @@ -53,9 +56,10 @@ function json_repair_decode( bool $ensureAscii = true, bool $omitEmptyValues = false, bool $omitIncompleteStrings = false, + ?DuplicateKeyPolicy $duplicateKeyPolicy = null, ?LoggerInterface $logger = null, ): mixed { - $repairer = new JsonRepairer($json, $ensureAscii, $omitEmptyValues, $omitIncompleteStrings); + $repairer = new JsonRepairer($json, $ensureAscii, $omitEmptyValues, $omitIncompleteStrings, $duplicateKeyPolicy); if ($logger instanceof LoggerInterface) { $repairer->setLogger($logger); From a4d6ab8d01771e4ed913595649846d22217c83ea Mon Sep 17 00:00:00 2001 From: Sean Tymon Date: Thu, 25 Jun 2026 08:34:13 +0100 Subject: [PATCH 04/12] spli tests --- tests/Datasets/JsonRepair.php | 4 + tests/Unit/ApiUsageTest.php | 142 +++++ tests/Unit/EdgeCasesTest.php | 125 ++++ tests/Unit/JsonRepairerTest.php | 721 ----------------------- tests/Unit/JsonRepairsTest.php | 133 +++++ tests/Unit/LoggingTest.php | 117 ++++ tests/Unit/OptionsTest.php | 111 ++++ tests/Unit/StreamingJsonRepairerTest.php | 24 + tests/Unit/ValuesAndStructuresTest.php | 131 ++++ 9 files changed, 787 insertions(+), 721 deletions(-) create mode 100644 tests/Unit/ApiUsageTest.php create mode 100644 tests/Unit/EdgeCasesTest.php delete mode 100644 tests/Unit/JsonRepairerTest.php create mode 100644 tests/Unit/JsonRepairsTest.php create mode 100644 tests/Unit/LoggingTest.php create mode 100644 tests/Unit/OptionsTest.php create mode 100644 tests/Unit/StreamingJsonRepairerTest.php create mode 100644 tests/Unit/ValuesAndStructuresTest.php diff --git a/tests/Datasets/JsonRepair.php b/tests/Datasets/JsonRepair.php index be88c66..c9a6f0e 100644 --- a/tests/Datasets/JsonRepair.php +++ b/tests/Datasets/JsonRepair.php @@ -328,6 +328,10 @@ ```', '{"key": "value"}', ], + 'surrounding text without backticks' => [ + 'Here is the JSON: { "foo": "bar" } The next 64 elements are:', + '{"foo": "bar"}', + ], ]); dataset('json_in_strings', [ diff --git a/tests/Unit/ApiUsageTest.php b/tests/Unit/ApiUsageTest.php new file mode 100644 index 0000000..3f01cca --- /dev/null +++ b/tests/Unit/ApiUsageTest.php @@ -0,0 +1,142 @@ +repair(); + expect(json_validate($result))->toBeTrue(); + expect(json_decode($result, true)['key'])->toBe('value'); + }); + + it('can decode repaired JSON', function (): void { + $repairer = new JsonRepairer("{'key': 'value', 'number': 123}"); + $decoded = $repairer->decode(); + + expect($decoded)->toBeArray(); + expect($decoded['key'])->toBe('value'); + expect($decoded['number'])->toBe(123); + }); + + it('can use json_repair_decode helper function', function (): void { + $decoded = json_repair_decode("{'key': 'value', 'number': 123}"); + + expect($decoded)->toBeArray(); + expect($decoded['key'])->toBe('value'); + expect($decoded['number'])->toBe(123); + }); + + it('works with JsonRepairer class directly with omitEmptyValues', function (): void { + $repairer = new JsonRepairer('{"key": }', omitEmptyValues: true); + $result = $repairer->repair(); + expect(json_validate($result))->toBeTrue(); + $decoded = json_decode($result, true); + expect($decoded)->toBe([]); + }); + + it('works with JsonRepairer class directly with omitIncompleteStrings', function (): void { + $repairer = new JsonRepairer('{"key": "val', omitIncompleteStrings: true); + $result = $repairer->repair(); + expect(json_validate($result))->toBeTrue(); + $decoded = json_decode($result, true); + expect($decoded)->toBe([]); + }); + + it('works with json_repair_decode with new options', function (): void { + $decoded = json_repair_decode('{"key": }', omitEmptyValues: true); + expect($decoded)->toBeArray(); + expect($decoded)->toBe([]); + }); +}); + +describe('RepairResult', function (): void { + it('returns structured repair details', function (): void { + $repairer = new JsonRepairer("{'key': 'value'}"); + $repairResult = $repairer->repairWithDetails(); + + expect($repairResult->wasAlreadyValid)->toBeFalse(); + expect($repairResult->json)->toBe('{"key": "value"}'); + expect($repairResult->fixes)->toContain('Starting JSON repair'); + }); + + it('marks valid JSON as already valid', function (): void { + $repairResult = (new JsonRepairer('{"key": "value"}'))->repairWithDetails(); + + expect($repairResult->wasAlreadyValid)->toBeTrue(); + expect($repairResult->json)->toBe('{"key": "value"}'); + expect($repairResult->fixes)->toContain('JSON is already valid, returning as-is'); + }); +}); + +describe('repairAll', function (): void { + it('repairs multiple top-level JSON values', function (): void { + $repairer = new JsonRepairer('{"a":1}{"b":2}'); + $results = $repairer->repairAll(); + + expect($results)->toHaveCount(2); + expect(json_decode($results[0], true))->toBe([ + 'a' => 1, + ]); + expect(json_decode($results[1], true))->toBe([ + 'b' => 2, + ]); + }); + + it('repairs NDJSON lines', function (): void { + $repairer = new JsonRepairer("{'a':1}\n{'b':2}"); + $results = $repairer->repairAll(); + + expect($results)->toHaveCount(2); + expect(json_decode($results[0], true))->toBe([ + 'a' => 1, + ]); + expect(json_decode($results[1], true))->toBe([ + 'b' => 2, + ]); + }); +}); + +describe('Duplicate key policy', function (): void { + it('keeps first duplicate key when configured', function (): void { + $repairer = new JsonRepairer('{a: 1, a: 2}', duplicateKeyPolicy: DuplicateKeyPolicy::KeepFirst); + $result = $repairer->repair(); + + expect(json_decode($result, true))->toBe([ + 'a' => 1, + ]); + }); + + it('keeps last duplicate key when configured', function (): void { + $repairer = new JsonRepairer('{a: 1, a: 2}', duplicateKeyPolicy: DuplicateKeyPolicy::KeepLast); + $result = $repairer->repair(); + + expect(json_decode($result, true))->toBe([ + 'a' => 2, + ]); + }); +}); + +describe('Scalar decode', function (): void { + it('decodes top-level scalar values', function (): void { + $repairer = new JsonRepairer('true'); + expect($repairer->decode())->toBeTrue(); + }); + + it('decodes top-level null via json_repair_decode', function (): void { + expect(json_repair_decode('null'))->toBeNull(); + }); + + it('decodes top-level string scalars', function (): void { + expect((new JsonRepairer('"hello"'))->decode())->toBe('hello'); + }); +}); diff --git a/tests/Unit/EdgeCasesTest.php b/tests/Unit/EdgeCasesTest.php new file mode 100644 index 0000000..3960a08 --- /dev/null +++ b/tests/Unit/EdgeCasesTest.php @@ -0,0 +1,125 @@ +toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + })->with('incomplete_json'); + + it('repairs incomplete JSON from streaming LLM responses', function (string $input, string $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + })->with('streaming_llm_responses'); + + it('handles complex nested structures', function (): void { + $input = '{"resourceType": "Bundle", "id": "1", "type": "collection", "entry": [{"resource": {"resourceType": "Patient", "id": "1", "name": [{"use": "official", "family": "Corwin", "given": ["Keisha", "Sunny"], "prefix": ["Mrs."}, {"use": "maiden", "family": "Goodwin", "given": ["Keisha", "Sunny"], "prefix": ["Mrs."]}]}}]}'; + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + + $decoded = json_decode($result, true); + expect($decoded)->toBeArray(); + expect($decoded['resourceType'])->toBe('Bundle'); + expect($decoded['id'])->toBe('1'); + expect($decoded['type'])->toBe('collection'); + expect($decoded['entry'])->toBeArray(); + expect($decoded['entry'][0]['resource']['resourceType'])->toBe('Patient'); + expect($decoded['entry'][0]['resource']['name'])->toBeArray(); + expect($decoded['entry'][0]['resource']['name'])->toHaveCount(1); + expect($decoded['entry'][0]['resource']['name'][0]['use'])->toBe('official'); + expect($decoded['entry'][0]['resource']['name'][0]['family'])->toBe('Corwin'); + expect($decoded['entry'][0]['resource']['name'][0]['given'])->toBe(['Keisha', 'Sunny']); + expect($decoded['entry'][0]['resource']['name'][0]['prefix'][0])->toBe('Mrs.'); + expect($decoded['entry'][0]['resource']['name'][0]['prefix'][1])->toBeArray(); + expect($decoded['entry'][0]['resource']['name'][0]['prefix'][1]['use'])->toBe('maiden'); + expect($decoded['entry'][0]['resource']['name'][0]['prefix'][1]['family'])->toBe('Goodwin'); + }); + + it('handles multiple JSON objects', function (string $input, ?string $expectedKey, ?string $expectedValue): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + + if ($expectedKey !== null) { + $decoded = json_decode($result, true); + + if ($expectedValue !== null) { + expect($decoded[$expectedKey])->toBe($expectedValue); + } else { + expect($decoded)->toBeArray(); + } + } + })->with('multiple_json_objects'); + + it( + 'extracts JSON from markdown code blocks', + function (string $input, ?string $expectedKey, ?string $expectedValue): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + + if ($expectedKey !== null) { + expect(json_decode($result, true)[$expectedKey])->toBe($expectedValue); + } + }, + )->with('markdown_code_blocks'); + + it('handles markdown links in strings', function (string $input, string $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + })->with('markdown_links'); + + it('handles leading and trailing characters', function (string $input, string $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + })->with('leading_trailing_characters'); + + it('handles JSON code blocks inside string values', function (string $input, string $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + })->with('json_in_strings'); + + it('handles whitespace normalization', function (): void { + $input = '{"key" : "value" , "key2" : "value2"}'; + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + $decoded = json_decode($result, true); + expect($decoded)->toBe([ + 'key' => 'value', + 'key2' => 'value2', + ]); + }); + + it('removes comments', function (string $input, string $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + })->with('comments'); +}); diff --git a/tests/Unit/JsonRepairerTest.php b/tests/Unit/JsonRepairerTest.php deleted file mode 100644 index d7a991d..0000000 --- a/tests/Unit/JsonRepairerTest.php +++ /dev/null @@ -1,721 +0,0 @@ -toBeTrue(); - expect(json_decode($result, true))->toBe(json_decode($json, true)); - })->with('valid_json'); - - it('handles non-JSON strings', function (string $input, string $expected): void { - $result = json_repair($input); - expect($result)->toBe($expected); - - if ($result !== '') { - expect(json_validate($result))->toBeTrue(); - expect(json_decode($result, true))->toBe([]); - } - })->with('parse_string'); - - it('repairs single quotes to double quotes', function (string $input, string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - })->with('single_quotes_to_double'); - - it('repairs unquoted keys', function (string $input, string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - })->with('unquoted_keys'); - - it('repairs missing quotes around keys', function (): void { - $result = json_repair('{key: "value"}'); - expect(json_validate($result))->toBeTrue(); - expect(json_decode($result, true)['key'])->toBe('value'); - }); - - it('handles mixed single and double quotes', function (string $input, string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - })->with('mixed_quotes'); - - it('handles quotes inside string values', function (string $input, string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - })->with('quotes_inside_strings'); - - it('repairs trailing commas', function (string $input, string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - })->with('trailing_commas'); - - it('repairs missing commas', function (string $input, string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - })->with('missing_commas'); - - it('repairs missing colons', function (): void { - $result = json_repair('{"key" "value"}'); - expect(json_validate($result))->toBeTrue(); - expect(json_decode($result, true)['key'])->toBe('value'); - }); - - it('repairs missing closing brackets', function (string $input, string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - })->with('missing_closing_brackets'); - - it('repairs missing closing braces', function (string $input, string $expectedPath, string $expectedValue): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - $decoded = json_decode($result, true); - - $value = $decoded; - foreach (explode('.', $expectedPath) as $key) { - $value = $value[$key]; - } - - expect($value)->toBe($expectedValue); - })->with('missing_closing_braces'); - - it('repairs missing values', function (string $input, string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - })->with('missing_values'); - - it('handles missing keys in objects', function (): void { - $input = '{: "value"}'; - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - $decoded = json_decode($result, true); - expect($decoded)->toBeArray(); - expect($decoded)->toHaveKey('value'); - expect($decoded['value'])->toBe(''); - }); -}); - -describe('Values and structures', function (): void { - it('repairs non-standard booleans and null', function (string $input, string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - })->with('booleans_and_null'); - - it('handles standalone booleans and null', function (string $input, string $expected): void { - $result = json_repair($input); - expect($result)->toBe($expected); - - if ($result !== '') { - expect(json_validate($result))->toBeTrue(); - } - })->with('standalone_booleans_null'); - - it('handles numbers correctly', function (string $input, int|float|string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - - if (is_string($expected)) { - return; - } - - expect(json_decode($result, true)['key'])->toBe($expected); - })->with('numbers'); - - it('repairs invalid numbers', function (string $input, string $expectedJson, int|float $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expectedJson); - expect(json_decode($result, true)['key'])->toBe($expected); - })->with('invalid_numbers'); - - it( - 'handles strings with special characters', - function (string $input, string $expectedKey, string $expectedValue): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - $decoded = json_decode($result, true); - expect($decoded[$expectedKey])->toBe($expectedValue); - }, - )->with('special_characters'); - - it('handles escape sequences', function (string $input, string $expectedValue): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - $decoded = json_decode($result, true); - expect($decoded)->toBeArray(); - expect($decoded)->toHaveKey('key'); - expect($decoded['key'])->toBe($expectedValue); - })->with('escape_sequences'); - - it('handles advanced escaping cases', function (string $input, string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - })->with('advanced_escaping'); - - it('escapes unicode characters when ensureAscii is true', function (): void { - $result = json_repair("{'city':'上海'}"); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe('{"city":"\\u4e0a\\u6d77"}'); - - $decoded = json_decode($result, true); - expect($decoded['city'])->toBe('上海'); - }); - - it('handles unicode characters when ensureAscii is false', function (): void { - $input = "{'test_中国人_ascii':'统一码'}"; - $result = json_repair($input, ensureAscii: false); - expect(json_validate($result))->toBeTrue(); - expect($result)->toContain('统一码'); - expect($result)->toContain('test_中国人_ascii'); - - $decoded = json_decode($result, true); - expect($decoded)->toHaveKey('test_中国人_ascii'); - expect($decoded['test_中国人_ascii'])->toBe('统一码'); - }); - - it('handles empty strings as values', function (string $input, string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - })->with('empty_strings'); - - it('handles nested structures', function (string $input): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect(json_decode($result, true))->toBe(json_decode($input, true)); - })->with('nested_structures'); - - it('handles empty structures', function (string $input, string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - })->with('empty_structures'); - - it('handles arrays with mixed types', function (string $input, string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - })->with('mixed_type_arrays'); -}); - -describe('Edge cases and special features', function (): void { - it('handles incomplete JSON at end of string', function (string $input, string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - })->with('incomplete_json'); - - it('repairs incomplete JSON from streaming LLM responses', function (string $input, string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - })->with('streaming_llm_responses'); - - it('handles complex nested structures', function (): void { - $input = '{"resourceType": "Bundle", "id": "1", "type": "collection", "entry": [{"resource": {"resourceType": "Patient", "id": "1", "name": [{"use": "official", "family": "Corwin", "given": ["Keisha", "Sunny"], "prefix": ["Mrs."}, {"use": "maiden", "family": "Goodwin", "given": ["Keisha", "Sunny"], "prefix": ["Mrs."]}]}}]}'; - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - - $decoded = json_decode($result, true); - expect($decoded)->toBeArray(); - expect($decoded['resourceType'])->toBe('Bundle'); - expect($decoded['id'])->toBe('1'); - expect($decoded['type'])->toBe('collection'); - expect($decoded['entry'])->toBeArray(); - expect($decoded['entry'][0]['resource']['resourceType'])->toBe('Patient'); - expect($decoded['entry'][0]['resource']['name'])->toBeArray(); - expect($decoded['entry'][0]['resource']['name'])->toHaveCount(1); - expect($decoded['entry'][0]['resource']['name'][0]['use'])->toBe('official'); - expect($decoded['entry'][0]['resource']['name'][0]['family'])->toBe('Corwin'); - expect($decoded['entry'][0]['resource']['name'][0]['given'])->toBe(['Keisha', 'Sunny']); - expect($decoded['entry'][0]['resource']['name'][0]['prefix'][0])->toBe('Mrs.'); - expect($decoded['entry'][0]['resource']['name'][0]['prefix'][1])->toBeArray(); - expect($decoded['entry'][0]['resource']['name'][0]['prefix'][1]['use'])->toBe('maiden'); - expect($decoded['entry'][0]['resource']['name'][0]['prefix'][1]['family'])->toBe('Goodwin'); - }); - - it('handles multiple JSON objects', function (string $input, ?string $expectedKey, ?string $expectedValue): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - - if ($expectedKey !== null) { - $decoded = json_decode($result, true); - - if ($expectedValue !== null) { - expect($decoded[$expectedKey])->toBe($expectedValue); - } else { - expect($decoded)->toBeArray(); - } - } - })->with('multiple_json_objects'); - - it( - 'extracts JSON from markdown code blocks', - function (string $input, ?string $expectedKey, ?string $expectedValue): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - - if ($expectedKey !== null) { - expect(json_decode($result, true)[$expectedKey])->toBe($expectedValue); - } - }, - )->with('markdown_code_blocks'); - - it('handles markdown links in strings', function (string $input, string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - })->with('markdown_links'); - - it('handles leading and trailing characters', function (string $input, string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - })->with('leading_trailing_characters'); - - it('handles JSON code blocks inside string values', function (string $input, string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - })->with('json_in_strings'); - - it('handles whitespace normalization', function (): void { - $input = '{"key" : "value" , "key2" : "value2"}'; - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - $decoded = json_decode($result, true); - expect($decoded)->toBe([ - 'key' => 'value', - 'key2' => 'value2', - ]); - }); - - it('removes comments', function (string $input, string $expected): void { - $result = json_repair($input); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - })->with('comments'); -}); - -describe('Options', function (): void { - describe('omitEmptyValues', function (): void { - it('omits empty values when omitEmptyValues is true', function (string $input, string $expected): void { - $result = json_repair($input, omitEmptyValues: true); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - })->with('omit_empty_values_true'); - - it('keeps empty values when omitEmptyValues is false', function (string $input, string $expected): void { - $result = json_repair($input, omitEmptyValues: false); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - })->with('omit_empty_values_false'); - - it('handles nested structures with omitEmptyValues', function (): void { - $input = '{"user": {"name": "John", "age": }, "meta": {"count": }}'; - $expected = '{"user": {"name": "John"}, "meta": {}}'; - $result = json_repair($input, omitEmptyValues: true); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe([ - 'user' => [ - 'name' => 'John', - ], - 'meta' => [], - ]); - }); - - it('handles edge case where removing key leaves empty object', function (): void { - $input = '{"key": }'; - $expected = '{}'; - $result = json_repair($input, omitEmptyValues: true); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe([]); - }); - }); - - describe('omitIncompleteStrings', function (): void { - it( - 'omits incomplete strings when omitIncompleteStrings is true', - function (string $input, string $expected): void { - $result = json_repair($input, omitIncompleteStrings: true); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - }, - )->with('omit_incomplete_strings_true'); - - it( - 'keeps incomplete strings when omitIncompleteStrings is false', - function (string $input, string $expected): void { - $result = json_repair($input, omitIncompleteStrings: false); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - }, - )->with('omit_incomplete_strings_false'); - - it('handles edge case where removing incomplete string leaves empty object', function (): void { - $input = '{"key": "val'; - $expected = '{}'; - $result = json_repair($input, omitIncompleteStrings: true); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe([]); - }); - }); - - describe('combined options', function (): void { - it( - 'handles both omitEmptyValues and omitIncompleteStrings together', - function (string $input, string $expected): void { - $result = json_repair($input, omitEmptyValues: true, omitIncompleteStrings: true); - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe($expected); - - $decoded = json_decode($result, true); - expect($decoded)->toBe(json_decode($expected, true)); - }, - )->with('combined_options'); - }); -}); - -describe('API usage', function (): void { - it('works with JsonRepairer class directly', function (): void { - $repairer = new JsonRepairer("{'key': 'value'}"); - $result = $repairer->repair(); - expect(json_validate($result))->toBeTrue(); - expect(json_decode($result, true)['key'])->toBe('value'); - }); - - it('can decode repaired JSON', function (): void { - $repairer = new JsonRepairer("{'key': 'value', 'number': 123}"); - $decoded = $repairer->decode(); - - expect($decoded)->toBeArray(); - expect($decoded['key'])->toBe('value'); - expect($decoded['number'])->toBe(123); - }); - - it('can use json_repair_decode helper function', function (): void { - $decoded = json_repair_decode("{'key': 'value', 'number': 123}"); - - expect($decoded)->toBeArray(); - expect($decoded['key'])->toBe('value'); - expect($decoded['number'])->toBe(123); - }); - - it('works with JsonRepairer class directly with omitEmptyValues', function (): void { - $repairer = new JsonRepairer('{"key": }', omitEmptyValues: true); - $result = $repairer->repair(); - expect(json_validate($result))->toBeTrue(); - $decoded = json_decode($result, true); - expect($decoded)->toBe([]); - }); - - it('works with JsonRepairer class directly with omitIncompleteStrings', function (): void { - $repairer = new JsonRepairer('{"key": "val', omitIncompleteStrings: true); - $result = $repairer->repair(); - expect(json_validate($result))->toBeTrue(); - $decoded = json_decode($result, true); - expect($decoded)->toBe([]); - }); - - it('works with json_repair_decode with new options', function (): void { - $decoded = json_repair_decode('{"key": }', omitEmptyValues: true); - expect($decoded)->toBeArray(); - expect($decoded)->toBe([]); - }); -}); - -describe('Logging', function (): void { - it('logs nothing for valid JSON', function (): void { - $logger = new TestLogger(); - - $result = json_repair('{"key": "value"}', logger: $logger); - - expect($logger->hasDebug('JSON is already valid, returning as-is'))->toBeTrue(); - expect($logger->records)->toHaveCount(1); - expect($result)->toBe('{"key": "value"}'); - }); - - it('logs repair actions for unclosed strings and brackets', function (): void { - $logger = new TestLogger(); - - $result = json_repair('{"key": "value', logger: $logger); - - expect($logger->hasDebug('Starting JSON repair'))->toBeTrue(); - expect($logger->hasDebug('Adding missing closing quote for unclosed string'))->toBeTrue(); - expect($logger->hasDebug('Adding missing closing bracket/brace'))->toBeTrue(); - - expect(json_validate($result))->toBeTrue(); - expect($result)->toBe('{"key": "value"}'); - }); - - it('logs quote conversions and boolean normalization', function (): void { - $logger = new TestLogger(); - - $result = json_repair("{'active': True}", logger: $logger); - - expect($logger->hasDebug('Converting single-quoted key to double quotes'))->toBeTrue(); - expect($logger->hasDebugThatPasses( - fn(array $record): bool => $record['message'] === 'Normalizing boolean/null value' - && $record['context']['from'] === 'True' - && $record['context']['to'] === 'true', - ))->toBeTrue(); - - expect($result)->toBe('{"active": true}'); - }); - - it('logs unquoted key and value repairs', function (): void { - $logger = new TestLogger(); - - $result = json_repair('{name: John}', logger: $logger); - - expect($logger->hasDebug('Adding quotes around unquoted key'))->toBeTrue(); - expect($logger->hasDebug('Found unquoted string value, adding quotes'))->toBeTrue(); - - expect($result)->toBe('{"name": "John"}'); - }); - - it('logs missing comma and colon insertions', function (): void { - $logger = new TestLogger(); - - $result = json_repair('{"a": 1 "b" 2}', logger: $logger); - - expect($logger->hasDebug('Inserting missing comma'))->toBeTrue(); - expect($logger->hasDebug('Inserting missing colon after key'))->toBeTrue(); - - expect(json_validate($result))->toBeTrue(); - }); - - it('logs context with position information', function (): void { - $logger = new TestLogger(); - - json_repair('{"key": value}', logger: $logger); - - // Verify that log entries include position and context - expect($logger->hasDebugThatPasses( - fn(array $record): bool => isset($record['context']['position']) - && isset($record['context']['context']) - && str_contains((string) $record['context']['context'], '>>>'), - ))->toBeTrue(); - }); - - it('logs markdown extraction', function (): void { - $logger = new TestLogger(); - - $result = json_repair('```json {"key": "value"} ```', logger: $logger); - - expect($logger->hasDebug('Extracted JSON from markdown code block'))->toBeTrue(); - expect($result)->toBe('{"key": "value"}'); - }); - - it('logs omitEmptyValues actions', function (): void { - $logger = new TestLogger(); - - $result = json_repair('{"a": 1, "b": }', omitEmptyValues: true, logger: $logger); - - expect($logger->hasDebug('Removing key with missing value (omitEmptyValues enabled)'))->toBeTrue(); - expect($result)->toBe('{"a": 1}'); - }); - - it('works with JsonRepairer class and setLogger', function (): void { - $logger = new TestLogger(); - - $repairer = new JsonRepairer("{'key': 'value'}"); - $repairer->setLogger($logger); - - $result = $repairer->repair(); - - expect($logger->hasDebugRecords())->toBeTrue(); - expect($result)->toBe('{"key": "value"}'); - }); -}); - -describe('RepairResult', function (): void { - it('returns structured repair details', function (): void { - $repairer = new JsonRepairer("{'key': 'value'}"); - $repairResult = $repairer->repairWithDetails(); - - expect($repairResult->wasAlreadyValid)->toBeFalse(); - expect($repairResult->json)->toBe('{"key": "value"}'); - expect($repairResult->fixes)->toContain('Starting JSON repair'); - }); - - it('marks valid JSON as already valid', function (): void { - $repairResult = (new JsonRepairer('{"key": "value"}'))->repairWithDetails(); - - expect($repairResult->wasAlreadyValid)->toBeTrue(); - expect($repairResult->json)->toBe('{"key": "value"}'); - expect($repairResult->fixes)->toContain('JSON is already valid, returning as-is'); - }); -}); - -describe('repairAll', function (): void { - it('repairs multiple top-level JSON values', function (): void { - $repairer = new JsonRepairer('{"a":1}{"b":2}'); - $results = $repairer->repairAll(); - - expect($results)->toHaveCount(2); - expect(json_decode($results[0], true))->toBe([ - 'a' => 1, - ]); - expect(json_decode($results[1], true))->toBe([ - 'b' => 2, - ]); - }); - - it('repairs NDJSON lines', function (): void { - $repairer = new JsonRepairer("{'a':1}\n{'b':2}"); - $results = $repairer->repairAll(); - - expect($results)->toHaveCount(2); - expect(json_decode($results[0], true))->toBe([ - 'a' => 1, - ]); - expect(json_decode($results[1], true))->toBe([ - 'b' => 2, - ]); - }); -}); - -describe('StreamingJsonRepairer', function (): void { - it('repairs incrementally fed chunks', function (): void { - $stream = new StreamingJsonRepairer(); - $stream->feed('{"key": '); - $stream->feed('"val'); - - $result = $stream->current(); - - expect(json_validate($result))->toBeTrue(); - expect(json_decode($result, true))->toBe([ - 'key' => 'val', - ]); - }); -}); - -describe('Duplicate key policy', function (): void { - it('keeps first duplicate key when configured', function (): void { - $repairer = new JsonRepairer('{a: 1, a: 2}', duplicateKeyPolicy: DuplicateKeyPolicy::KeepFirst); - $result = $repairer->repair(); - - expect(json_decode($result, true))->toBe([ - 'a' => 1, - ]); - }); - - it('keeps last duplicate key when configured', function (): void { - $repairer = new JsonRepairer('{a: 1, a: 2}', duplicateKeyPolicy: DuplicateKeyPolicy::KeepLast); - $result = $repairer->repair(); - - expect(json_decode($result, true))->toBe([ - 'a' => 2, - ]); - }); -}); - -describe('Scalar decode', function (): void { - it('decodes top-level scalar values', function (): void { - $repairer = new JsonRepairer('true'); - expect($repairer->decode())->toBeTrue(); - }); - - it('decodes top-level null via json_repair_decode', function (): void { - expect(json_repair_decode('null'))->toBeNull(); - }); - - it('decodes top-level string scalars', function (): void { - expect((new JsonRepairer('"hello"'))->decode())->toBe('hello'); - }); -}); diff --git a/tests/Unit/JsonRepairsTest.php b/tests/Unit/JsonRepairsTest.php new file mode 100644 index 0000000..6d3fd07 --- /dev/null +++ b/tests/Unit/JsonRepairsTest.php @@ -0,0 +1,133 @@ +toBeTrue(); + expect(json_decode($result, true))->toBe(json_decode($json, true)); + })->with('valid_json'); + + it('handles non-JSON strings', function (string $input, string $expected): void { + $result = json_repair($input); + expect($result)->toBe($expected); + + if ($result !== '') { + expect(json_validate($result))->toBeTrue(); + expect(json_decode($result, true))->toBe([]); + } + })->with('parse_string'); + + it('repairs single quotes to double quotes', function (string $input, string $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + })->with('single_quotes_to_double'); + + it('repairs unquoted keys', function (string $input, string $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + })->with('unquoted_keys'); + + it('repairs missing quotes around keys', function (): void { + $result = json_repair('{key: "value"}'); + expect(json_validate($result))->toBeTrue(); + expect(json_decode($result, true)['key'])->toBe('value'); + }); + + it('handles mixed single and double quotes', function (string $input, string $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + })->with('mixed_quotes'); + + it('handles quotes inside string values', function (string $input, string $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + })->with('quotes_inside_strings'); + + it('repairs trailing commas', function (string $input, string $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + })->with('trailing_commas'); + + it('repairs missing commas', function (string $input, string $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + })->with('missing_commas'); + + it('repairs missing colons', function (): void { + $result = json_repair('{"key" "value"}'); + expect(json_validate($result))->toBeTrue(); + expect(json_decode($result, true)['key'])->toBe('value'); + }); + + it('repairs missing closing brackets', function (string $input, string $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + })->with('missing_closing_brackets'); + + it('repairs missing closing braces', function (string $input, string $expectedPath, string $expectedValue): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + $decoded = json_decode($result, true); + + $value = $decoded; + foreach (explode('.', $expectedPath) as $key) { + $value = $value[$key]; + } + + expect($value)->toBe($expectedValue); + })->with('missing_closing_braces'); + + it('repairs missing values', function (string $input, string $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + })->with('missing_values'); + + it('handles missing keys in objects', function (): void { + $input = '{: "value"}'; + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + $decoded = json_decode($result, true); + expect($decoded)->toBeArray(); + expect($decoded)->toHaveKey('value'); + expect($decoded['value'])->toBe(''); + }); +}); diff --git a/tests/Unit/LoggingTest.php b/tests/Unit/LoggingTest.php new file mode 100644 index 0000000..6cdefd4 --- /dev/null +++ b/tests/Unit/LoggingTest.php @@ -0,0 +1,117 @@ +hasDebug('JSON is already valid, returning as-is'))->toBeTrue(); + expect($logger->records)->toHaveCount(1); + expect($result)->toBe('{"key": "value"}'); + }); + + it('logs repair actions for unclosed strings and brackets', function (): void { + $logger = new TestLogger(); + + $result = json_repair('{"key": "value', logger: $logger); + + expect($logger->hasDebug('Starting JSON repair'))->toBeTrue(); + expect($logger->hasDebug('Adding missing closing quote for unclosed string'))->toBeTrue(); + expect($logger->hasDebug('Adding missing closing bracket/brace'))->toBeTrue(); + + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe('{"key": "value"}'); + }); + + it('logs quote conversions and boolean normalization', function (): void { + $logger = new TestLogger(); + + $result = json_repair("{'active': True}", logger: $logger); + + expect($logger->hasDebug('Converting single-quoted key to double quotes'))->toBeTrue(); + expect($logger->hasDebugThatPasses( + fn(array $record): bool => $record['message'] === 'Normalizing boolean/null value' + && $record['context']['from'] === 'True' + && $record['context']['to'] === 'true', + ))->toBeTrue(); + + expect($result)->toBe('{"active": true}'); + }); + + it('logs unquoted key and value repairs', function (): void { + $logger = new TestLogger(); + + $result = json_repair('{name: John}', logger: $logger); + + expect($logger->hasDebug('Adding quotes around unquoted key'))->toBeTrue(); + expect($logger->hasDebug('Found unquoted string value, adding quotes'))->toBeTrue(); + + expect($result)->toBe('{"name": "John"}'); + }); + + it('logs missing comma and colon insertions', function (): void { + $logger = new TestLogger(); + + $result = json_repair('{"a": 1 "b" 2}', logger: $logger); + + expect($logger->hasDebug('Inserting missing comma'))->toBeTrue(); + expect($logger->hasDebug('Inserting missing colon after key'))->toBeTrue(); + + expect(json_validate($result))->toBeTrue(); + }); + + it('logs context with position information', function (): void { + $logger = new TestLogger(); + + json_repair('{"key": value}', logger: $logger); + + // Verify that log entries include position and context + expect($logger->hasDebugThatPasses( + fn(array $record): bool => isset($record['context']['position']) + && isset($record['context']['context']) + && str_contains((string) $record['context']['context'], '>>>'), + ))->toBeTrue(); + }); + + it('logs markdown extraction', function (): void { + $logger = new TestLogger(); + + $result = json_repair('```json {"key": "value"} ```', logger: $logger); + + expect($logger->hasDebug('Extracted JSON from markdown code block'))->toBeTrue(); + expect($result)->toBe('{"key": "value"}'); + }); + + it('logs omitEmptyValues actions', function (): void { + $logger = new TestLogger(); + + $result = json_repair('{"a": 1, "b": }', omitEmptyValues: true, logger: $logger); + + expect($logger->hasDebug('Removing key with missing value (omitEmptyValues enabled)'))->toBeTrue(); + expect($result)->toBe('{"a": 1}'); + }); + + it('works with JsonRepairer class and setLogger', function (): void { + $logger = new TestLogger(); + + $repairer = new JsonRepairer("{'key': 'value'}"); + $repairer->setLogger($logger); + + $result = $repairer->repair(); + + expect($logger->hasDebugRecords())->toBeTrue(); + expect($result)->toBe('{"key": "value"}'); + }); +}); diff --git a/tests/Unit/OptionsTest.php b/tests/Unit/OptionsTest.php new file mode 100644 index 0000000..8400802 --- /dev/null +++ b/tests/Unit/OptionsTest.php @@ -0,0 +1,111 @@ +toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + })->with('omit_empty_values_true'); + + it('keeps empty values when omitEmptyValues is false', function (string $input, string $expected): void { + $result = json_repair($input, omitEmptyValues: false); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + })->with('omit_empty_values_false'); + + it('handles nested structures with omitEmptyValues', function (): void { + $input = '{"user": {"name": "John", "age": }, "meta": {"count": }}'; + $expected = '{"user": {"name": "John"}, "meta": {}}'; + $result = json_repair($input, omitEmptyValues: true); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe([ + 'user' => [ + 'name' => 'John', + ], + 'meta' => [], + ]); + }); + + it('handles edge case where removing key leaves empty object', function (): void { + $input = '{"key": }'; + $expected = '{}'; + $result = json_repair($input, omitEmptyValues: true); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe([]); + }); + }); + + describe('omitIncompleteStrings', function (): void { + it( + 'omits incomplete strings when omitIncompleteStrings is true', + function (string $input, string $expected): void { + $result = json_repair($input, omitIncompleteStrings: true); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + }, + )->with('omit_incomplete_strings_true'); + + it( + 'keeps incomplete strings when omitIncompleteStrings is false', + function (string $input, string $expected): void { + $result = json_repair($input, omitIncompleteStrings: false); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + }, + )->with('omit_incomplete_strings_false'); + + it('handles edge case where removing incomplete string leaves empty object', function (): void { + $input = '{"key": "val'; + $expected = '{}'; + $result = json_repair($input, omitIncompleteStrings: true); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe([]); + }); + }); + + describe('combined options', function (): void { + it( + 'handles both omitEmptyValues and omitIncompleteStrings together', + function (string $input, string $expected): void { + $result = json_repair($input, omitEmptyValues: true, omitIncompleteStrings: true); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + }, + )->with('combined_options'); + }); +}); diff --git a/tests/Unit/StreamingJsonRepairerTest.php b/tests/Unit/StreamingJsonRepairerTest.php new file mode 100644 index 0000000..7f83675 --- /dev/null +++ b/tests/Unit/StreamingJsonRepairerTest.php @@ -0,0 +1,24 @@ +feed('{"key": '); + $stream->feed('"val'); + + $result = $stream->current(); + + expect(json_validate($result))->toBeTrue(); + expect(json_decode($result, true))->toBe([ + 'key' => 'val', + ]); + }); +}); diff --git a/tests/Unit/ValuesAndStructuresTest.php b/tests/Unit/ValuesAndStructuresTest.php new file mode 100644 index 0000000..05356b8 --- /dev/null +++ b/tests/Unit/ValuesAndStructuresTest.php @@ -0,0 +1,131 @@ +toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + })->with('booleans_and_null'); + + it('handles standalone booleans and null', function (string $input, string $expected): void { + $result = json_repair($input); + expect($result)->toBe($expected); + + if ($result !== '') { + expect(json_validate($result))->toBeTrue(); + } + })->with('standalone_booleans_null'); + + it('handles numbers correctly', function (string $input, int|float|string $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + + if (is_string($expected)) { + return; + } + + expect(json_decode($result, true)['key'])->toBe($expected); + })->with('numbers'); + + it('repairs invalid numbers', function (string $input, string $expectedJson, int|float $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expectedJson); + expect(json_decode($result, true)['key'])->toBe($expected); + })->with('invalid_numbers'); + + it( + 'handles strings with special characters', + function (string $input, string $expectedKey, string $expectedValue): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + $decoded = json_decode($result, true); + expect($decoded[$expectedKey])->toBe($expectedValue); + }, + )->with('special_characters'); + + it('handles escape sequences', function (string $input, string $expectedValue): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + $decoded = json_decode($result, true); + expect($decoded)->toBeArray(); + expect($decoded)->toHaveKey('key'); + expect($decoded['key'])->toBe($expectedValue); + })->with('escape_sequences'); + + it('handles advanced escaping cases', function (string $input, string $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + })->with('advanced_escaping'); + + it('escapes unicode characters when ensureAscii is true', function (): void { + $result = json_repair("{'city':'上海'}"); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe('{"city":"\\u4e0a\\u6d77"}'); + + $decoded = json_decode($result, true); + expect($decoded['city'])->toBe('上海'); + }); + + it('handles unicode characters when ensureAscii is false', function (): void { + $input = "{'test_中国人_ascii':'统一码'}"; + $result = json_repair($input, ensureAscii: false); + expect(json_validate($result))->toBeTrue(); + expect($result)->toContain('统一码'); + expect($result)->toContain('test_中国人_ascii'); + + $decoded = json_decode($result, true); + expect($decoded)->toHaveKey('test_中国人_ascii'); + expect($decoded['test_中国人_ascii'])->toBe('统一码'); + }); + + it('handles empty strings as values', function (string $input, string $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + })->with('empty_strings'); + + it('handles nested structures', function (string $input): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect(json_decode($result, true))->toBe(json_decode($input, true)); + })->with('nested_structures'); + + it('handles empty structures', function (string $input, string $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + })->with('empty_structures'); + + it('handles arrays with mixed types', function (string $input, string $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect($result)->toBe($expected); + + $decoded = json_decode($result, true); + expect($decoded)->toBe(json_decode($expected, true)); + })->with('mixed_type_arrays'); +}); From 96b359dec711283c4d5e06bbeacab521feda472c Mon Sep 17 00:00:00 2001 From: Sean Tymon Date: Thu, 25 Jun 2026 08:55:12 +0100 Subject: [PATCH 05/12] run bench in CI --- .github/workflows/benchmark.yml | 45 +++++++++++++++++++++++++++++++++ .gitignore | 1 + composer.json | 2 ++ phpbench.json | 2 ++ 4 files changed, 50 insertions(+) create mode 100644 .github/workflows/benchmark.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..9b7a1be --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,45 @@ +name: Benchmarks + +permissions: + contents: read + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + benchmark: + runs-on: ubuntu-latest + name: Benchmark regression + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup PHP and Composer + uses: ./.github/actions/setup-php-composer + with: + php-version: 8.3 + + # Store a baseline from the PR's base branch on this same runner, so the + # comparison is hardware-consistent (baselines are never committed). + - name: Store baseline from base branch + run: | + git checkout ${{ github.event.pull_request.base.sha }} + composer install --prefer-dist --no-interaction --no-progress + vendor/bin/phpbench run --tag=base --progress=none + + - name: Compare PR against baseline + run: | + git checkout ${{ github.event.pull_request.head.sha }} + composer install --prefer-dist --no-interaction --no-progress + vendor/bin/phpbench run \ + --ref=base \ + --report=aggregate \ + --progress=none \ + --assert="mode(variant.time.avg) <= mode(baseline.time.avg) +/- 15%" diff --git a/.gitignore b/.gitignore index 419e4db..f0a1dbb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ vendor composer.lock +.phpbench .vscode .env .DS_Store diff --git a/composer.json b/composer.json index b86972e..7147aef 100644 --- a/composer.json +++ b/composer.json @@ -47,6 +47,8 @@ "scripts": { "test": "pest", "benchmark": "phpbench run", + "benchmark:base": "phpbench run --tag=base --progress=none", + "benchmark:ci": "phpbench run --ref=base --report=aggregate --progress=none --assert=\"mode(variant.time.avg) <= mode(baseline.time.avg) +/- 15%\"", "ecs": "ecs check --fix", "rector": "rector process", "stan": "phpstan analyse", diff --git a/phpbench.json b/phpbench.json index ce0b109..00fb672 100644 --- a/phpbench.json +++ b/phpbench.json @@ -5,7 +5,9 @@ "runner.iterations": 10, "runner.revs": 100, "runner.warmup": 2, + "runner.retry_threshold": 5, "runner.time_unit": "microseconds", + "storage.xml_storage_path": ".phpbench/storage", "report.generators": { "compare": { "generator": "table", From 0dccd5e9af2cf66e752c31c39148e740d2df5534 Mon Sep 17 00:00:00 2001 From: Sean Tymon Date: Thu, 25 Jun 2026 09:02:09 +0100 Subject: [PATCH 06/12] add to pr --- .github/workflows/benchmark.yml | 65 ++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 9b7a1be..7722b1f 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -2,6 +2,7 @@ name: Benchmarks permissions: contents: read + pull-requests: write on: pull_request: @@ -35,11 +36,73 @@ jobs: vendor/bin/phpbench run --tag=base --progress=none - name: Compare PR against baseline + id: compare run: | git checkout ${{ github.event.pull_request.head.sha }} composer install --prefer-dist --no-interaction --no-progress + set +e vendor/bin/phpbench run \ --ref=base \ --report=aggregate \ --progress=none \ - --assert="mode(variant.time.avg) <= mode(baseline.time.avg) +/- 15%" + --no-ansi \ + --assert="mode(variant.time.avg) <= mode(baseline.time.avg) +/- 15%" \ + > benchmark-report.txt 2>&1 + echo "exit_code=$?" >> "$GITHUB_OUTPUT" + cat benchmark-report.txt + + - name: Write job summary + if: always() + run: | + { + echo "## Benchmark results" + echo + echo '```' + cat benchmark-report.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Comment results on PR + if: always() + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const report = fs.readFileSync('benchmark-report.txt', 'utf8'); + const passed = '${{ steps.compare.outputs.exit_code }}' === '0'; + const status = passed + ? 'No performance regression detected (within 15% tolerance).' + : 'Possible performance regression detected (exceeds 15% tolerance).'; + const marker = ''; + const body = [ + marker, + '## Benchmark results', + '', + status, + '', + '
', + 'Aggregate report vs. base', + '', + '```', + report, + '```', + '', + '
', + '', + `_Compared against base \`${context.payload.pull_request.base.sha.slice(0, 7)}\`._`, + ].join('\n'); + const { owner, repo } = context.repo; + const issue_number = context.payload.pull_request.number; + const { data: comments } = await github.rest.issues.listComments({ owner, repo, issue_number }); + const existing = comments.find((c) => c.body && c.body.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ owner, repo, issue_number, body }); + } + + - name: Fail on regression + if: always() && steps.compare.outputs.exit_code != '0' + run: | + echo "Benchmark regression exceeded the configured tolerance." + exit 1 From 99da22b2fcfa2839399fd34ee61d3d5497caea88 Mon Sep 17 00:00:00 2001 From: Sean Tymon Date: Thu, 25 Jun 2026 09:15:08 +0100 Subject: [PATCH 07/12] improve --- src/Concerns/OutputTracking.php | 12 +++++++----- src/JsonRepairer.php | 20 +++++++++++++------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/Concerns/OutputTracking.php b/src/Concerns/OutputTracking.php index 508cb5a..0bd54e3 100644 --- a/src/Concerns/OutputTracking.php +++ b/src/Concerns/OutputTracking.php @@ -23,12 +23,14 @@ private function syncOutputTail(): void { $length = strlen($this->output); - for ($j = $this->outputSyncedLength; $j < $length; $j++) { - $char = $this->output[$j]; + if ($length === $this->outputSyncedLength) { + return; + } + + $trimmed = rtrim(substr($this->output, $this->outputSyncedLength), " \t\n\r\f\v"); - if (! in_array($char, [' ', "\t", "\n", "\r", "\f", "\v"], true)) { - $this->lastNonWhitespaceChar = $char; - } + if ($trimmed !== '') { + $this->lastNonWhitespaceChar = $trimmed[strlen($trimmed) - 1]; } $this->outputSyncedLength = $length; diff --git a/src/JsonRepairer.php b/src/JsonRepairer.php index d0f7aeb..76c1489 100644 --- a/src/JsonRepairer.php +++ b/src/JsonRepairer.php @@ -66,25 +66,31 @@ public function __construct( public function repair(): string { - return $this->repairWithDetails()->json; + if (json_validate($this->json)) { + $this->log('JSON is already valid, returning as-is'); + + return $this->json; + } + + $this->log('Starting JSON repair'); + + return $this->repairInternal(extractFirstOnly: true); } public function repairWithDetails(): RepairResult { + $this->beginFixCollection(); + if (json_validate($this->json)) { - $this->beginFixCollection(); $this->log('JSON is already valid, returning as-is'); - $fixes = $this->endFixCollection(); - return new RepairResult($this->json, true, $fixes); + return new RepairResult($this->json, true, $this->endFixCollection()); } - $this->beginFixCollection(); $this->log('Starting JSON repair'); $repaired = $this->repairInternal(extractFirstOnly: true); - $fixes = $this->endFixCollection(); - return new RepairResult($repaired, false, $fixes); + return new RepairResult($repaired, false, $this->endFixCollection()); } /** From ce884f8a76143a1318404eecca4426b88be0cd41 Mon Sep 17 00:00:00 2001 From: Sean Tymon Date: Thu, 25 Jun 2026 23:10:06 +0100 Subject: [PATCH 08/12] fixes --- src/Concerns/InputSanitization.php | 2 +- src/JsonRepairer.php | 59 ++++++++++++++++---------- tests/Unit/ApiUsageTest.php | 66 ++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 23 deletions(-) diff --git a/src/Concerns/InputSanitization.php b/src/Concerns/InputSanitization.php index efe7fde..154410a 100644 --- a/src/Concerns/InputSanitization.php +++ b/src/Concerns/InputSanitization.php @@ -410,7 +410,7 @@ private function extractAllTopLevelJson(string $input): array } $depth++; - } elseif ($char === '}' || $char === ']') { + } elseif (($char === '}' || $char === ']') && $depth > 0) { $depth--; if ($depth === 0 && $start !== -1) { diff --git a/src/JsonRepairer.php b/src/JsonRepairer.php index 76c1489..f9192a0 100644 --- a/src/JsonRepairer.php +++ b/src/JsonRepairer.php @@ -446,29 +446,17 @@ private function handleDuplicateKey(string $keyName): bool return true; } - $this->log('Replacing duplicate key (keep-last policy)', [ + // Keep-last is resolved during finalization: the duplicate value is emitted + // normally and json_decode() retains the final occurrence. Surgically removing + // the earlier occurrence here would also delete any intervening keys. + $this->log('Keeping last duplicate key (keep-last policy)', [ 'key' => $keyName, ]); - $previousStart = $this->objectKeysStack[$depth][$keyName]; - $this->removeKeyValueRegion($previousStart, $this->currentKeyStart); $this->objectKeysStack[$depth][$keyName] = $this->currentKeyStart; return false; } - private function removeKeyValueRegion(int $keyStart, int $nextKeyStart): void - { - $before = substr($this->output, 0, $keyStart); - $before = rtrim($before); - - if (str_ends_with($before, ',')) { - $before = rtrim(substr($before, 0, -1)); - } - - $after = substr($this->output, $nextKeyStart); - $this->setOutput($before . $after); - } - private function extractCompletedKeyName(): string { if ($this->currentKeyStart < 0) { @@ -521,15 +509,42 @@ private function skipValueAt(string $json, int $i): int } if ($char === '{' || $char === '[') { - $close = $char === '{' ? '}' : ']'; - $depth = 1; - $i++; + $depth = 0; + + while ($i < $length) { + $current = $json[$i]; + + if ($current === '"' || $current === "'") { + $delimiter = $current; + $i++; + + while ($i < $length) { + if ($json[$i] === '\\') { + $i += 2; - while ($i < $length && $depth > 0) { - if ($json[$i] === $char) { + continue; + } + + if ($json[$i] === $delimiter) { + $i++; + + break; + } + + $i++; + } + + continue; + } + + if ($current === '{' || $current === '[') { $depth++; - } elseif ($json[$i] === $close) { + } elseif ($current === '}' || $current === ']') { $depth--; + + if ($depth === 0) { + return $i + 1; + } } $i++; diff --git a/tests/Unit/ApiUsageTest.php b/tests/Unit/ApiUsageTest.php index 3f01cca..0e524cb 100644 --- a/tests/Unit/ApiUsageTest.php +++ b/tests/Unit/ApiUsageTest.php @@ -104,6 +104,19 @@ 'b' => 2, ]); }); + + it('repairs top-level values despite leading stray closing brackets', function (): void { + $repairer = new JsonRepairer('}{"a":1}{"b":2}'); + $results = $repairer->repairAll(); + + expect($results)->toHaveCount(2); + expect(json_decode($results[0], true))->toBe([ + 'a' => 1, + ]); + expect(json_decode($results[1], true))->toBe([ + 'b' => 2, + ]); + }); }); describe('Duplicate key policy', function (): void { @@ -124,6 +137,59 @@ 'a' => 2, ]); }); + + it('keeps first non-adjacent duplicate key without dropping intervening keys', function (): void { + $repairer = new JsonRepairer('{a: 1, b: 2, a: 3}', duplicateKeyPolicy: DuplicateKeyPolicy::KeepFirst); + $result = $repairer->repair(); + + expect(json_validate($result))->toBeTrue(); + expect(json_decode($result, true))->toBe([ + 'a' => 1, + 'b' => 2, + ]); + }); + + it('keeps last non-adjacent duplicate key without dropping intervening keys', function (): void { + $repairer = new JsonRepairer('{a: 1, b: 2, a: 3}', duplicateKeyPolicy: DuplicateKeyPolicy::KeepLast); + $result = $repairer->repair(); + + expect(json_validate($result))->toBeTrue(); + expect(json_decode($result, true))->toBe([ + 'a' => 3, + 'b' => 2, + ]); + }); + + it('keeps first when skipped duplicate value is a structure containing brackets in strings', function (): void { + $repairer = new JsonRepairer( + '{a: 1, a: {b: [1, 2], c: "}}"}, z: 9}', + duplicateKeyPolicy: DuplicateKeyPolicy::KeepFirst, + ); + $result = $repairer->repair(); + + expect(json_validate($result))->toBeTrue(); + expect(json_decode($result, true))->toBe([ + 'a' => 1, + 'z' => 9, + ]); + }); + + it( + 'keeps first when skipped duplicate value is an array containing its closing bracket in a string', + function (): void { + $repairer = new JsonRepairer( + '{a: 1, a: ["]", 3], z: 9}', + duplicateKeyPolicy: DuplicateKeyPolicy::KeepFirst, + ); + $result = $repairer->repair(); + + expect(json_validate($result))->toBeTrue(); + expect(json_decode($result, true))->toBe([ + 'a' => 1, + 'z' => 9, + ]); + }, + ); }); describe('Scalar decode', function (): void { From ab8376e43c86ca99162378aceb70ec093f59dfbe Mon Sep 17 00:00:00 2001 From: Sean Tymon Date: Thu, 25 Jun 2026 23:17:22 +0100 Subject: [PATCH 09/12] fixes --- src/Concerns/StateMachine.php | 18 +++++++++--------- src/JsonRepairer.php | 5 ----- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/Concerns/StateMachine.php b/src/Concerns/StateMachine.php index d27f623..bef072c 100644 --- a/src/Concerns/StateMachine.php +++ b/src/Concerns/StateMachine.php @@ -11,15 +11,6 @@ */ trait StateMachine { - /** - * Handle an escape sequence within a string. - * - * Processes escape sequences like \", \\, \/, \b, \f, \n, \r, \t, and - * unicode escapes (\uXXXX). Invalid or incomplete escapes are treated - * as escaped backslash followed by the character. - * - * @return int Number of extra characters consumed beyond the escape character itself - */ /** * @var string */ @@ -652,6 +643,15 @@ private function handleNumber(string $json, int $i): int return $i; } + /** + * Handle an escape sequence within a string. + * + * Processes escape sequences like \", \\, \/, \b, \f, \n, \r, \t, and + * unicode escapes (\uXXXX). Invalid or incomplete escapes are treated + * as escaped backslash followed by the character. + * + * @return int Number of extra characters consumed beyond the escape character itself + */ private function handleEscapeSequence(string $char, string $json): int { if (str_contains(self::VALID_ESCAPES, $char)) { diff --git a/src/JsonRepairer.php b/src/JsonRepairer.php index f9192a0..8deeb56 100644 --- a/src/JsonRepairer.php +++ b/src/JsonRepairer.php @@ -183,11 +183,6 @@ private function repairInternal(bool $extractFirstOnly): string $this->pos = $i; if ($this->state === ParserState::STATE_IN_STRING_ESCAPE) { - if ($i >= strlen($json)) { - $this->state = ParserState::STATE_IN_STRING; - break; - } - $extraCharsConsumed = $this->handleEscapeSequence($char, $json); $this->state = ParserState::STATE_IN_STRING; $i += 1 + $extraCharsConsumed; From 36a85a72e9e70ca570c336fe73653101a41e2157 Mon Sep 17 00:00:00 2001 From: Sean Tymon Date: Thu, 25 Jun 2026 23:29:19 +0100 Subject: [PATCH 10/12] increase tolerance --- .github/workflows/benchmark.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 7722b1f..3afaadb 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -46,7 +46,7 @@ jobs: --report=aggregate \ --progress=none \ --no-ansi \ - --assert="mode(variant.time.avg) <= mode(baseline.time.avg) +/- 15%" \ + --assert="mode(variant.time.avg) <= mode(baseline.time.avg) +/- 25%" \ > benchmark-report.txt 2>&1 echo "exit_code=$?" >> "$GITHUB_OUTPUT" cat benchmark-report.txt @@ -71,8 +71,8 @@ jobs: const report = fs.readFileSync('benchmark-report.txt', 'utf8'); const passed = '${{ steps.compare.outputs.exit_code }}' === '0'; const status = passed - ? 'No performance regression detected (within 15% tolerance).' - : 'Possible performance regression detected (exceeds 15% tolerance).'; + ? 'No performance regression detected (within 25% tolerance).' + : 'Possible performance regression detected (exceeds 25% tolerance).'; const marker = ''; const body = [ marker, From 41584649d0c139500c51be81cf216a82c8323cc5 Mon Sep 17 00:00:00 2001 From: Sean Tymon Date: Thu, 25 Jun 2026 23:34:22 +0100 Subject: [PATCH 11/12] more fixes --- src/Concerns/StateMachine.php | 37 ++++++++++++++++++++++++-- tests/Unit/ValuesAndStructuresTest.php | 21 +++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/src/Concerns/StateMachine.php b/src/Concerns/StateMachine.php index bef072c..7e8cf40 100644 --- a/src/Concerns/StateMachine.php +++ b/src/Concerns/StateMachine.php @@ -380,7 +380,7 @@ private function handleObjectValue(string $json, int $i): int } // Handle numbers - if (ctype_digit($char) || $char === '-' || $char === '+' || $char === '.') { + if ($this->startsNumber($json, $i, $length)) { $this->state = ParserState::STATE_IN_NUMBER; return $i; @@ -478,7 +478,7 @@ private function handleArrayValue(string $json, int $i): int } // Handle numbers - if (ctype_digit($char) || $char === '-' || $char === '+' || $char === '.') { + if ($this->startsNumber($json, $i, $length)) { $this->state = ParserState::STATE_IN_NUMBER; return $i; @@ -487,6 +487,39 @@ private function handleArrayValue(string $json, int $i): int return $i + 1; } + /** + * Determine whether the character at $i begins a valid JSON number. + * + * A digit always starts a number. A sign (`-`/`+`) or a leading dot only + * starts a number when at least one digit follows (directly or after a dot), + * so malformed values like `.`, `-.`, or `+` are not treated as numbers + * (which would otherwise emit invalid output such as `0.`). + */ + private function startsNumber(string $json, int $i, int $length): bool + { + $char = $json[$i]; + + if (ctype_digit($char)) { + return true; + } + + if (! in_array($char, ['-', '+', '.'], true)) { + return false; + } + + $j = $i; + + if ($json[$j] === '-' || $json[$j] === '+') { + $j++; + } + + if ($j < $length && $json[$j] === '.') { + $j++; + } + + return $j < $length && ctype_digit($json[$j]); + } + /** * Handle the state expecting a comma or closing bracket/brace. * diff --git a/tests/Unit/ValuesAndStructuresTest.php b/tests/Unit/ValuesAndStructuresTest.php index 05356b8..2ae948a 100644 --- a/tests/Unit/ValuesAndStructuresTest.php +++ b/tests/Unit/ValuesAndStructuresTest.php @@ -47,6 +47,27 @@ expect(json_decode($result, true)['key'])->toBe($expected); })->with('invalid_numbers'); + it('does not treat a lone sign or dot as a number', function (string $input): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + })->with([ + 'object dot only' => ['{"key": .}'], + 'object negative dot' => ['{"key": -.}'], + 'object positive dot' => ['{"key": +.}'], + 'object sign only' => ['{"key": -}'], + 'object dot then key' => ['{"a": ., "b": 1}'], + 'array dot only' => ['[.]'], + 'array negative dot' => ['[-.]'], + 'array dot among numbers' => ['[1, ., 2]'], + ]); + + it('still parses valid leading-dot and signed numbers', function (): void { + expect(json_decode(json_repair('{"key": .5}'), true)['key'])->toBe(0.5); + expect(json_decode(json_repair('{"key": -.5}'), true)['key'])->toBe(-0.5); + expect(json_decode(json_repair('{"key": +.5}'), true)['key'])->toBe(0.5); + expect(json_decode(json_repair('[1, 2.5, 3]'), true))->toBe([1, 2.5, 3]); + }); + it( 'handles strings with special characters', function (string $input, string $expectedKey, string $expectedValue): void { From 84213009a9911a32144668e48f226f2bb57b2220 Mon Sep 17 00:00:00 2001 From: Sean Tymon Date: Thu, 25 Jun 2026 23:59:01 +0100 Subject: [PATCH 12/12] more fixes --- src/Concerns/StateMachine.php | 4 +- src/JsonRepairer.php | 63 ++++++++++++++++---------- tests/Unit/ApiUsageTest.php | 40 +++++++++++++++- tests/Unit/ValuesAndStructuresTest.php | 25 ++++++++++ 4 files changed, 105 insertions(+), 27 deletions(-) diff --git a/src/Concerns/StateMachine.php b/src/Concerns/StateMachine.php index 7e8cf40..0833ba9 100644 --- a/src/Concerns/StateMachine.php +++ b/src/Concerns/StateMachine.php @@ -645,7 +645,7 @@ private function handleNumber(string $json, int $i): int } if ($i < $length && ($json[$i] === 'e' || $json[$i] === 'E')) { - $exponentStart = $i; + $outputLengthBeforeExponent = strlen($this->output); $this->output .= $json[$i]; $i++; @@ -666,7 +666,7 @@ private function handleNumber(string $json, int $i): int if ($i > $digitStart) { $this->output .= substr($json, $digitStart, $i - $digitStart); } else { - $this->truncateOutput(strlen($this->output) - ($digitStart - $exponentStart)); + $this->truncateOutput($outputLengthBeforeExponent); } } diff --git a/src/JsonRepairer.php b/src/JsonRepairer.php index 8deeb56..8ee3882 100644 --- a/src/JsonRepairer.php +++ b/src/JsonRepairer.php @@ -386,25 +386,6 @@ private function finalizeOutput(): string } } - if ($this->duplicateKeyPolicy === DuplicateKeyPolicy::KeepLast && $this->output !== '') { - $decoded = json_decode($this->output, true); - - if (json_last_error() === JSON_ERROR_NONE) { - $flags = JSON_UNESCAPED_SLASHES; - - if (! $this->ensureAscii) { - $flags |= JSON_UNESCAPED_UNICODE; - } - - $encoded = json_encode($decoded, $flags); - - if ($encoded !== false) { - $this->output = $encoded; - $encodedViaRoundTrip = true; - } - } - } - if (! $encodedViaRoundTrip && $this->output !== '' && ! json_validate($this->output)) { throw JsonRepairException::invalidJsonAfterRepair($this->output); } @@ -441,17 +422,51 @@ private function handleDuplicateKey(string $keyName): bool return true; } - // Keep-last is resolved during finalization: the duplicate value is emitted - // normally and json_decode() retains the final occurrence. Surgically removing - // the earlier occurrence here would also delete any intervening keys. - $this->log('Keeping last duplicate key (keep-last policy)', [ + $this->log('Replacing duplicate key (keep-last policy)', [ 'key' => $keyName, ]); - $this->objectKeysStack[$depth][$keyName] = $this->currentKeyStart; + $this->removePreviousKeyOccurrence($depth, $keyName); return false; } + /** + * Remove the earlier occurrence of a duplicate key (keep-last policy) from + * the output, preserving every other key/value verbatim. + * + * Only the region from the previous key up to the *immediately following* + * key is removed (its value plus the trailing separator), so intervening + * keys are kept. Tracked output offsets are shifted to account for the + * removed span. This operates on the output string directly to avoid a + * json_decode()/json_encode() round-trip, which would reformat output and + * corrupt numbers that exceed PHP's native precision. + */ + private function removePreviousKeyOccurrence(int $depth, string $keyName): void + { + $previousStart = $this->objectKeysStack[$depth][$keyName]; + $nextStart = $this->currentKeyStart; + + foreach ($this->objectKeysStack[$depth] as $offset) { + if ($offset > $previousStart && $offset < $nextStart) { + $nextStart = $offset; + } + } + + $removedLength = $nextStart - $previousStart; + $this->setOutput(substr($this->output, 0, $previousStart) . substr($this->output, $nextStart)); + + unset($this->objectKeysStack[$depth][$keyName]); + + foreach ($this->objectKeysStack[$depth] as $name => $offset) { + if ($offset >= $nextStart) { + $this->objectKeysStack[$depth][$name] = $offset - $removedLength; + } + } + + $this->currentKeyStart -= $removedLength; + $this->objectKeysStack[$depth][$keyName] = $this->currentKeyStart; + } + private function extractCompletedKeyName(): string { if ($this->currentKeyStart < 0) { diff --git a/tests/Unit/ApiUsageTest.php b/tests/Unit/ApiUsageTest.php index 0e524cb..fb93f83 100644 --- a/tests/Unit/ApiUsageTest.php +++ b/tests/Unit/ApiUsageTest.php @@ -155,11 +155,49 @@ expect(json_validate($result))->toBeTrue(); expect(json_decode($result, true))->toBe([ - 'a' => 3, 'b' => 2, + 'a' => 3, ]); }); + it('keeps last across multiple non-adjacent duplicates', function (): void { + $repairer = new JsonRepairer('{a: 1, b: 2, a: 3, b: 4}', duplicateKeyPolicy: DuplicateKeyPolicy::KeepLast); + $result = $repairer->repair(); + + expect(json_validate($result))->toBeTrue(); + expect(json_decode($result, true))->toBe([ + 'a' => 3, + 'b' => 4, + ]); + }); + + it('does not corrupt large integers when keep-last is enabled', function (): void { + $repairer = new JsonRepairer( + '{big: 12345678901234567890, a: 1}', + duplicateKeyPolicy: DuplicateKeyPolicy::KeepLast, + ); + $result = $repairer->repair(); + + expect(json_validate($result))->toBeTrue(); + expect($result)->toContain('12345678901234567890'); + expect($result)->not->toContain('e+'); + }); + + it('preserves large integers while still deduplicating with keep-last', function (): void { + $repairer = new JsonRepairer( + '{big: 12345678901234567890, a: 1, a: 2}', + duplicateKeyPolicy: DuplicateKeyPolicy::KeepLast, + ); + $result = $repairer->repair(); + + expect(json_validate($result))->toBeTrue(); + expect($result)->toContain('12345678901234567890'); + expect($result)->not->toContain('e+'); + + $decoded = json_decode($result, true); + expect($decoded['a'])->toBe(2); + }); + it('keeps first when skipped duplicate value is a structure containing brackets in strings', function (): void { $repairer = new JsonRepairer( '{a: 1, a: {b: [1, 2], c: "}}"}, z: 9}', diff --git a/tests/Unit/ValuesAndStructuresTest.php b/tests/Unit/ValuesAndStructuresTest.php index 2ae948a..687cdeb 100644 --- a/tests/Unit/ValuesAndStructuresTest.php +++ b/tests/Unit/ValuesAndStructuresTest.php @@ -68,6 +68,31 @@ expect(json_decode(json_repair('[1, 2.5, 3]'), true))->toBe([1, 2.5, 3]); }); + it( + 'drops an incomplete exponent without removing the whole number', + function (string $input, int|float $expected): void { + $result = json_repair($input); + expect(json_validate($result))->toBeTrue(); + expect(json_decode($result, true)['key'])->toBe($expected); + }, + )->with([ + 'plus exponent no digits' => ['{"key": 1e+}', 1], + 'minus exponent no digits' => ['{"key": 12e-}', 12], + 'bare exponent no digits' => ['{"key": 1e}', 1], + 'plus exponent after decimal' => ['{"key": 1.5e+}', 1.5], + ]); + + it('drops an incomplete exponent inside an array', function (): void { + $result = json_repair('[1e+]'); + expect(json_validate($result))->toBeTrue(); + expect(json_decode($result, true))->toBe([1]); + }); + + it('preserves complete signed exponents', function (): void { + expect(json_decode(json_repair('{"key": 1e+5}'), true)['key'])->toBe(100000.0); + expect(json_decode(json_repair('{"key": 1.5e-3}'), true)['key'])->toBe(0.0015); + }); + it( 'handles strings with special characters', function (string $input, string $expectedKey, string $expectedValue): void {