diff --git a/Build/Scripts/checkIntegrityXliff.php b/Build/Scripts/checkIntegrityXliff.php new file mode 100644 index 000000000..53041d848 --- /dev/null +++ b/Build/Scripts/checkIntegrityXliff.php @@ -0,0 +1,304 @@ +findXliff(); + $output = new ConsoleOutput(); + $output->setFormatter(new OutputFormatter(true)); + + $testResults = []; + $errors = []; + + foreach ($filesToProcess as $labelFile) { + assert($labelFile instanceof \SplFileInfo); + $fullFilePath = $labelFile->getRealPath(); + $result = $this->checkValidLabels($fullFilePath); + if (isset($result['error'])) { + $errors['EXT:' . $result['extensionKey'] . ':' . $result['shortLabelFile']] = $result['error']; + } + $testResults[] = $result; + } + + if ($testResults === []) { + return 1; + } + + // Only show full table output if verbose is on + if ($isVerbose) { + $table = new Table($output); + $table->setHeaders(['EXT', 'File', 'Status', 'Errorcode']); + foreach ($testResults as $result) { + $table->addRow([ + $result['extensionKey'], + $result['shortLabelFile'], + (!isset($result['error']) ? "\xF0\x9F\x91\x8C" : "\xF0\x9F\x92\x80"), + $result['errorcode'] ?? '', + ]); + } + $table->setFooterTitle(count($testResults) . ' files, ' . count($errors) . ' Errors'); + $table->render(); + } else { + // Non-verbose: just show a summary line + if ($errors !== []) { + $output->writeln('' . count($errors) . ' error(s) found in ' . count($testResults) . ' files.'); + } + } + + if ($errors === []) { + return 0; + } + + // Only show detailed error table if verbose + if ($isVerbose) { + $output->writeln(''); + $table = new Table($output); + $table->setHeaders(['File', 'Error']); + foreach ($errors as $file => $errorMessage) { + $table->addRow([$file, $errorMessage]); + $table->addRow([new TableSeparator(), new TableSeparator()]); + } + $table->setColumnMaxWidth(0, 40); + $table->setColumnMaxWidth(1, 80); + $table->render(); + } else { + // Compact error summary + foreach ($errors as $file => $message) { + $output->writeln("$file: $message"); + } + } + + return 1; + } + + private function findXliff(): Finder + { + $finder = new Finder(); + return $finder + ->files() + ->in(__DIR__ . '/../../' . self::pathToXliffFiles . '/') + ->name('*.xlf'); + } + + private function checkValidLabels(string $labelFile): array + { + $extensionKey = 'N/A'; + $shortLabelFile = basename($labelFile); + if (preg_match('@/([a-z][a-z0-9_]*)/' . self::pathToXliffFiles . '/(.+)$@imsU', $labelFile, $matches)) { + $extensionKey = $matches[1]; + $shortLabelFile = $matches[2]; + } + + $result = [ + 'shortLabelFile' => $shortLabelFile, + 'extensionKey' => $extensionKey, + ]; + + $xml = simplexml_load_file($labelFile); + if ($xml === false) { + $result['error'] = 'XML not parsable'; + $result['errorcode'] = 'XML'; + return $result; + } + + $attributes = (array)$xml->attributes(); + $version = $attributes['@attributes']['version'] ?? ''; + $supportedVersions = ['1.2', '2.0']; + if (!in_array($version, $supportedVersions, true)) { + $result['error'] = 'Incompatible version: ' . $version . ' (expected: ' . implode(', ', + $supportedVersions) . ')'; + $result['errorcode'] = 'XLF version'; + return $result; + } + + $dom = XmlUtils::loadFile($labelFile, null); + $errors = XliffUtils::validateSchema($dom); + if ($errors) { + $result['error'] = sprintf('File %s has errors: ', $labelFile); + foreach ($errors as $error) { + $result['error'] .= ($error['message'] ?? '') . ' '; + } + $result['errorcode'] = 'XLF linting'; + return $result; + } + + $fileAttributes = (array)$xml->file->attributes(); + if ($version === '1.2') { + $namespaces = $xml->getNamespaces(true); + if (isset($namespaces[''])) { + // Normalize empty namespace to "xml" + $namespaces['xml'] = $namespaces['']; + unset($namespaces['']); + } + $ns = 'urn:oasis:names:tc:xliff:document:1.2'; + if ($namespaces !== ['xml' => $ns]) { + $result['error'] = 'Invalid XLIFF namespace: ' . json_encode($namespaces) . ' (expected: ' . $ns . ')'; + $result['errorcode'] = 'XML-NS'; + return $result; + } + $xml->registerXPathNamespace('x', $ns); + + $sourceLanguage = $fileAttributes['@attributes']['source-language'] ?? ''; + $datatype = $fileAttributes['@attributes']['datatype'] ?? ''; + $original = $fileAttributes['@attributes']['original'] ?? ''; + $date = $fileAttributes['@attributes']['date'] ?? ''; + + $isIso = ($extensionKey === 'core' && str_starts_with($shortLabelFile, 'Iso/')); + + if ($sourceLanguage !== 'en') { + $result['error'] = 'Invalid source-language: ' . $sourceLanguage; + $result['errorcode'] = 'file.source-language'; + return $result; + } + + if ($datatype !== 'plaintext') { + $result['error'] = 'Invalid datatype: ' . $datatype; + $result['errorcode'] = 'file.datatype'; + return $result; + } + + $expectedOriginals = [ + 'EXT:' . $extensionKey . '/' . self::pathToXliffFiles . '/' . $shortLabelFile, + 'messages', // @todo is this right? + ]; + + if ($isIso) { + $expectedOriginals[] = 'EXT:core/Resources/Private/Language/countries.xlf'; + } + if (!in_array($original, $expectedOriginals, true)) { + $result['error'] = 'Invalid original: ' . $original . ' (expected: ' . implode(', ', + $expectedOriginals) . ')'; + $result['errorcode'] = 'file.original'; + return $result; + } + + if (!$isIso && (strtotime($date) === false || strtotime($date) === 0)) { + $result['error'] = 'Invalid date: ' . $date; + $result['errorcode'] = 'file.date'; + return $result; + } + + // verify these are deprecated: + $transUnits = $xml->xpath('/x:xliff/x:file/x:body/x:trans-unit'); + $seenKeys = []; + foreach ($transUnits as $unit) { + $unitAttributes = (array)$unit; + $unitId = $unitAttributes['@attributes']['id'] ?? ''; + if ($unitId === '') { + $result['error'] = 'TransUnit without ID specified.'; + $result['errorcode'] = 'trans-unit'; + return $result; + } + if (isset($seenKeys[$unitId])) { + $result['error'] = 'Duplicate trans-unit id: ' . $unitId; + $result['errorcode'] = 'trans-unit.duplicate-id'; + return $result; + } + + if (in_array($unitId, self::expectedXliffDeprecations, true) + && ($unitAttributes['@attributes'][self::XliffDeprecationKey] ?? '') === '' + ) { + $result['error'] = 'TransUnit ' . $unitId . ' missing ' . self::XliffDeprecationKey . ' attribute.'; + $result['errorcode'] = 'trans-unit.' . self::XliffDeprecationKey; + return $result; + } + $seenKeys[$unitId] = $unitId; + } + + if (preg_match(self::xliffModuleRegularExpression, $labelFile)) { + // Hit on any "backend module file". + foreach (self::xliffModuleRequiredKeys as $requiredKey) { + if (!isset($seenKeys[$requiredKey])) { + $result['error'] = 'Backend module missing label ' . $requiredKey . '.'; + $result['errorcode'] = 'missing ' . $requiredKey; + return $result; + } + } + } + } else { + $fileId = $fileAttributes['@attributes']['id'] ?? ''; + if ($fileId === '') { + $result['error'] = 'Missing file.id'; + $result['errorcode'] = 'file.id'; + return $result; + } + + $ns = 'urn:oasis:names:tc:xliff:document:2.0'; + $xml->registerXPathNamespace('x', $ns); + + // In XLIFF 2.0, translatable content is in + $units = $xml->xpath('/x:xliff/x:file/x:unit'); + $seenUnitIds = []; + + foreach ($units as $unit) { + $attrs = $unit->attributes(); + $unitId = isset($attrs['id']) ? (string)$attrs['id'] : ''; + + if ($unitId === '') { + $result['error'] = 'Unit without ID specified.'; + $result['errorcode'] = 'unit'; + return $result; + } + + if (isset($seenUnitIds[$unitId])) { + $result['error'] = 'Duplicate unit id: ' . $unitId; + $result['errorcode'] = 'unit.duplicate-id'; + return $result; + } + + $seenUnitIds[$unitId] = true; + } + + // XLIFF 2.0 has no deprecation syntax check yet. + } + + // Currently, "locallang.xlf" and "messages.xlf" inside + // the same directory are not working, as only one gets parsed. + $labelFileName = basename($labelFile); + $labelDirName = dirname($labelFile); + if ($labelFileName === 'messages.xlf') { + if (file_exists($labelDirName . '/locallang.xlf') && file_exists($labelDirName . '/messages.xlf')) { + $result['error'] = 'Cannot have message.xlf AND locallang.xlf files in ' . $labelDirName; + $result['errorcode'] = 'file.locallang+messages'; + return $result; + } + } + + return $result; + } +} + +exit((new CheckIntegrityXliff())->execute($argv)); diff --git a/Build/Scripts/runTests.sh b/Build/Scripts/runTests.sh index fef616ea7..37029038c 100755 --- a/Build/Scripts/runTests.sh +++ b/Build/Scripts/runTests.sh @@ -185,6 +185,7 @@ Options: Specifies which script/tool to run - cgl: Fixes the code style with the PHP Coding Standards Fixer (PHP-CS-Fixer). Set -n for dry-run. - checkComposerNormalize: Checks the order of the composer.json entries. + - checkIntegrityXliff: checks for all xlf files for validity and deprecated usages - clean: clean up build, cache and testing related files and folders - cleanCache: clean up cache related files and folders - cleanRenderedDocumentation: clean up rendered documentation files and folders (Documentation-GENERATED-temp) @@ -203,7 +204,6 @@ Options: - lintJson: JSON linting - lintPhp: PHP linting - lintTypoScript: TypoScript linting - - lintXliff: XLIFF linting - lintYaml: YAML linting - npm: "npm" with all remaining arguments dispatched. - phpCsFixer fixes code to follow the standards. Set -n for dry-run. @@ -375,11 +375,6 @@ lintTypoScript() { ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name lintTypoScript-${SUFFIX} -e COMPOSER_CACHE_DIR=.cache/composer -e COMPOSER_ROOT_VERSION=${COMPOSER_ROOT_VERSION} ${IMAGE_PHP} /bin/sh -c "${COMMAND}" } -lintXliff() { - COMMAND="php Build/Scripts/xliffLint.sh lint:xliff Resources/Private/Language" - ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name lintXliff-${SUFFIX} ${IMAGE_PHP} ${COMMAND} -} - lintYaml() { COMMAND="composer check:yaml:lint" ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name lintYaml-${SUFFIX} -e COMPOSER_CACHE_DIR=.cache/composer -e COMPOSER_ROOT_VERSION=${COMPOSER_ROOT_VERSION} ${IMAGE_PHP} /bin/sh -c "${COMMAND}" @@ -610,6 +605,10 @@ case ${TEST_SUITE} in ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name composer-normalize-${SUFFIX} -e COMPOSER_CACHE_DIR=.cache/composer -e COMPOSER_ROOT_VERSION=${COMPOSER_ROOT_VERSION} ${IMAGE_PHP} /bin/sh -c "${COMMAND}" SUITE_EXIT_CODE=$? ;; + checkIntegrityXliff) + ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name check-integrity-set-labels-${SUFFIX} ${IMAGE_PHP} php -dxdebug.mode=off Build/Scripts/checkIntegrityXliff.php + SUITE_EXIT_CODE=$? + ;; clean) cleanCacheFiles cleanRenderedDocumentationFiles @@ -724,10 +723,6 @@ case ${TEST_SUITE} in lintTypoScript SUITE_EXIT_CODE=$? ;; - lintXliff) - lintXliff - SUITE_EXIT_CODE=$? - ;; lintYaml) lintYaml SUITE_EXIT_CODE=$? diff --git a/Build/Scripts/xliffLint.sh b/Build/Scripts/xliffLint.sh deleted file mode 100644 index 08b74be0c..000000000 --- a/Build/Scripts/xliffLint.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env php -add(new XliffLintCommand(null, null, null, false)); -$application->add(new LintCommand()); - -exit($application->run()); diff --git a/composer.json b/composer.json index 0d8980651..49c78df1c 100644 --- a/composer.json +++ b/composer.json @@ -66,7 +66,9 @@ "spaze/phpstan-disallowed-calls": "4.12.0", "ssch/typo3-rector": "2.15.2 || 3.14.1", "ssch/typo3-rector-testing-framework": "2.0.1 || 3.0.0", + "symfony/config": "^6.4 || ^7.4", "symfony/console": "^6.4 || ^7.4", + "symfony/finder": "^6.4 || ^7.4", "symfony/translation": "^6.4 || ^7.4", "symfony/yaml": "^6.4 || ^7.4", "tomasvotruba/cognitive-complexity": "0.2.3 || 1.1.1", @@ -160,7 +162,7 @@ "check:tests:create-directories": "mkdir -p .Build/public/typo3temp/var/tests", "check:tests:unit": "phpunit -c Build/phpunit/UnitTests.xml", "check:typoscript:lint": "typoscript-lint -c Build/typoscript-lint/config.yml --ansi -n --fail-on-warnings -vvv Configuration/TypoScript Tests/Functional/Controller/Fixtures/TypoScript", - "check:xliff:lint": "php Build/Scripts/xliffLint.sh lint:xliff Resources/Private/Language", + "check:xliff:lint": "php Build/Scripts/checkIntegrityXliff.php", "check:yaml:lint": "find . ! -path '*.Build/*' ! -path '*node_modules/*' \\( -name '*.yaml' -o -name '*.yml' \\) | xargs -r php ./.Build/bin/yaml-lint", "coverage:create-directories": "mkdir -p build/coverage build/logs", "fix": [