[FEATURE] Use PHP script for XLIFF check in runTests.sh

Build/Scripts/checkIntegrityXliff.php

@@ -0,0 +1,304 @@
+<?php
+
+declare(strict_types=1);
+
+use Symfony\Component\Config\Util\XmlUtils;
+use Symfony\Component\Console\Formatter\OutputFormatter;
+use Symfony\Component\Console\Helper\Table;
+use Symfony\Component\Console\Helper\TableSeparator;
+use Symfony\Component\Console\Output\ConsoleOutput;
+use Symfony\Component\Finder\Finder;
+use Symfony\Component\Translation\Util\XliffUtils;
+
+require __DIR__ . '/../../.Build/vendor/autoload.php';
+
+if (PHP_SAPI !== 'cli') {
+    die('Script must be called from command line.' . chr(10));
+}
+
+final class CheckIntegrityXliff
+{
+    private const expectedXliffDeprecations = [
+        'mlang_labels_tablabel',
+        'mlang_labels_tabdescr',
+        'mlang_tabs_tab',
+    ];
+    private const xliffModuleRegularExpression = '@Language/(module\.xlf|Modules/.+\.xlf)$@i';
+    private const xliffModuleRequiredKeys = [
+        'title',
+        'description',
+        'short_description',
+    ];
+    private const XliffDeprecationKey = 'x-unused-since';
+    private const pathToXliffFiles = 'Resources/Private/Language';
+
+    public function execute(array $argv = []): int
+    {
+        $isVerbose = in_array('-v', $argv, true) || in_array('--verbose', $argv, true);
+
+        $filesToProcess = $this->findXliff();
+        $output = new ConsoleOutput();
+        $output->setFormatter(new OutputFormatter(true));
+
+        $testResults = [];
+        $errors = [];
+
+        foreach ($filesToProcess as $labelFile) {
+            assert($labelFile instanceof \SplFileInfo);
+            $fullFilePath = $labelFile->getRealPath();
+            $result = $this->checkValidLabels($fullFilePath);
+            if (isset($result['error'])) {
+                $errors['EXT:' . $result['extensionKey'] . ':' . $result['shortLabelFile']] = $result['error'];
+            }
+            $testResults[] = $result;
+        }
+
+        if ($testResults === []) {
+            return 1;
+        }
+
+        // Only show full table output if verbose is on
+        if ($isVerbose) {
+            $table = new Table($output);
+            $table->setHeaders(['EXT', 'File', 'Status', 'Errorcode']);
+            foreach ($testResults as $result) {
+                $table->addRow([
+                    $result['extensionKey'],
+                    $result['shortLabelFile'],
+                    (!isset($result['error']) ? "\xF0\x9F\x91\x8C" : "\xF0\x9F\x92\x80"),
+                    $result['errorcode'] ?? '',
+                ]);
+            }
+            $table->setFooterTitle(count($testResults) . ' files, ' . count($errors) . ' Errors');
+            $table->render();
+        } else {
+            // Non-verbose: just show a summary line
+            if ($errors !== []) {
+                $output->writeln('<error>' . count($errors) . ' error(s) found in ' . count($testResults) . ' files.</error>');
+            }
+        }
+
+        if ($errors === []) {
+            return 0;
+        }
+
+        // Only show detailed error table if verbose
+        if ($isVerbose) {
+            $output->writeln('');
+            $table = new Table($output);
+            $table->setHeaders(['File', 'Error']);
+            foreach ($errors as $file => $errorMessage) {
+                $table->addRow([$file, $errorMessage]);
+                $table->addRow([new TableSeparator(), new TableSeparator()]);
+            }
+            $table->setColumnMaxWidth(0, 40);
+            $table->setColumnMaxWidth(1, 80);
+            $table->render();
+        } else {
+            // Compact error summary
+            foreach ($errors as $file => $message) {
+                $output->writeln("<error>$file:</error> $message");
+            }
+        }
+
+        return 1;
+    }
+
+    private function findXliff(): Finder
+    {
+        $finder = new Finder();
+        return $finder
+            ->files()
+            ->in(__DIR__ . '/../../' . self::pathToXliffFiles . '/')
+            ->name('*.xlf');
+    }
+
+    private function checkValidLabels(string $labelFile): array
+    {
+        $extensionKey = 'N/A';
+        $shortLabelFile = basename($labelFile);
+        if (preg_match('@/([a-z][a-z0-9_]*)/' . self::pathToXliffFiles . '/(.+)$@imsU', $labelFile, $matches)) {
+            $extensionKey = $matches[1];
+            $shortLabelFile = $matches[2];
+        }
+
+        $result = [
+            'shortLabelFile' => $shortLabelFile,
+            'extensionKey' => $extensionKey,
+        ];
+
+        $xml = simplexml_load_file($labelFile);
+        if ($xml === false) {
+            $result['error'] = 'XML not parsable';
+            $result['errorcode'] = 'XML';
+            return $result;
+        }
+
+        $attributes = (array)$xml->attributes();
+        $version = $attributes['@attributes']['version'] ?? '';
+        $supportedVersions = ['1.2', '2.0'];
+        if (!in_array($version, $supportedVersions, true)) {
+            $result['error'] = 'Incompatible version: ' . $version . ' (expected: ' . implode(', ',
+                    $supportedVersions) . ')';
+            $result['errorcode'] = 'XLF version';
+            return $result;
+        }
+
+        $dom = XmlUtils::loadFile($labelFile, null);
+        $errors = XliffUtils::validateSchema($dom);
+        if ($errors) {
+            $result['error'] = sprintf('File %s has errors: ', $labelFile);
+            foreach ($errors as $error) {
+                $result['error'] .= ($error['message'] ?? '') . ' ';
+            }
+            $result['errorcode'] = 'XLF linting';
+            return $result;
+        }
+
+        $fileAttributes = (array)$xml->file->attributes();
+        if ($version === '1.2') {
+            $namespaces = $xml->getNamespaces(true);
+            if (isset($namespaces[''])) {
+                // Normalize empty namespace to "xml"
+                $namespaces['xml'] = $namespaces[''];
+                unset($namespaces['']);
+            }
+            $ns = 'urn:oasis:names:tc:xliff:document:1.2';
+            if ($namespaces !== ['xml' => $ns]) {
+                $result['error'] = 'Invalid XLIFF namespace: ' . json_encode($namespaces) . ' (expected: ' . $ns . ')';
+                $result['errorcode'] = 'XML-NS';
+                return $result;
+            }
+            $xml->registerXPathNamespace('x', $ns);
+
+            $sourceLanguage = $fileAttributes['@attributes']['source-language'] ?? '';
+            $datatype = $fileAttributes['@attributes']['datatype'] ?? '';
+            $original = $fileAttributes['@attributes']['original'] ?? '';
+            $date = $fileAttributes['@attributes']['date'] ?? '';
+
+            $isIso = ($extensionKey === 'core' && str_starts_with($shortLabelFile, 'Iso/'));
+
+            if ($sourceLanguage !== 'en') {
+                $result['error'] = 'Invalid source-language: ' . $sourceLanguage;
+                $result['errorcode'] = 'file.source-language';
+                return $result;
+            }
+
+            if ($datatype !== 'plaintext') {
+                $result['error'] = 'Invalid datatype: ' . $datatype;
+                $result['errorcode'] = 'file.datatype';
+                return $result;
+            }
+
+            $expectedOriginals = [
+                'EXT:' . $extensionKey . '/' . self::pathToXliffFiles . '/' . $shortLabelFile,
+                'messages', // @todo is this right?
+            ];
+
+            if ($isIso) {
+                $expectedOriginals[] = 'EXT:core/Resources/Private/Language/countries.xlf';
+            }
+            if (!in_array($original, $expectedOriginals, true)) {
+                $result['error'] = 'Invalid original: ' . $original . ' (expected: ' . implode(', ',
+                        $expectedOriginals) . ')';
+                $result['errorcode'] = 'file.original';
+                return $result;
+            }
+
+            if (!$isIso && (strtotime($date) === false || strtotime($date) === 0)) {
+                $result['error'] = 'Invalid date: ' . $date;
+                $result['errorcode'] = 'file.date';
+                return $result;
+            }
+
+            // verify these are deprecated:
+            $transUnits = $xml->xpath('/x:xliff/x:file/x:body/x:trans-unit');
+            $seenKeys = [];
+            foreach ($transUnits as $unit) {
+                $unitAttributes = (array)$unit;
+                $unitId = $unitAttributes['@attributes']['id'] ?? '';
+                if ($unitId === '') {
+                    $result['error'] = 'TransUnit without ID specified.';
+                    $result['errorcode'] = 'trans-unit';
+                    return $result;
+                }
+                if (isset($seenKeys[$unitId])) {
+                    $result['error'] = 'Duplicate trans-unit id: ' . $unitId;
+                    $result['errorcode'] = 'trans-unit.duplicate-id';
+                    return $result;
+                }
+
+                if (in_array($unitId, self::expectedXliffDeprecations, true)
+                    && ($unitAttributes['@attributes'][self::XliffDeprecationKey] ?? '') === ''
+                ) {
+                    $result['error'] = 'TransUnit ' . $unitId . ' missing ' . self::XliffDeprecationKey . ' attribute.';
+                    $result['errorcode'] = 'trans-unit.' . self::XliffDeprecationKey;
+                    return $result;
+                }
+                $seenKeys[$unitId] = $unitId;
+            }
+
+            if (preg_match(self::xliffModuleRegularExpression, $labelFile)) {
+                // Hit on any "backend module file".
+                foreach (self::xliffModuleRequiredKeys as $requiredKey) {
+                    if (!isset($seenKeys[$requiredKey])) {
+                        $result['error'] = 'Backend module missing label ' . $requiredKey . '.';
+                        $result['errorcode'] = 'missing ' . $requiredKey;
+                        return $result;
+                    }
+                }
+            }
+        } else {
+            $fileId = $fileAttributes['@attributes']['id'] ?? '';
+            if ($fileId === '') {
+                $result['error'] = 'Missing file.id';
+                $result['errorcode'] = 'file.id';
+                return $result;
+            }
+
+            $ns = 'urn:oasis:names:tc:xliff:document:2.0';
+            $xml->registerXPathNamespace('x', $ns);
+
+            // In XLIFF 2.0, translatable content is in <unit id="...">
+            $units = $xml->xpath('/x:xliff/x:file/x:unit');
+            $seenUnitIds = [];
+
+            foreach ($units as $unit) {
+                $attrs = $unit->attributes();
+                $unitId = isset($attrs['id']) ? (string)$attrs['id'] : '';
+
+                if ($unitId === '') {
+                    $result['error'] = 'Unit without ID specified.';
+                    $result['errorcode'] = 'unit';
+                    return $result;
+                }
+
+                if (isset($seenUnitIds[$unitId])) {
+                    $result['error'] = 'Duplicate unit id: ' . $unitId;
+                    $result['errorcode'] = 'unit.duplicate-id';
+                    return $result;
+                }
+
+                $seenUnitIds[$unitId] = true;
+            }
+
+            // XLIFF 2.0 has no deprecation syntax check yet.
+        }
+
+        // Currently, "locallang.xlf" and "messages.xlf" inside
+        // the same directory are not working, as only one gets parsed.
+        $labelFileName = basename($labelFile);
+        $labelDirName = dirname($labelFile);
+        if ($labelFileName === 'messages.xlf') {
+            if (file_exists($labelDirName . '/locallang.xlf') && file_exists($labelDirName . '/messages.xlf')) {
+                $result['error'] = 'Cannot have message.xlf AND locallang.xlf files in ' . $labelDirName;
+                $result['errorcode'] = 'file.locallang+messages';
+                return $result;
+            }
+        }
+
+        return $result;
+    }
+}
+
+exit((new CheckIntegrityXliff())->execute($argv));

Build/Scripts/runTests.sh

@@ -185,6 +185,7 @@ Options:
         Specifies which script/tool to run
             - cgl: Fixes the code style with the PHP Coding Standards Fixer (PHP-CS-Fixer). Set -n for dry-run.
             - checkComposerNormalize: Checks the order of the composer.json entries.
+            - checkIntegrityXliff: checks for all xlf files for validity and deprecated usages
             - clean: clean up build, cache and testing related files and folders
             - cleanCache: clean up cache related files and folders
             - cleanRenderedDocumentation: clean up rendered documentation files and folders (Documentation-GENERATED-temp)
@@ -203,7 +204,6 @@ Options:
             - lintJson: JSON linting
             - lintPhp: PHP linting
             - lintTypoScript: TypoScript linting
-            - lintXliff: XLIFF linting
             - lintYaml: YAML linting
             - npm: "npm" with all remaining arguments dispatched.
             - phpCsFixer fixes code to follow the standards. Set -n for dry-run.
@@ -375,11 +375,6 @@ lintTypoScript() {
     ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name lintTypoScript-${SUFFIX} -e COMPOSER_CACHE_DIR=.cache/composer -e COMPOSER_ROOT_VERSION=${COMPOSER_ROOT_VERSION} ${IMAGE_PHP} /bin/sh -c "${COMMAND}"
 }
 
-lintXliff() {
-    COMMAND="php Build/Scripts/xliffLint.sh lint:xliff Resources/Private/Language"
-    ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name lintXliff-${SUFFIX} ${IMAGE_PHP} ${COMMAND}
-}
-
 lintYaml() {
     COMMAND="composer check:yaml:lint"
     ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name lintYaml-${SUFFIX} -e COMPOSER_CACHE_DIR=.cache/composer -e COMPOSER_ROOT_VERSION=${COMPOSER_ROOT_VERSION} ${IMAGE_PHP} /bin/sh -c "${COMMAND}"
@@ -610,6 +605,10 @@ case ${TEST_SUITE} in
         ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name composer-normalize-${SUFFIX} -e COMPOSER_CACHE_DIR=.cache/composer -e COMPOSER_ROOT_VERSION=${COMPOSER_ROOT_VERSION} ${IMAGE_PHP} /bin/sh -c "${COMMAND}"
         SUITE_EXIT_CODE=$?
         ;;
+    checkIntegrityXliff)
+        ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name check-integrity-set-labels-${SUFFIX} ${IMAGE_PHP} php -dxdebug.mode=off Build/Scripts/checkIntegrityXliff.php
+        SUITE_EXIT_CODE=$?
+        ;;
     clean)
         cleanCacheFiles
         cleanRenderedDocumentationFiles
@@ -724,10 +723,6 @@ case ${TEST_SUITE} in
         lintTypoScript
         SUITE_EXIT_CODE=$?
         ;;
-    lintXliff)
-        lintXliff
-        SUITE_EXIT_CODE=$?
-        ;;
     lintYaml)
         lintYaml
         SUITE_EXIT_CODE=$?

composer.json

@@ -66,7 +66,9 @@
 		"spaze/phpstan-disallowed-calls": "4.12.0",
 		"ssch/typo3-rector": "2.15.2 || 3.14.1",
 		"ssch/typo3-rector-testing-framework": "2.0.1 || 3.0.0",
+		"symfony/config": "^6.4 || ^7.4",
 		"symfony/console": "^6.4 || ^7.4",
+		"symfony/finder": "^6.4 || ^7.4",
 		"symfony/translation": "^6.4 || ^7.4",
 		"symfony/yaml": "^6.4 || ^7.4",
 		"tomasvotruba/cognitive-complexity": "0.2.3 || 1.1.1",
@@ -160,7 +162,7 @@
 		"check:tests:create-directories": "mkdir -p .Build/public/typo3temp/var/tests",
 		"check:tests:unit": "phpunit -c Build/phpunit/UnitTests.xml",
 		"check:typoscript:lint": "typoscript-lint -c Build/typoscript-lint/config.yml --ansi -n --fail-on-warnings -vvv Configuration/TypoScript Tests/Functional/Controller/Fixtures/TypoScript",
-		"check:xliff:lint": "php Build/Scripts/xliffLint.sh lint:xliff Resources/Private/Language",
+		"check:xliff:lint": "php Build/Scripts/checkIntegrityXliff.php",
 		"check:yaml:lint": "find . ! -path '*.Build/*' ! -path '*node_modules/*' \\( -name '*.yaml' -o -name '*.yml' \\) | xargs -r php ./.Build/bin/yaml-lint",
 		"coverage:create-directories": "mkdir -p build/coverage build/logs",
 		"fix": [