feat: provide --flatten option for hoisting selectors (#22)

Author: ramsey

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 
+- Provide `--flatten` extraction option to tell the extractor to hoist selectors and flatten sentences as much as possible. For example, `I have {count, plural, one{a dog} other{many dogs}}` becomes `{count, plural, one{I have a dog} other{I have many dogs}}`. The goal is to provide as many full sentences as possible, since fragmented sentences are not translator-friendly.
 - Provide `--add-missing-ids` extraction option to update source code with auto-generated identifiers
 - Add `Util\FormatHelper` that provides `getReader()` and `getWriter()` methods
 - Introduce `Format\Format` final static class for format constants

src/Console/Command/ExtractCommand.php

@@ -167,6 +167,16 @@ protected function configure(): void
                 'Whether to preserve whitespace and newlines in extracted '
                     . 'messages.',
             )
+            ->addOption(
+                'flatten',
+                null,
+                InputOption::VALUE_NONE,
+                'Whether to hoist selectors & flatten sentences as much as possible, '
+                    . 'e.g: "I have {count, plural, one{a dog} other{many dogs}}" '
+                    . 'becomes "{count, plural, one{I have a dog} other{I have many '
+                    . 'dogs}}". The goal is to provide as many full sentences as '
+                    . 'possible, since fragmented sentences are not translator-friendly.',
+            )
             ->addOption(
                 'add-missing-ids',
                 null,
@@ -239,6 +249,7 @@ private function buildOptions(InputInterface $input): MessageExtractorOptions
         $options->extractSourceLocation = (bool) $input->getOption('extract-source-location');
         $options->throws = (bool) $input->getOption('throws');
         $options->preserveWhitespace = (bool) $input->getOption('preserve-whitespace');
+        $options->flatten = (bool) $input->getOption('flatten');
         $options->addGeneratedIdsToSourceCode = (bool) $input->getOption('add-missing-ids');
 
         /** @var string $inputFunctionNames */

src/Descriptor.php

@@ -72,6 +72,11 @@ public function getDefaultMessage(): ?string
         return $this->defaultMessage;
     }
 
+    public function setDefaultMessage(string $defaultMessage): void
+    {
+        $this->defaultMessage = $defaultMessage;
+    }
+
     public function getDescription(): ?string
     {
         return $this->description;

src/Extractor/MessageExtractor.php

@@ -23,7 +23,9 @@
 namespace FormatPHP\Extractor;
 
 use Closure;
+use FormatPHP\Descriptor;
 use FormatPHP\DescriptorCollection;
+use FormatPHP\DescriptorInterface;
 use FormatPHP\Exception\FormatPHPExceptionInterface;
 use FormatPHP\Exception\ImproperContextException;
 use FormatPHP\Exception\InvalidArgumentException;
@@ -34,6 +36,9 @@
 use FormatPHP\Extractor\Parser\ParserErrorCollection;
 use FormatPHP\Format\WriterInterface;
 use FormatPHP\Format\WriterOptions;
+use FormatPHP\Icu\MessageFormat\Manipulator;
+use FormatPHP\Icu\MessageFormat\Parser as MessageFormatParser;
+use FormatPHP\Icu\MessageFormat\Printer;
 use FormatPHP\Util\FileSystemHelper;
 use FormatPHP\Util\FormatHelper;
 use FormatPHP\Util\Globber;
@@ -62,6 +67,8 @@ class MessageExtractor
     private MessageExtractorOptions $options;
     private ParserErrorCollection $errors;
     private FormatHelper $formatHelper;
+    private Manipulator $manipulator;
+    private Printer $printer;
 
     public function __construct(
         MessageExtractorOptions $options,
@@ -76,6 +83,8 @@ public function __construct(
         $this->file = $file;
         $this->formatHelper = $formatHelper;
         $this->errors = new ParserErrorCollection();
+        $this->manipulator = new Manipulator();
+        $this->printer = new Printer();
     }
 
     /**
@@ -210,6 +219,12 @@ private function loadDescriptorParser(string $parserNameOrScript): DescriptorPar
      */
     private function write(callable $formatter, DescriptorCollection $descriptors): void
     {
+        if ($this->options->flatten === true) {
+            /** @var DescriptorInterface[] $flattened */
+            $flattened = $descriptors->map($this->flattenMessage())->toArray();
+            $descriptors = new DescriptorCollection($flattened);
+        }
+
         $file = $this->options->outFile ?? 'php://output';
 
         $writerOptions = new WriterOptions();
@@ -251,4 +266,21 @@ public function __invoke(
             }
         };
     }
+
+    private function flattenMessage(): Closure
+    {
+        return function (Descriptor $descriptor): Descriptor {
+            $message = $descriptor->getDefaultMessage();
+            $messageFormatParser = new MessageFormatParser((string) $message);
+            $result = $messageFormatParser->parse();
+
+            /** @var MessageFormatParser\Type\ElementCollection $messageAst */
+            $messageAst = $result->val;
+
+            $hoistedAst = $this->manipulator->hoistSelectors($messageAst);
+            $descriptor->setDefaultMessage($this->printer->printAst($hoistedAst));
+
+            return $descriptor;
+        };
+    }
 }

src/Extractor/MessageExtractorOptions.php

@@ -22,6 +22,8 @@
 
 namespace FormatPHP\Extractor;
 
+use FormatPHP\Icu\MessageFormat\Manipulator;
+
 /**
  * MessageExtractor options
  */
@@ -77,6 +79,13 @@ class MessageExtractorOptions
      */
     public bool $preserveWhitespace = false;
 
+    /**
+     * Whether to hoist selectors and flatten sentences as much as possible
+     *
+     * @see Manipulator::hoistSelectors()
+     */
+    public bool $flatten = false;
+
     /**
      * Function and method names to parse from the application source code
      *

src/Icu/MessageFormat/Manipulator.php

@@ -0,0 +1,79 @@
+<?php
+
+/**
+ * This file is part of skillshare/formatphp
+ *
+ * skillshare/formatphp is open source software: you can distribute
+ * it and/or modify it under the terms of the MIT License
+ * (the "License"). You may not use this file except in
+ * compliance with the License.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ *
+ * @copyright Copyright (c) Skillshare, Inc. <https://www.skillshare.com>
+ * @license https://opensource.org/licenses/MIT MIT License
+ */
+
+declare(strict_types=1);
+
+namespace FormatPHP\Icu\MessageFormat;
+
+use FormatPHP\Icu\MessageFormat\Parser\Type\ElementCollection;
+use FormatPHP\Icu\MessageFormat\Parser\Type\PluralElement;
+use FormatPHP\Icu\MessageFormat\Parser\Type\SelectElement;
+
+use function array_slice;
+use function array_values;
+
+/**
+ * Provides functionality to manipulate a parsed AST
+ *
+ * @internal
+ */
+class Manipulator
+{
+    /**
+     * Hoist all selectors to the beginning of the AST & flatten the
+     * resulting options
+     *
+     * For example,
+     *
+     *     I have {count, plural, one{a dog} other{many dogs}}
+     *
+     * becomes
+     *
+     *     {count, plural, one{I have a dog} other{I have many dogs}}
+     *
+     * If there are multiple selectors, the order of which one is hoisted 1st
+     * is non-deterministic.
+     *
+     * The goal is to provide as many full sentences as possible since
+     * fragmented sentences are not translator-friendly.
+     */
+    public function hoistSelectors(ElementCollection $ast): ElementCollection
+    {
+        for ($i = 0; $i < $ast->count(); $i++) {
+            $element = $ast[$i];
+
+            if ($element instanceof PluralElement || $element instanceof SelectElement) {
+                $cloned = clone $element;
+                $options = $cloned->options;
+                foreach ($options as $option) {
+                    $option->value = $this->hoistSelectors(new ElementCollection([
+                        ...array_values(array_slice($ast->toArray(), 0, $i)),
+                        ...array_values($option->value->toArray()),
+                        ...array_values(array_slice($ast->toArray(), $i + 1)),
+                    ]));
+                }
+
+                return new ElementCollection([$cloned]);
+            }
+        }
+
+        return $ast;
+    }
+}

src/Icu/MessageFormat/Parser/Type/AbstractElement.php

@@ -24,7 +24,9 @@
 
 abstract class AbstractElement implements ElementInterface
 {
+    use DeepCloner;
+
     public ElementType $type;
-    public ?string $value = null;
-    public ?Location $location = null;
+    public string $value;
+    public Location $location;
 }

src/Icu/MessageFormat/Parser/Type/AbstractSkeleton.php

@@ -24,6 +24,8 @@
 
 abstract class AbstractSkeleton implements SkeletonInterface
 {
+    use DeepCloner;
+
     public SkeletonType $type;
     public Location $location;
 }

src/Icu/MessageFormat/Parser/Type/DeepCloner.php

@@ -0,0 +1,84 @@
+<?php
+
+/**
+ * This file is part of skillshare/formatphp
+ *
+ * skillshare/formatphp is open source software: you can distribute
+ * it and/or modify it under the terms of the MIT License
+ * (the "License"). You may not use this file except in
+ * compliance with the License.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ *
+ * @copyright Copyright (c) Skillshare, Inc. <https://www.skillshare.com>
+ * @license https://opensource.org/licenses/MIT MIT License
+ */
+
+declare(strict_types=1);
+
+namespace FormatPHP\Icu\MessageFormat\Parser\Type;
+
+use ReflectionObject;
+
+use function is_array;
+use function is_object;
+
+trait DeepCloner
+{
+    public function __clone()
+    {
+        $this->cloneMyProperties();
+    }
+
+    private function cloneMyProperties(): void
+    {
+        $reflection = new ReflectionObject($this);
+
+        foreach ($reflection->getProperties() as $reflectionProperty) {
+            /** @var mixed $propertyValue */
+            $propertyValue = $reflectionProperty->getValue($this);
+            $reflectionProperty->setValue($this, $this->cloneValue($propertyValue));
+        }
+    }
+
+    /**
+     * @param mixed $value
+     *
+     * @return mixed The cloned value
+     */
+    private function cloneValue($value)
+    {
+        if (is_array($value)) {
+            return $this->cloneArray($value);
+        }
+
+        if (is_object($value)) {
+            return clone $value;
+        }
+
+        return $value;
+    }
+
+    /**
+     * @param array<array-key, mixed> $value
+     *
+     * @return mixed[]
+     *
+     * @psalm-suppress MixedAssignment
+     */
+    private function cloneArray(array $value): array
+    {
+        /** @var mixed[] $clone */
+        $clone = [];
+
+        foreach ($value as $k => $v) {
+            $clone[$k] = $this->cloneValue($v);
+        }
+
+        return $clone;
+    }
+}

src/Icu/MessageFormat/Parser/Type/ElementCollection.php

@@ -65,4 +65,15 @@ public function jsonSerialize()
     {
         return $this->toArray();
     }
+
+    public function __clone()
+    {
+        $items = [];
+
+        foreach ($this->data as $datum) {
+            $items[] = clone $datum;
+        }
+
+        $this->data = $items;
+    }
 }