feat: allow passing custom parsers to the extractor

Author: ramsey

src/ConfigInterface.php

@@ -23,7 +23,6 @@
 namespace FormatPHP;
 
 use FormatPHP\Extractor\IdInterpolator;
-use FormatPHP\Extractor\IdInterpolatorOptions;
 use FormatPHP\Intl\LocaleInterface;
 
 /**
@@ -39,32 +38,7 @@ public function getDefaultLocale(): ?LocaleInterface;
     /**
      * Returns a pattern that defines how to generate missing message IDs
      *
-     * If message descriptors are missing the id property, we will use this
-     * pattern to automatically generate IDs for them.
-     *
-     * The pattern follows this format:
-     *
-     *     [hashAlgorithm:contenthash:encodingAlgorithm:length]
-     *
-     * When passing this value, provide the hashAlgorithm, encodingAlgorithm,
-     * and length, and formatphp will calculate the contenthash.
-     *
-     * For example, if you wish to use `haval160,4` as the hashing algorithm,
-     * `hex` as the encoding algorithm, with a length of 10, you would pass
-     * the following string:
-     *
-     *     [haval160,4:contenthash:hex:10]
-     *
-     * See <https://www.php.net/hash_algos> for available hashing algorithms.
-     *
-     * The following binary-to-text encodings are supported:
-     *
-     * - `base64`
-     * - `base64url`
-     * - `hex`
-     *
      * @see IdInterpolator
-     * @see IdInterpolatorOptions
      */
     public function getIdInterpolatorPattern(): string;
 

src/Console/Command/ExtractCommand.php

@@ -39,6 +39,7 @@
 
 use function array_map;
 use function array_merge;
+use function array_unique;
 use function explode;
 use function getcwd;
 
@@ -47,8 +48,6 @@
  */
 class ExtractCommand extends AbstractCommand
 {
-    private const DEFAULT_FUNCTION_NAMES = ['formatMessage'];
-
     private const STANDARD_IGNORES = [
         '.arch-params',
         '.bzr',
@@ -130,6 +129,13 @@ protected function configure(): void
                 'Comma-separated list of additional function names to search '
                     . 'for when extracting messages.',
             )
+            ->addOption(
+                'parser',
+                'p',
+                InputOption::VALUE_IS_ARRAY | InputOption::VALUE_REQUIRED,
+                'Parser name or path to a parser script to apply additional '
+                    . 'parsing in addition to the default PHP parsing.',
+            )
             ->addOption(
                 'ignore',
                 null,
@@ -201,6 +207,10 @@ private function buildOptions(InputInterface $input): MessageExtractorOptions
         $idInterpolationPattern = $input->getOption('id-interpolation-pattern');
         $options->idInterpolationPattern = $idInterpolationPattern ?? IdInterpolator::DEFAULT_ID_INTERPOLATION_PATTERN;
 
+        /** @var string[] $parsers */
+        $parsers = (array) $input->getOption('parser');
+        $options->parsers = array_unique(array_merge($options->parsers, $parsers));
+
         /** @var string[] $ignore */
         $ignore = (array) $input->getOption('ignore');
         $options->ignore = array_merge(self::STANDARD_IGNORES, $ignore);
@@ -213,12 +223,10 @@ private function buildOptions(InputInterface $input): MessageExtractorOptions
         $options->throws = (bool) $input->getOption('throws');
         $options->preserveWhitespace = (bool) $input->getOption('preserve-whitespace');
 
-        /** @var string $additionalFunctionNames */
-        $additionalFunctionNames = $input->getOption('additional-function-names') ?? '';
-        $options->additionalFunctionNames = array_merge(
-            self::DEFAULT_FUNCTION_NAMES,
-            array_map('trim', explode(',', $additionalFunctionNames)),
-        );
+        /** @var string $inputFunctionNames */
+        $inputFunctionNames = $input->getOption('additional-function-names') ?? '';
+        $additionalFunctionNames = array_map('trim', explode(',', $inputFunctionNames));
+        $options->functionNames = array_unique(array_merge($options->functionNames, $additionalFunctionNames));
 
         return $options;
     }

src/Extractor/IdInterpolator.php

@@ -23,7 +23,6 @@
 namespace FormatPHP\Extractor;
 
 use Closure;
-use FormatPHP\ConfigInterface;
 use FormatPHP\DescriptorInterface;
 use FormatPHP\Exception\InvalidArgumentException;
 use FormatPHP\Exception\UnableToGenerateMessageIdException;
@@ -46,7 +45,29 @@
 /**
  * IdInterpolator supports generation of message descriptor IDs
  *
- * @see ConfigInterface::getIdInterpolatorPattern()
+ * If message descriptors are missing the id property, we will use this
+ * pattern to automatically generate IDs for them.
+ *
+ * The pattern follows this format:
+ *
+ *     [hashAlgorithm:contenthash:encodingAlgorithm:length]
+ *
+ * When passing this value, provide the hashAlgorithm, encodingAlgorithm,
+ * and length, and formatphp will calculate the contenthash.
+ *
+ * For example, if you wish to use `haval160,4` as the hashing algorithm,
+ * `hex` as the encoding algorithm, with a length of 10, you would pass
+ * the following string:
+ *
+ *     [haval160,4:contenthash:hex:10]
+ *
+ * See <https://www.php.net/hash_algos> for available hashing algorithms.
+ *
+ * The following binary-to-text encodings are supported:
+ *
+ * - `base64`
+ * - `base64url`
+ * - `hex`
  */
 class IdInterpolator
 {
@@ -62,8 +83,6 @@ class IdInterpolator
      * If the message descriptor does not have a default message, we cannot
      * generate an ID, so we throw `UnableToGenerateMessageId`.
      *
-     * @see ConfigInterface::getIdInterpolatorPattern()
-     *
      * @throws InvalidArgumentException
      * @throws UnableToGenerateMessageIdException
      */

src/Extractor/IdInterpolatorOptions.php

@@ -22,21 +22,30 @@
 
 namespace FormatPHP\Extractor;
 
-use FormatPHP\ConfigInterface;
-
 /**
  * IdInterpolator options
  *
- * @see ConfigInterface::getIdInterpolatorPattern()
+ * @see IdInterpolator
  */
 class IdInterpolatorOptions
 {
     private const DEFAULT_HASHING_ALGORITHM = 'sha512';
     private const DEFAULT_ENCODING_ALGORITHM = 'base64';
     private const DEFAULT_LENGTH = 6;
 
+    /**
+     * The hashing algorithm to use when creating the ID
+     */
     public string $hashingAlgorithm;
+
+    /**
+     * The encoding algorithm to use to encode the ID
+     */
     public string $encodingAlgorithm;
+
+    /**
+     * The length of the ID; if the generated ID is longer than this, it will be truncated
+     */
     public int $length;
 
     public function __construct(

src/Extractor/MessageExtractor.php

@@ -31,6 +31,7 @@
 use FormatPHP\Exception\UnableToWriteFileException;
 use FormatPHP\Extractor\Parser\Descriptor\PhpParser;
 use FormatPHP\Extractor\Parser\DescriptorParserInterface;
+use FormatPHP\Extractor\Parser\ParserErrorCollection;
 use FormatPHP\Format\Writer\FormatPHPWriter;
 use FormatPHP\Format\Writer\SimpleWriter;
 use FormatPHP\Format\Writer\SmartlingWriter;
@@ -67,11 +68,7 @@ class MessageExtractor
     private Globber $globber;
     private LoggerInterface $logger;
     private MessageExtractorOptions $options;
-
-    /**
-     * @var DescriptorParserInterface[]
-     */
-    private array $parsers;
+    private ParserErrorCollection $errors;
 
     /**
      * @throws LogicException
@@ -86,7 +83,7 @@ public function __construct(
         $this->logger = $logger;
         $this->globber = $globber;
         $this->file = $file;
-        $this->parsers = $this->loadParsers();
+        $this->errors = new ParserErrorCollection();
     }
 
     /**
@@ -135,14 +132,19 @@ public function process(array $files): void
         $this->writeOutput($this->prepareOutput($formatter, $descriptors));
     }
 
+    public function getErrors(): ParserErrorCollection
+    {
+        return $this->errors;
+    }
+
     /**
      * @throws UnableToProcessFileException
      */
     private function parse(DescriptorCollection $descriptors, string $filePath): DescriptorCollection
     {
-        foreach ($this->parsers as $parser) {
+        foreach ($this->getDescriptorParsers() as $parser) {
             /** @var DescriptorCollection $descriptors */
-            $descriptors = $descriptors->merge($parser->parse($filePath));
+            $descriptors = $descriptors->merge($parser($filePath, $this->options, $this->errors));
         }
 
         return $descriptors;
@@ -153,25 +155,50 @@ private function parse(DescriptorCollection $descriptors, string $filePath): Des
      *
      * @throws LogicException
      */
-    private function loadParsers(): array
+    private function getDescriptorParsers(): array
     {
         $parsers = [];
-        $parsers[] = $this->getPhpParser();
+
+        foreach ($this->options->parsers as $parser) {
+            $parsers[] = $this->loadDescriptorParser($parser);
+        }
 
         return $parsers;
     }
 
     /**
      * @throws LogicException
      */
-    private function getPhpParser(): PhpParser
+    private function loadDescriptorParser(string $parserNameOrScript): DescriptorParserInterface
     {
-        return new PhpParser(
-            $this->file,
-            $this->options->additionalFunctionNames,
-            $this->options->pragma,
-            $this->options->preserveWhitespace,
-        );
+        switch (strtolower($parserNameOrScript)) {
+            case 'php':
+                return new PhpParser($this->file);
+        }
+
+        if (class_exists($parserNameOrScript) && is_a($parserNameOrScript, DescriptorParserInterface::class, true)) {
+            $parser = new $parserNameOrScript();
+        } else {
+            /** @var Closure(string,MessageExtractorOptions,ParserErrorCollection):DescriptorCollection | null $parser */
+            $parser = $this->file->loadClosureFromScript($parserNameOrScript);
+        }
+
+        if ($parser instanceof DescriptorParserInterface) {
+            return $parser;
+        }
+
+        if (is_callable($parser)) {
+            return $this->createInvokableDescriptorParser($parser);
+        }
+
+        throw new InvalidArgumentException(sprintf(
+            'The parser provided is not a known descriptor parser, an instance of '
+            . '%s, or a callable of the shape `callable(string,%s,%s):%s`.',
+            DescriptorParserInterface::class,
+            MessageExtractorOptions::class,
+            ParserErrorCollection::class,
+            DescriptorCollection::class,
+        ));
     }
 
     /**
@@ -256,4 +283,25 @@ private function writeOutput(string $output): void
 
         $this->file->writeContents($stream, $output);
     }
+
+    private function createInvokableDescriptorParser(callable $parser): DescriptorParserInterface
+    {
+        return new class ($parser) implements DescriptorParserInterface {
+            private Closure $parser;
+
+            public function __construct(callable $parser)
+            {
+                $this->parser = Closure::fromCallable($parser);
+            }
+
+            public function __invoke(
+                string $filePath,
+                MessageExtractorOptions $options,
+                ParserErrorCollection $errors
+            ): DescriptorCollection {
+                /** @var DescriptorCollection */
+                return ($this->parser)($filePath, $options, $errors);
+            }
+        };
+    }
 }

src/Extractor/MessageExtractorOptions.php

@@ -27,23 +27,74 @@
  */
 class MessageExtractorOptions
 {
+    private const DEFAULT_FUNCTION_NAMES = ['formatMessage'];
+    private const DEFAULT_PARSERS = ['php'];
+
+    /**
+     * Formatter name or path to a formatter script that controls the shape
+     * of the JSON produced for $outFile
+     */
     public ?string $format = null;
+
+    /**
+     * Target file path to save the JSON output file of all translations
+     * extracted from the files
+     */
     public ?string $outFile = null;
+
+    /**
+     * If message descriptors are missing the id property, we will use this
+     * pattern to automatically generate IDs for them
+     *
+     * @see IdInterpolator
+     */
     public string $idInterpolationPattern = IdInterpolator::DEFAULT_ID_INTERPOLATION_PATTERN;
+
+    /**
+     * Whether to extract metadata for the source files
+     *
+     * If true, the extracted descriptors will each include `file`, `start`,
+     * `end`, `line`, and `col` properties.
+     */
     public bool $extractSourceLocation = false;
+
+    /**
+     * Whether to throw an exception when failing to process any file in the batch
+     *
+     * The default is to emit warnings, while continuing to process the rest
+     * of the files.
+     */
     public bool $throws = false;
+
+    /**
+     * Allows parsing of additional custom pragma to include custom metadata in
+     * the extracted messages
+     */
     public ?string $pragma = null;
+
+    /**
+     * Whether to preserve whitespace and newlines in extracted messages
+     */
     public bool $preserveWhitespace = false;
 
     /**
+     * Function and method names to parse from the application source code
+     *
      * @var string[]
      */
-    public array $additionalFunctionNames = [];
+    public array $functionNames = self::DEFAULT_FUNCTION_NAMES;
 
     /**
      * Glob file path patterns to ignore
      *
      * @var string[]
      */
     public array $ignore = [];
+
+    /**
+     * Parsers to use for extracting format messages from application source code
+     *
+     * @var string[]
+     */
+    public array $parsers = self::DEFAULT_PARSERS;
 }