Merge pull request #134 from transifex/i18next

Add i18next backend support

Author: Nikos Vasileiou

README.md

@@ -30,6 +30,11 @@ Transifex Native support for localizing Vue components.
 Transifex Native support for localizing ExpressJS applications.
 [Read more](https://github.com/transifex/transifex-javascript/tree/master/packages/express)
 
+## Transifex Native for i18next
+
+Transifex Native backend support for i18next
+[Read more](https://github.com/transifex/transifex-javascript/tree/master/packages/i18next)
+
 ## Transifex Native CLI
 
 Command line tool for extracting phrases from source files and pushing content to Transifex.
@@ -51,7 +56,6 @@ A javascript library for building SDKs for APIs that implement the {json:api}
 specification. This is what our API SDK is based on.
 [Read more](https://github.com/transifex/transifex-javascript/tree/master/packages/jsonapi)
 
-
 # License
 
 Licensed under Apache License 2.0, see [LICENSE](LICENSE) file.

packages/api/.eslintrc.json

@@ -7,7 +7,7 @@
   },
   "parser": "babel-eslint",
   "parserOptions": {
-      "ecmaVersion": 12,
-      "sourceType": "module"
+    "ecmaVersion": 12,
+    "sourceType": "module"
   }
 }

packages/cli/README.md

@@ -113,6 +113,7 @@ OPTIONS
   --dry-run                              dry run, do not push to Transifex
   --key-generator=source|hash            [default: source] use hashed or source based keys
   --no-wait                              disable polling for upload results
+  --parser=auto|i18next                  [default: auto] file parser to use
   --purge                                purge content on Transifex
   --secret=secret                        native project secret
   --token=token                          native project public token
@@ -145,6 +146,7 @@ DESCRIPTION
   txjs-cli push --with-tags-only="home,error"
   txjs-cli push --without-tags-only="custom"
   txjs-cli push --token=mytoken --secret=mysecret
+  txjs-cli push en.json --parser=i18next
   TRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli push
 ```
 

packages/cli/src/api/extract.js

@@ -9,6 +9,7 @@ const pug = require('pug');
 const { parseHTMLTemplateFile } = require('./parsers/angularHTML');
 const { babelExtractPhrases } = require('./parsers/babel');
 const { extractVuePhrases } = require('./parsers/vue');
+const { extractI18NextPhrases } = require('./parsers/i18next');
 
 /**
  * Parse file and extract phrases using AST
@@ -20,29 +21,49 @@ const { extractVuePhrases } = require('./parsers/vue');
  * @param {String[]} options.filterWithTags
  * @param {String[]} options.filterWithoutTags
  * @param {Boolean} options.useHashedKeys
+ * @param {String} options.parser
  * @returns {Object}
  */
 function extractPhrases(file, relativeFile, options = {}) {
   const HASHES = {};
   let source = fs.readFileSync(file, 'utf8');
 
-  // Handle simple templates that compile to javascript
+  // i18next JSON
+  if (options.parser === 'i18next') {
+    extractI18NextPhrases(HASHES, source, relativeFile, options);
+    return HASHES;
+  }
+
+  // PUBG templates
   if (path.extname(file) === '.pug') {
     source = pug.compileClient(source);
-  } else if (path.extname(file) === '.ejs') {
+    babelExtractPhrases(HASHES, source, relativeFile, options);
+    return HASHES;
+  }
+
+  // EJS templates
+  if (path.extname(file) === '.ejs') {
     const template = new ejs.Template(source);
     template.generateSource();
     source = template.source;
+    babelExtractPhrases(HASHES, source, relativeFile, options);
+    return HASHES;
   }
 
+  // HTML templates
   if (path.extname(file) === '.html') {
     parseHTMLTemplateFile(HASHES, file, relativeFile, options);
-  } else if (path.extname(file) === '.vue') {
+    return HASHES;
+  }
+
+  // Vue templates
+  if (path.extname(file) === '.vue') {
     extractVuePhrases(HASHES, source, relativeFile, options);
-  } else if (path.extname(file) !== '.html') {
-    babelExtractPhrases(HASHES, source, relativeFile, options);
+    return HASHES;
   }
 
+  // default
+  babelExtractPhrases(HASHES, source, relativeFile, options);
   return HASHES;
 }
 

packages/cli/src/api/parsers/i18next.js

@@ -0,0 +1,99 @@
+const { implodePlurals } = require('@transifex/native');
+const _ = require('lodash');
+const { mergePayload } = require('../merge');
+const { createPayload, isPayloadValid } = require('./utils');
+
+const PLURAL_MAP = {
+  _1: 'one',
+  _2: 'two',
+  _3: 'few',
+  _4: 'many',
+  _5: 'other',
+  _one: 'one',
+  _two: 'two',
+  _few: 'few',
+  _many: 'many',
+  _other: 'other',
+};
+
+/**
+ * Parse i18next JSON v3 & v4 files
+ *
+ * @param {Object} HASHES A map of keys and content for phrases
+ * @param {String} source The content we want to parse
+ * @param {String} relativeFile occurence
+ * @param {Object} options
+ * @param {String[]} options.appendTags
+ * @param {String[]} options.filterWithTags
+ * @param {String[]} options.filterWithoutTags
+ * @param {Boolean} options.useHashedKeys
+ */
+function extractI18NextPhrases(HASHES, source, relativeFile, options) {
+  let json = {};
+  try {
+    json = JSON.parse(source);
+  } catch (err) {
+    return;
+  }
+
+  const plurals = {};
+
+  _.each(json, (string, key) => {
+    // nesting
+    if (!_.isString(string) && !_.isArray(string) && _.isObject(string)) {
+      _.each(string, (innerString, innerKey) => {
+        if (!_.isString(innerString)) return;
+        const partial = createPayload(innerString, {}, relativeFile, options);
+        if (!isPayloadValid(partial, options)) return;
+
+        mergePayload(HASHES, {
+          [`${key}.${innerKey}`]: {
+            string: partial.string,
+            meta: partial.meta,
+          },
+        });
+      });
+      return;
+    }
+
+    if (!_.isString(string)) return;
+
+    let isPlural = false;
+    _.each(_.keys(PLURAL_MAP), (suffix) => {
+      if (_.endsWith(key, suffix)) {
+        isPlural = true;
+        const pluralKey = key.slice(0, -suffix.length);
+        plurals[pluralKey] = plurals[pluralKey] || {};
+        plurals[pluralKey][PLURAL_MAP[suffix]] = string;
+      }
+    });
+    if (isPlural) return;
+
+    const partial = createPayload(string, {}, relativeFile, options);
+    if (!isPayloadValid(partial, options)) return;
+
+    mergePayload(HASHES, {
+      [key]: {
+        string: partial.string,
+        meta: partial.meta,
+      },
+    });
+  });
+
+  // add plurals
+  _.each(plurals, (plural, key) => {
+    const partial = createPayload(implodePlurals(plural, 'count'), {}, relativeFile, options);
+    if (!isPayloadValid(partial, options)) return;
+
+    mergePayload(HASHES, {
+      [`${key}_txplural`]: {
+        string: partial.string,
+        meta: partial.meta,
+      },
+    });
+  });
+}
+
+module.exports = {
+  extractI18NextPhrases,
+};

packages/cli/src/commands/push.js

@@ -73,6 +73,7 @@ class PushCommand extends Command {
       filterWithTags,
       filterWithoutTags,
       useHashedKeys,
+      parser: flags.parser,
     };
 
     _.each(allFiles, (file) => {
@@ -255,6 +256,7 @@ txjs-cli push --append-tags="master,release:2.5"
 txjs-cli push --with-tags-only="home,error"
 txjs-cli push --without-tags-only="custom"
 txjs-cli push --token=mytoken --secret=mysecret
+txjs-cli push en.json --parser=i18next
 TRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli push
 `;
 
@@ -307,6 +309,11 @@ PushCommand.flags = {
     description: 'CDS host URL',
     default: '',
   }),
+  parser: flags.string({
+    description: 'file parser to use',
+    default: 'auto',
+    options: ['auto', 'i18next'],
+  }),
   'key-generator': flags.string({
     description: 'use hashed or source based keys',
     default: 'source',

packages/cli/test/api/extract.i18next.test.js

@@ -0,0 +1,64 @@
+/* globals describe, it */
+
+const { expect } = require('chai');
+const { extractPhrases } = require('../../src/api/extract');
+
+describe('extractPhrases with i18next parser', () => {
+  it('works with json v4', async () => {
+    expect(await extractPhrases('test/fixtures/i18next_v4.json', 'v4.json', { parser: 'i18next' }))
+      .to.deep.equal({
+        'keyDeep.inner': {
+          string: 'value',
+          meta: { context: [], tags: [], occurrences: ['v4.json'] },
+        },
+        key: {
+          string: 'value',
+          meta: { context: [], tags: [], occurrences: ['v4.json'] },
+        },
+        keyNesting: {
+          string: 'reuse $t(keyDeep.inner)',
+          meta: { context: [], tags: [], occurrences: ['v4.json'] },
+        },
+        keyInterpolate: {
+          string: 'replace this {{value}}',
+          meta: { context: [], tags: [], occurrences: ['v4.json'] },
+        },
+        keyInterpolateUnescaped: {
+          string: 'replace this {{- value}}',
+          meta: { context: [], tags: [], occurrences: ['v4.json'] },
+        },
+        keyInterpolateWithFormatting: {
+          string: 'replace this {{value, format}}',
+          meta: { context: [], tags: [], occurrences: ['v4.json'] },
+        },
+        keyContext_male: {
+          string: 'the male variant',
+          meta: { context: [], tags: [], occurrences: ['v4.json'] },
+        },
+        keyContext_female: {
+          string: 'the female variant',
+          meta: { context: [], tags: [], occurrences: ['v4.json'] },
+        },
+        keyPluralSimple_txplural: {
+          string: '{count, plural, one {the singular} other {the plural}}',
+          meta: { context: [], tags: [], occurrences: ['v4.json'] },
+        },
+        keyPluralMultipleEgArabic_zero: {
+          string: 'the plural form 0',
+          meta: { context: [], tags: [], occurrences: ['v4.json'] },
+        },
+        keyPluralMultipleEgArabic_txplural: {
+          string: '{count, plural, one {the plural form 1} two {the plural form 2} few {the plural form 3} many {the plural form 4} other {the plural form 5}}',
+          meta: { context: [], tags: [], occurrences: ['v4.json'] },
+        },
+        'keyWithObjectValue.valueA': {
+          string: 'return this with valueB',
+          meta: { context: [], tags: [], occurrences: ['v4.json'] },
+        },
+        'keyWithObjectValue.valueB': {
+          string: 'more text',
+          meta: { context: [], tags: [], occurrences: ['v4.json'] },
+        },
+      });
+  });
+});

packages/cli/test/fixtures/i18next_v4.json

@@ -0,0 +1,22 @@
+{
+  "key": "value",
+  "keyDeep": {
+    "inner": "value"
+  },
+  "keyNesting": "reuse $t(keyDeep.inner)",
+  "keyInterpolate": "replace this {{value}}",
+  "keyInterpolateUnescaped": "replace this {{- value}}",
+  "keyInterpolateWithFormatting": "replace this {{value, format}}",
+  "keyContext_male": "the male variant",
+  "keyContext_female": "the female variant",
+  "keyPluralSimple_one": "the singular",
+  "keyPluralSimple_other": "the plural",
+  "keyPluralMultipleEgArabic_zero": "the plural form 0",
+  "keyPluralMultipleEgArabic_one": "the plural form 1",
+  "keyPluralMultipleEgArabic_two": "the plural form 2",
+  "keyPluralMultipleEgArabic_few": "the plural form 3",
+  "keyPluralMultipleEgArabic_many": "the plural form 4",
+  "keyPluralMultipleEgArabic_other": "the plural form 5",
+  "keyWithArrayValue": ["multipe", "things"],
+  "keyWithObjectValue": { "valueA": "return this with valueB", "valueB": "more text" }
+}

packages/i18next/.eslintrc.json

@@ -0,0 +1,9 @@
+{
+  "extends": ["airbnb-base"],
+  "rules": {
+    "import/no-extraneous-dependencies": "off",
+    "no-plusplus": "off",
+    "no-underscore-dangle": "off"
+  },
+  "ignorePatterns": ["/**/*.d.ts"]
+}

packages/i18next/.npmignore

@@ -0,0 +1,3 @@
+.nyc_output
+tests
+.eslintrc.json