feat(JF-466): add language recognition

Author: axherrm

pom.xml

@@ -320,6 +320,13 @@
       <version>${pdfbox.version}</version>
     </dependency>
 
+    <!-- Language recognition -->
+    <dependency>
+      <groupId>com.github.pemistahl</groupId>
+      <artifactId>lingua</artifactId>
+      <version>1.2.2</version>
+    </dependency>
+
     <!-- Using Hamcrest in a Maven Project see http://hamcrest.org/JavaHamcrest/distributables#maven-upgrade-example -->
     <dependency>
       <groupId>org.hamcrest</groupId>

src/main/java/org/jadice/filetype/matchers/PDFMatcher.java

@@ -15,6 +15,8 @@
 import javax.xml.transform.stream.StreamResult;
 import javax.xml.transform.stream.StreamSource;
 
+import com.github.pemistahl.lingua.api.LanguageDetector;
+import com.github.pemistahl.lingua.api.LanguageDetectorBuilder;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
 import org.apache.pdfbox.pdmodel.PDDocumentInformation;
@@ -39,7 +41,7 @@
 
 /**
  * A {@link Matcher} for PDF documents .
- * 
+ * <p>
  * Caveat: for performance reasons, this should only be called from a context where the stream has
  * already be identified as a PDF file/stream.
  */
@@ -68,6 +70,15 @@ public class PDFMatcher extends Matcher {
   public static final String TEXT_LENGTH_KEY = "text-length";
   public static final String TEXT_LENGTH_PER_PAGE_KEY = "text-length-per-page";
 
+  /**
+   * Most likely language of the text of the PDF, analyzed with <a href="https://github.com/pemistahl/lingua">lingua</a>
+   */
+  public static final String MOST_LIKELY_TEXT_LANGUAGE = "most-likely-text-language";
+  /**
+   * All possible languages of the PDF's text, sorted by their confidence value, analyzed with <a href="https://github.com/pemistahl/lingua">lingua</a>
+   */
+  public static final String TEXT_LANGUAGE_CONFIDENCE_VALUES = "text-language-confidence-values";
+
   private static boolean lookForText() {
     return "true".equalsIgnoreCase(System.getProperty(PDFMatcher.class.getName() + ".lookForText", "false"));
   }
@@ -251,6 +262,8 @@ private static PDEmbeddedFile getEmbeddedFile(final PDComplexFileSpecification f
    *   <li>{@link #CONTAINS_TEXT_KEY} whether the whole document contains any text (without line breaks)</li>
    *   <li>{@link #TEXT_LENGTH_PER_PAGE_KEY} list of integers that indicate how long the text in each page is (only set if there is text at all)</li>
    *   <li>{@link #TEXT_LENGTH_KEY} length of the text of the whole document (only set if there is text at all)</li>
+   *   <li>{@link #MOST_LIKELY_TEXT_LANGUAGE} detected language (only set if there is text at all)</li>
+   *   <li>{@link #TEXT_LANGUAGE_CONFIDENCE_VALUES} map of all possible languages, sorted by their confidence value (only set if there is text at all)</li>
    * </ul>
    *
    * @param pdfDetails map to which the results get added
@@ -274,9 +287,32 @@ private static void addTextInfo(final Map<String, Object> pdfDetails, final PDDo
       final String pdfText = new PDFTextStripper().getText(doc);
       pdfDetails.put(TEXT_LENGTH_PER_PAGE_KEY, textLengthPerPages);
       pdfDetails.put(TEXT_LENGTH_KEY, pdfText.replaceAll("([\\r\\n])", "").length());
+      addLanguageInformation(pdfDetails, pdfText);
     }
   }
 
+  /**
+   * Adds information about the given text to the given map.
+   * The most likely text language will be {@link com.github.pemistahl.lingua.api.Language#UNKNOWN} in case
+   * language detection is not reliably possible.
+   *
+   * @param pdfDetails map to which the results get added
+   * @param text text to analyze
+   */
+  public static void addLanguageInformation(final Map<String, Object> pdfDetails, final String text) {
+    LanguageDetectorBuilder languageDetectorBuilder =
+        LanguageDetectorBuilder
+            .fromAllLanguages()
+            .withMinimumRelativeDistance(0.1);
+    if (text.length() > 120)
+      languageDetectorBuilder.withLowAccuracyMode();
+    final LanguageDetector languageDetector = languageDetectorBuilder.build();
+    final long startTime = System.currentTimeMillis();
+    pdfDetails.put(TEXT_LANGUAGE_CONFIDENCE_VALUES, languageDetector.computeLanguageConfidenceValues(text));
+    pdfDetails.put(MOST_LIKELY_TEXT_LANGUAGE, languageDetector.detectLanguageOf(text).toString());
+    LOGGER.debug("Language recognition took {} ms.", System.currentTimeMillis() - startTime);
+  }
+
   /**
    * Reads the whole stream to determine the length of it.
    *

src/test/java/TestPDFMatcher.java

@@ -158,7 +158,7 @@ void testSignedPDFs(final String urlString, final int expectedSignatureCount) th
   @SuppressWarnings("unchecked")
   @ParameterizedTest
   @CsvFileSource(resources = "/pdf/contains-text.csv", numLinesToSkip = 1)
-  void testContainsText(final String filePath, final boolean expected) throws IOException {
+  void testContainsText(final String filePath, final boolean expected, final String language) throws IOException {
     System.setProperty(PDFMatcher.class.getName() + ".lookForText", "true");
     Map<String, Object> result = ANALYZER.analyze(new File(filePath));
     assertNotNull(result);
@@ -174,6 +174,10 @@ void testContainsText(final String filePath, final boolean expected) throws IOEx
       final List<Integer> textLengthPerPages = (List<Integer>) pdfDetails.get(PDFMatcher.TEXT_LENGTH_PER_PAGE_KEY);
       final int sum = textLengthPerPages.stream().mapToInt(Integer::intValue).sum();
       assertEquals(totalTextLength, sum);
+      if (!language.equals("null")) {
+        assertEquals(language, pdfDetails.get(PDFMatcher.MOST_LIKELY_TEXT_LANGUAGE));
+      }
+      assertTrue(pdfDetails.containsKey(PDFMatcher.TEXT_LANGUAGE_CONFIDENCE_VALUES));
     }
     System.clearProperty(PDFMatcher.class.getName() + ".lookForText");
   }

src/test/resources/pdf/contains-text.csv

@@ -1,19 +1,19 @@
-path,contains-text
-src/test/resources/pdf/normal/lorem-ipsum.pdf,true
-src/test/resources/pdf/normal/PDF-testdoc-1.3-Karte.pdf,false
-src/test/resources/pdf/normal/PDF-testdoc-1.4-hallo.pdf,true
-src/test/resources/pdf/normal/PDF-testdoc-1.4-hallo-annotation.pdf,true
-src/test/resources/pdf/normal/PDF-testdoc-1.4-hallo-complex.pdf,true
-src/test/resources/pdf/normal/PDF-testdoc-1.4-lines.pdf,false
-src/test/resources/pdf/normal/PDF-testdoc-1.4-MultiImage.pdf,false
-src/test/resources/pdf/normal/PDF-testdoc-1.4-RotatedImages.pdf,true
-src/test/resources/pdf/normal/PDF-testdoc-1.4-SingleImage.pdf,false
-src/test/resources/pdf/normal/PDF-testdoc-1.4-TextTest.pdf,true
-src/test/resources/pdf/encrypted/02_enc40bit_no-pw.pdf,true
-src/test/resources/pdf/encrypted/03_enc40bit_pw-owner.pdf,true
-src/test/resources/pdf/encrypted/06_enc128bit_no-pw.pdf,true
-src/test/resources/pdf/encrypted/07_enc128bit_pw-owner.pdf,true
-src/test/resources/pdf/encrypted/10_enc128bit-aes_no-pw.pdf,true
-src/test/resources/pdf/encrypted/11_enc128bit-aes_pw-owner.pdf,true
-src/test/resources/pdf/encrypted/Contrapunctus I BWV 1080.pdf,true
-src/test/resources/pdf/portfolio/portable-collection-1.pdf,true
+path,contains-text,language(ignored if null)
+src/test/resources/pdf/normal/lorem-ipsum.pdf,true,LATIN
+src/test/resources/pdf/normal/PDF-testdoc-1.3-Karte.pdf,false,null
+src/test/resources/pdf/normal/PDF-testdoc-1.4-hallo.pdf,true,null
+src/test/resources/pdf/normal/PDF-testdoc-1.4-hallo-annotation.pdf,true,null
+src/test/resources/pdf/normal/PDF-testdoc-1.4-hallo-complex.pdf,true,null
+src/test/resources/pdf/normal/PDF-testdoc-1.4-lines.pdf,false,null
+src/test/resources/pdf/normal/PDF-testdoc-1.4-MultiImage.pdf,false,null
+src/test/resources/pdf/normal/PDF-testdoc-1.4-RotatedImages.pdf,true,null
+src/test/resources/pdf/normal/PDF-testdoc-1.4-SingleImage.pdf,false,null
+src/test/resources/pdf/normal/PDF-testdoc-1.4-TextTest.pdf,true,null
+src/test/resources/pdf/encrypted/02_enc40bit_no-pw.pdf,true,null
+src/test/resources/pdf/encrypted/03_enc40bit_pw-owner.pdf,true,null
+src/test/resources/pdf/encrypted/06_enc128bit_no-pw.pdf,true,null
+src/test/resources/pdf/encrypted/07_enc128bit_pw-owner.pdf,true,null
+src/test/resources/pdf/encrypted/10_enc128bit-aes_no-pw.pdf,true,null
+src/test/resources/pdf/encrypted/11_enc128bit-aes_pw-owner.pdf,true,null
+src/test/resources/pdf/encrypted/Contrapunctus I BWV 1080.pdf,true,null
+src/test/resources/pdf/portfolio/portable-collection-1.pdf,true,GERMAN