undo language recognition to move to another branch

Author: axherrm

pom.xml

@@ -320,13 +320,6 @@
       <version>${pdfbox.version}</version>
     </dependency>
 
-    <!-- Language recognition -->
-    <dependency>
-      <groupId>com.github.pemistahl</groupId>
-      <artifactId>lingua</artifactId>
-      <version>1.2.2</version>
-    </dependency>
-
     <!-- for pdf signature validation -->
     <dependency>
       <groupId>org.bouncycastle</groupId>

src/main/java/org/jadice/filetype/matchers/PDFMatcher.java

@@ -21,8 +21,6 @@
 import javax.xml.transform.stream.StreamResult;
 import javax.xml.transform.stream.StreamSource;
 
-import com.github.pemistahl.lingua.api.LanguageDetector;
-import com.github.pemistahl.lingua.api.LanguageDetectorBuilder;
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.io.IOUtils;
 import org.apache.pdfbox.pdmodel.PDDocument;
@@ -267,8 +265,6 @@ private static PDEmbeddedFile getEmbeddedFile(final PDComplexFileSpecification f
    *   <li>{@link #CONTAINS_TEXT_KEY} whether the whole document contains any text (without line breaks)</li>
    *   <li>{@link #TEXT_LENGTH_PER_PAGE_KEY} list of integers that indicate how long the text in each page is (only set if there is text at all)</li>
    *   <li>{@link #TEXT_LENGTH_KEY} length of the text of the whole document (only set if there is text at all)</li>
-   *   <li>{@link #MOST_LIKELY_TEXT_LANGUAGE} detected language (only set if there is text at all)</li>
-   *   <li>{@link #TEXT_LANGUAGE_CONFIDENCE_VALUES} map of all possible languages, sorted by their confidence value (only set if there is text at all)</li>
    * </ul>
    *
    * @param pdfDetails map to which the results get added
@@ -292,32 +288,9 @@ private static void addTextInfo(final Map<String, Object> pdfDetails, final PDDo
       final String pdfText = new PDFTextStripper().getText(doc);
       pdfDetails.put(TEXT_LENGTH_PER_PAGE_KEY, textLengthPerPages);
       pdfDetails.put(TEXT_LENGTH_KEY, pdfText.replaceAll("([\\r\\n])", "").length());
-      addLanguageInformation(pdfDetails, pdfText);
     }
   }
 
-  /**
-   * Adds information about the given text to the given map.
-   * The most likely text language will be {@link com.github.pemistahl.lingua.api.Language#UNKNOWN} in case
-   * language detection is not reliably possible.
-   *
-   * @param pdfDetails map to which the results get added
-   * @param text text to analyze
-   */
-  public static void addLanguageInformation(final Map<String, Object> pdfDetails, final String text) {
-    LanguageDetectorBuilder languageDetectorBuilder =
-        LanguageDetectorBuilder
-            .fromAllLanguages()
-            .withMinimumRelativeDistance(0.1);
-    if (text.length() > 120)
-      languageDetectorBuilder.withLowAccuracyMode();
-    final LanguageDetector languageDetector = languageDetectorBuilder.build();
-    final long startTime = System.currentTimeMillis();
-    pdfDetails.put(TEXT_LANGUAGE_CONFIDENCE_VALUES, languageDetector.computeLanguageConfidenceValues(text));
-    pdfDetails.put(MOST_LIKELY_TEXT_LANGUAGE, languageDetector.detectLanguageOf(text).toString());
-    LOGGER.debug("Language recognition took {} ms.", System.currentTimeMillis() - startTime);
-  }
-
   /**
    * Checks if the PDF is an electronic invoice.
    *

src/test/java/TestPDFMatcher.java

@@ -177,10 +177,6 @@ void testContainsText(final String filePath, final boolean expected, final Strin
       final List<Integer> textLengthPerPages = (List<Integer>) pdfDetails.get(PDFMatcher.TEXT_LENGTH_PER_PAGE_KEY);
       final int sum = textLengthPerPages.stream().mapToInt(Integer::intValue).sum();
       assertEquals(totalTextLength, sum);
-      if (!language.equals("null")) {
-        assertEquals(language, pdfDetails.get(PDFMatcher.MOST_LIKELY_TEXT_LANGUAGE));
-      }
-      assertTrue(pdfDetails.containsKey(PDFMatcher.TEXT_LANGUAGE_CONFIDENCE_VALUES));
     }
     System.clearProperty(PDFMatcher.class.getName() + ".lookForText");
   }

src/test/resources/pdf/contains-text.csv

@@ -1,19 +1,19 @@
-path,contains-text,language(ignored if null)
-src/test/resources/pdf/normal/lorem-ipsum.pdf,true,LATIN
-src/test/resources/pdf/normal/PDF-testdoc-1.3-Karte.pdf,false,null
-src/test/resources/pdf/normal/PDF-testdoc-1.4-hallo.pdf,true,null
-src/test/resources/pdf/normal/PDF-testdoc-1.4-hallo-annotation.pdf,true,null
-src/test/resources/pdf/normal/PDF-testdoc-1.4-hallo-complex.pdf,true,null
-src/test/resources/pdf/normal/PDF-testdoc-1.4-lines.pdf,false,null
-src/test/resources/pdf/normal/PDF-testdoc-1.4-MultiImage.pdf,false,null
-src/test/resources/pdf/normal/PDF-testdoc-1.4-RotatedImages.pdf,true,null
-src/test/resources/pdf/normal/PDF-testdoc-1.4-SingleImage.pdf,false,null
-src/test/resources/pdf/normal/PDF-testdoc-1.4-TextTest.pdf,true,null
-src/test/resources/pdf/encrypted/02_enc40bit_no-pw.pdf,true,null
-src/test/resources/pdf/encrypted/03_enc40bit_pw-owner.pdf,true,null
-src/test/resources/pdf/encrypted/06_enc128bit_no-pw.pdf,true,null
-src/test/resources/pdf/encrypted/07_enc128bit_pw-owner.pdf,true,null
-src/test/resources/pdf/encrypted/10_enc128bit-aes_no-pw.pdf,true,null
-src/test/resources/pdf/encrypted/11_enc128bit-aes_pw-owner.pdf,true,null
-src/test/resources/pdf/encrypted/Contrapunctus I BWV 1080.pdf,true,null
-src/test/resources/pdf/portfolio/portable-collection-1.pdf,true,GERMAN
+path,contains-text
+src/test/resources/pdf/normal/lorem-ipsum.pdf,true
+src/test/resources/pdf/normal/PDF-testdoc-1.3-Karte.pdf,false
+src/test/resources/pdf/normal/PDF-testdoc-1.4-hallo.pdf,true
+src/test/resources/pdf/normal/PDF-testdoc-1.4-hallo-annotation.pdf,true
+src/test/resources/pdf/normal/PDF-testdoc-1.4-hallo-complex.pdf,true
+src/test/resources/pdf/normal/PDF-testdoc-1.4-lines.pdf,false
+src/test/resources/pdf/normal/PDF-testdoc-1.4-MultiImage.pdf,false
+src/test/resources/pdf/normal/PDF-testdoc-1.4-RotatedImages.pdf,true
+src/test/resources/pdf/normal/PDF-testdoc-1.4-SingleImage.pdf,false
+src/test/resources/pdf/normal/PDF-testdoc-1.4-TextTest.pdf,true
+src/test/resources/pdf/encrypted/02_enc40bit_no-pw.pdf,true
+src/test/resources/pdf/encrypted/03_enc40bit_pw-owner.pdf,true
+src/test/resources/pdf/encrypted/06_enc128bit_no-pw.pdf,true
+src/test/resources/pdf/encrypted/07_enc128bit_pw-owner.pdf,true
+src/test/resources/pdf/encrypted/10_enc128bit-aes_no-pw.pdf,true
+src/test/resources/pdf/encrypted/11_enc128bit-aes_pw-owner.pdf,true
+src/test/resources/pdf/encrypted/Contrapunctus I BWV 1080.pdf,true
+src/test/resources/pdf/portfolio/portable-collection-1.pdf,true