update token filter to generate ngrams from min to max length size

Author: fimac

lib/cipherstash/analysis/token_filters.rb

@@ -15,13 +15,24 @@ def perform(str_or_array)
 
       class NGram < Base
         def perform(str_or_array)
-          token_length = @opts["tokenLength"] || 3
+          min_length = @opts["minLength"] || 3
+          max_length = @opts["maxLength"] || 8
+
           Array(str_or_array).flat_map do |token|
-            [].tap do |out|
-              (token.length - token_length + 1).times do |i|
-                out << token[i, token_length]
+            token_length = token.length
+
+            ngrams = [].tap do |out|
+              (min_length..max_length).each do |n|
+                ngram = token.chars.each_cons(n).map(&:join)
+                out << ngram
+              end
+
+              if token_length > max_length
+                out << token
               end
             end
+
+            ngrams.flatten
           end
         end
       end