raise error if tokenLength is provided and return message stating it has been deprecated

Author: fimac

lib/cipherstash/analysis/text_processor.rb

@@ -46,6 +46,10 @@ def build_token_filters(array)
             TokenFilters::Downcase.new(obj)
 
           when "ngram"
+            if obj["tokenLength"]
+              raise CipherStash::Client::Error::InternalError, "'tokenLength' is deprecated. Use 'minLength' and 'maxLength' for the ngram filter."
+            end
+
             unless obj["minLength"].instance_of?(Integer) && obj["maxLength"].instance_of?(Integer)
               raise CipherStash::Client::Error::InternalError, "The values provided to the min and max length must be of type Integer."
             end

spec/cipherstash/analysis/text_processor_spec.rb

@@ -44,6 +44,18 @@
         }.to raise_error(CipherStash::Client::Error::InternalError, "The ngram filter min length must be less than or equal to the max length")
     end
 
+    it "raises an error if tokenLength is provided" do
+      expect {
+        CipherStash::Analysis::TextProcessor.new({
+          "tokenFilters" => [
+            { "kind" => "downcase" },
+            { "kind" => "ngram", "tokenLength" => 3 }
+          ],
+          "tokenizer" => { "kind" => "standard" }
+        })
+      }.to raise_error(CipherStash::Client::Error::InternalError, "'tokenLength' is deprecated. Use 'minLength' and 'maxLength' for the ngram filter.")
+    end
+
     it "splits text into ngrams using min length of 3 and max length of 8" do
       tokenizer =
         CipherStash::Analysis::TextProcessor.new({