feat: Adds more fields to the search index

Px.Search.Lucene/CaseInsensitiveKeywordAnalyzer.cs

@@ -0,0 +1,18 @@
+using Lucene.Net.Analysis.Core;
+
+namespace Px.Search.Lucene
+{
+    internal class CaseInsensitiveKeywordAnalyzer : Analyzer
+    {
+        protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
+        {
+            // Keeps the text as a single token (no word splitting)
+            Tokenizer source = new KeywordTokenizer(reader);
+
+            // Converts that single token to lowercase
+            TokenStream filter = new LowerCaseFilter(LuceneVersion.LUCENE_48, source);
+
+            return new TokenStreamComponents(source, filter);
+        }
+    }
+}

Px.Search.Lucene/Config/ILuceneConfigurationService.cs

@@ -4,5 +4,7 @@ public interface ILuceneConfigurationService
     {
         LuceneConfigurationOptions GetConfiguration();
         string GetIndexDirectoryPath();
+
+        string[] GetSearchFields();
     }
 }

Px.Search.Lucene/Config/LuceneConfigurationOptions.cs

@@ -3,5 +3,7 @@
     public class LuceneConfigurationOptions
     {
         public string? IndexDirectory { get; set; }
+
+        public string[] SearchFields { get; set; } = Array.Empty<string>();
     }
 }

Px.Search.Lucene/Config/LuceneConfigurationService.cs

@@ -1,4 +1,6 @@
-namespace Px.Search.Lucene.Config
+using System.Configuration;
+
+namespace Px.Search.Lucene.Config
 {
     public class LuceneConfigurationService : ILuceneConfigurationService
     {
@@ -25,7 +27,7 @@
 
             if (string.IsNullOrWhiteSpace(luceneOptions.IndexDirectory))
             {
                 throw new Exception("Index directory not configured for Lucene index");
             }
 
             string path = luceneOptions.IndexDirectory;
@@ -42,5 +44,16 @@
 
             return indexDirectory;
         }
+
+        public string[] GetSearchFields()
+        {
+            var luceneOptions = GetConfiguration();
+            if (luceneOptions.SearchFields == null || luceneOptions.SearchFields.Length == 0)
+            {
+                throw new ConfigurationErrorsException("Search fields not configured for Lucene index");
+            }
+            return luceneOptions.SearchFields;
+        }
     }
+
 }

Px.Search.Lucene/LuceneAnalyzer.cs

@@ -1,10 +1,12 @@
-namespace Px.Search.Lucene
+using Lucene.Net.Analysis.Miscellaneous;
+
+namespace Px.Search.Lucene
 {
-    public class LuceneAnalyzer
+    public static class LuceneAnalyzer
     {
-        internal static LuceneVersion luceneVersion = LuceneVersion.LUCENE_48;
+        internal const LuceneVersion luceneVersion = LuceneVersion.LUCENE_48;
 
-        internal static Analyzer GetAnalyzer(string language)
+        internal static Analyzer GetDefaultAnalyzer(string language)
         {
             switch (language)
             {
@@ -44,9 +46,26 @@ internal static Analyzer GetAnalyzer(string language)
 
         }
 
-    }
-    // TODO ? Should read langs from config and prepare a static analyzerByLanguage dictionary? 
+        internal static Analyzer GetAnalyzer(string language)
+        {
+            var defaultAnalyzer = GetDefaultAnalyzer(language);
+            var keywordAnalyzer = new CaseInsensitiveKeywordAnalyzer();
 
-    // depricated: Analyzer analyzer = new SnowballAnalyzer(LuceneVersion.LUCENE_48, "English");
+            return new PerFieldAnalyzerWrapper(defaultAnalyzer,
+                new Dictionary<string, Analyzer>
+                {
+                    { SearchConstants.SEARCH_FIELD_TAGS, keywordAnalyzer },
+                    { SearchConstants.SEARCH_FIELD_MATRIX, keywordAnalyzer },
+                    { SearchConstants.SEARCH_FIELD_CODES, keywordAnalyzer },
+                    { SearchConstants.SEARCH_FIELD_GROUPINGCODES, keywordAnalyzer },
+                    { SearchConstants.SEARCH_FIELD_VALUESETCODES, keywordAnalyzer },
+                    { SearchConstants.SEARCH_FIELD_LEVEL_CODE, keywordAnalyzer },
+                    { SearchConstants.SEARCH_FIELD_META_ID, keywordAnalyzer },
+                    { SearchConstants.SEARCH_FIELD_LEGACY_ID, keywordAnalyzer }
+                });
+
+        }
+
+    }
 }
 

Px.Search.Lucene/LuceneBackend.cs

@@ -3,14 +3,16 @@
     public class LuceneBackend : ISearchBackend
     {
 
         private readonly ILuceneConfigurationService _luceneConfigurationService;
 
         private readonly string _path;
+        private readonly string[] _searchFields;
 
         public LuceneBackend(ILuceneConfigurationService luceneConfigurationService)
         {
             _luceneConfigurationService = luceneConfigurationService;
             _path = _luceneConfigurationService.GetIndexDirectoryPath();
+            _searchFields = _luceneConfigurationService.GetSearchFields();
         }
 
         public IIndex GetIndex()
@@ -20,7 +22,7 @@
 
         public ISearcher GetSearcher(string language)
         {
-            return new LuceneSearcher(_path, language);
+            return new LuceneSearcher(_path, language, _searchFields);
         }
     }
 }

Px.Search.Lucene/LuceneIndex.cs

@@ -222,6 +222,31 @@ private static Document GetDocument(TableInformation tbl, PXMeta meta)
             doc.Add(new TextField(SearchConstants.SEARCH_FIELD_TIME_UNIT, tbl.TimeUnit, Field.Store.YES));
             doc.Add(new TextField(SearchConstants.SEARCH_SUBJECT_CODE, tbl.SubjectCode, Field.Store.YES));
             doc.Add(new StoredField(SearchConstants.SEARCH_FIELD_PATHS, GetBytes(tbl.Paths)));
+
+            foreach (var path in tbl.Paths)
+            {
+                foreach (var level in path)
+                {
+                    doc.Add(new TextField(SearchConstants.SEARCH_FIELD_LEVEL_CODE, level.Code, Field.Store.NO));
+                    doc.Add(new TextField(SearchConstants.SEARCH_FIELD_LEVEL_NAME, level.Text, Field.Store.NO));
+                }
+            }
+
+            if (!string.IsNullOrWhiteSpace(meta.MetaId))
+            {
+                doc.Add(new TextField(SearchConstants.SEARCH_FIELD_META_ID, meta.MetaId, Field.Store.NO));
+
+                foreach (var variable in meta.Variables.Where(v => !string.IsNullOrWhiteSpace(v.MetaId)))
+                {
+                    doc.Add(new TextField(SearchConstants.SEARCH_FIELD_META_ID, variable.MetaId, Field.Store.NO));
+                }
+            }
+
+            if (!string.IsNullOrWhiteSpace(meta.MainTable))
+            {
+                doc.Add(new TextField(SearchConstants.SEARCH_FIELD_META_ID, meta.MainTable, Field.Store.NO));
+            }
+
             doc.Add(new StoredField(SearchConstants.SEARCH_AVAILABLE_LANGUAGES, string.Join("|", tbl.Languages)));
             if (!string.IsNullOrEmpty(meta.Synonyms))
             {

Px.Search.Lucene/LuceneSearcher.cs

@@ -9,12 +9,14 @@
         private readonly IndexSearcher _indexSearcher;
         private static readonly Operator _defaultOperator = Operator.AND;
         private readonly Analyzer _analyzer;
+        private readonly string[] _fieldNames;
+
 
         /// <summary>
         /// Constructor
         /// </summary>
         /// <param name="indexDirectory">Index directory</param>
-        public LuceneSearcher(string indexDirectory, string language)
+        public LuceneSearcher(string indexDirectory, string language, string[] fieldNames)
         {
             if (string.IsNullOrWhiteSpace(indexDirectory))
             {
@@ -26,6 +28,7 @@
             IndexReader reader = DirectoryReader.Open(fsDir);
             _indexSearcher = new IndexSearcher(reader);
             _analyzer = LuceneAnalyzer.GetAnalyzer(language);
+            _fieldNames = fieldNames;
         }
 
 
@@ -43,7 +46,7 @@
             var skipRecords = pageSize * (pageNumber - 1);
             var searchResultContainer = new SearchResultContainer();
             var searchResultList = new List<SearchResult>();
-            string[] fields = GetSearchFields();
+            string[] fields = _fieldNames;
             Query luceneQuery;
             QueryParser queryParser = new MultiFieldQueryParser(LuceneAnalyzer.luceneVersion,
                                                        fields,
@@ -151,7 +154,7 @@
                 );
             searchResult.Description = doc.Get(SearchConstants.SEARCH_FIELD_DESCRIPTION);
             searchResult.SortCode = doc.Get(SearchConstants.SEARCH_FIELD_SORTCODE);
             if (DateTime.TryParse(doc.Get(SearchConstants.SEARCH_FIELD_UPDATED), out updated))
             {
                 searchResult.Updated = updated;
             }
@@ -171,40 +174,5 @@
 
             return searchResult;
         }
-
-        /// <summary>
-        /// Get fields in index to search in
-        /// </summary>
-        /// <returns></returns>
-        private static string[] GetSearchFields()
-        {
-            string[] fields;
-
-            // Default fields
-            fields = new[] { SearchConstants.SEARCH_FIELD_DOCID,
-                                SearchConstants.SEARCH_FIELD_SEARCHID,
-                                SearchConstants.SEARCH_FIELD_UPDATED,
-                                SearchConstants.SEARCH_FIELD_MATRIX,
-                                SearchConstants.SEARCH_FIELD_TITLE,
-                                SearchConstants.SEARCH_FIELD_DESCRIPTION,
-                                SearchConstants.SEARCH_FIELD_SORTCODE,
-                                SearchConstants.SEARCH_FIELD_CATEGORY,
-                                SearchConstants.SEARCH_FIELD_FIRSTPERIOD,
-                                SearchConstants.SEARCH_FIELD_LASTPERIOD,
-                                SearchConstants.SEARCH_FIELD_VARIABLES,
-                                SearchConstants.SEARCH_FIELD_PERIOD,
-                                SearchConstants.SEARCH_FIELD_VALUES,
-                                SearchConstants.SEARCH_FIELD_CODES,
-                                SearchConstants.SEARCH_FIELD_GROUPINGS,
-                                SearchConstants.SEARCH_FIELD_GROUPINGCODES,
-                                SearchConstants.SEARCH_FIELD_VALUESETS,
-                                SearchConstants.SEARCH_FIELD_VALUESETCODES,
-                                SearchConstants.SEARCH_FIELD_DISCONTINUED,
-                                SearchConstants.SEARCH_FIELD_TAGS
-            };
-
-            return fields;
-        }
-
     }
 }

Px.Search.Lucene/SearchConstants.cs

@@ -1,6 +1,6 @@
 namespace Px.Search.Lucene
 {
     public class SearchConstants
     {
         // Search field constants
         public const string SEARCH_FIELD_DOCID = "docid";
@@ -27,6 +27,10 @@
         public const string SEARCH_FIELD_SOURCE = "source";
         public const string SEARCH_FIELD_TIME_UNIT = "timeunit";
         public const string SEARCH_FIELD_PATHS = "paths";
+        public const string SEARCH_FIELD_LEVEL_CODE = "levelcode";
+        public const string SEARCH_FIELD_LEVEL_NAME = "levelname";
+        public const string SEARCH_FIELD_META_ID = "metaid";
+        public const string SEARCH_FIELD_LEGACY_ID = "legacyid";
         public const string SEARCH_SUBJECT_CODE = "subjectcode";
         public const string SEARCH_AVAILABLE_LANGUAGES = "languages";
     }

PxWeb.UnitTests/Search/LuceneAnalyzerTest.cs

@@ -21,6 +21,7 @@ private void TestSearcher(string language, string searchFor, int expectedCount)
         {
             LuceneConfigurationOptions luceneConfigurationOptions = new LuceneConfigurationOptions();
             luceneConfigurationOptions.IndexDirectory = @"Database/_INDEX/";
+            luceneConfigurationOptions.SearchFields = new string[] { "docid", "searchid", "updated", "matrix", "title", "description", "sortcode", "category", "firstperiod", "lastperiod", "variables", "period", "values", "codes", "groupings", "groupingcodes", "valuesets", "valuesetcodes", "discontinued", "tags" };
 
             //seeking "C:\\repos\\github\\pxtools\\PxwebApi3\\PxWebApi\\PxWeb\\wwwroot" 
             string pathRunning = Directory.GetCurrentDirectory();