diff --git a/Px.Search.Lucene/CaseInsensitiveKeywordAnalyzer.cs b/Px.Search.Lucene/CaseInsensitiveKeywordAnalyzer.cs new file mode 100644 index 00000000..02141d0e --- /dev/null +++ b/Px.Search.Lucene/CaseInsensitiveKeywordAnalyzer.cs @@ -0,0 +1,18 @@ +using Lucene.Net.Analysis.Core; + +namespace Px.Search.Lucene +{ + internal class CaseInsensitiveKeywordAnalyzer : Analyzer + { + protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) + { + // Keeps the text as a single token (no word splitting) + Tokenizer source = new KeywordTokenizer(reader); + + // Converts that single token to lowercase + TokenStream filter = new LowerCaseFilter(LuceneVersion.LUCENE_48, source); + + return new TokenStreamComponents(source, filter); + } + } +} diff --git a/Px.Search.Lucene/Config/ILuceneConfigurationService.cs b/Px.Search.Lucene/Config/ILuceneConfigurationService.cs index f76d0b0b..d8c74243 100644 --- a/Px.Search.Lucene/Config/ILuceneConfigurationService.cs +++ b/Px.Search.Lucene/Config/ILuceneConfigurationService.cs @@ -4,5 +4,7 @@ public interface ILuceneConfigurationService { LuceneConfigurationOptions GetConfiguration(); string GetIndexDirectoryPath(); + + string[] GetSearchFields(); } } diff --git a/Px.Search.Lucene/Config/LuceneConfigurationOptions.cs b/Px.Search.Lucene/Config/LuceneConfigurationOptions.cs index 5a67dd65..b27e9061 100644 --- a/Px.Search.Lucene/Config/LuceneConfigurationOptions.cs +++ b/Px.Search.Lucene/Config/LuceneConfigurationOptions.cs @@ -3,5 +3,7 @@ public class LuceneConfigurationOptions { public string? IndexDirectory { get; set; } + + public string[] SearchFields { get; set; } = Array.Empty(); } } diff --git a/Px.Search.Lucene/Config/LuceneConfigurationService.cs b/Px.Search.Lucene/Config/LuceneConfigurationService.cs index d220053f..a224a215 100644 --- a/Px.Search.Lucene/Config/LuceneConfigurationService.cs +++ b/Px.Search.Lucene/Config/LuceneConfigurationService.cs @@ -1,4 +1,6 @@ -namespace Px.Search.Lucene.Config +using System.Configuration; + +namespace Px.Search.Lucene.Config { public class LuceneConfigurationService : ILuceneConfigurationService { @@ -42,5 +44,16 @@ public string GetIndexDirectoryPath() return indexDirectory; } + + public string[] GetSearchFields() + { + var luceneOptions = GetConfiguration(); + if (luceneOptions.SearchFields == null || luceneOptions.SearchFields.Length == 0) + { + throw new ConfigurationErrorsException("Search fields not configured for Lucene index"); + } + return luceneOptions.SearchFields; + } } + } diff --git a/Px.Search.Lucene/LuceneAnalyzer.cs b/Px.Search.Lucene/LuceneAnalyzer.cs index ea8e0fa9..8a4b8abd 100644 --- a/Px.Search.Lucene/LuceneAnalyzer.cs +++ b/Px.Search.Lucene/LuceneAnalyzer.cs @@ -1,10 +1,12 @@ -namespace Px.Search.Lucene +using Lucene.Net.Analysis.Miscellaneous; + +namespace Px.Search.Lucene { - public class LuceneAnalyzer + public static class LuceneAnalyzer { - internal static LuceneVersion luceneVersion = LuceneVersion.LUCENE_48; + internal const LuceneVersion luceneVersion = LuceneVersion.LUCENE_48; - internal static Analyzer GetAnalyzer(string language) + internal static Analyzer GetDefaultAnalyzer(string language) { switch (language) { @@ -44,9 +46,26 @@ internal static Analyzer GetAnalyzer(string language) } - } - // TODO ? Should read langs from config and prepare a static analyzerByLanguage dictionary? + internal static Analyzer GetAnalyzer(string language) + { + var defaultAnalyzer = GetDefaultAnalyzer(language); + var keywordAnalyzer = new CaseInsensitiveKeywordAnalyzer(); - // depricated: Analyzer analyzer = new SnowballAnalyzer(LuceneVersion.LUCENE_48, "English"); + return new PerFieldAnalyzerWrapper(defaultAnalyzer, + new Dictionary + { + { SearchConstants.SEARCH_FIELD_TAGS, keywordAnalyzer }, + { SearchConstants.SEARCH_FIELD_MATRIX, keywordAnalyzer }, + { SearchConstants.SEARCH_FIELD_CODES, keywordAnalyzer }, + { SearchConstants.SEARCH_FIELD_GROUPINGCODES, keywordAnalyzer }, + { SearchConstants.SEARCH_FIELD_VALUESETCODES, keywordAnalyzer }, + { SearchConstants.SEARCH_FIELD_LEVEL_CODE, keywordAnalyzer }, + { SearchConstants.SEARCH_FIELD_META_ID, keywordAnalyzer }, + { SearchConstants.SEARCH_FIELD_LEGACY_ID, keywordAnalyzer } + }); + + } + + } } diff --git a/Px.Search.Lucene/LuceneBackend.cs b/Px.Search.Lucene/LuceneBackend.cs index f9fca7b7..281abd9f 100644 --- a/Px.Search.Lucene/LuceneBackend.cs +++ b/Px.Search.Lucene/LuceneBackend.cs @@ -6,11 +6,13 @@ public class LuceneBackend : ISearchBackend private readonly ILuceneConfigurationService _luceneConfigurationService; private readonly string _path; + private readonly string[] _searchFields; public LuceneBackend(ILuceneConfigurationService luceneConfigurationService) { _luceneConfigurationService = luceneConfigurationService; _path = _luceneConfigurationService.GetIndexDirectoryPath(); + _searchFields = _luceneConfigurationService.GetSearchFields(); } public IIndex GetIndex() @@ -20,7 +22,7 @@ public IIndex GetIndex() public ISearcher GetSearcher(string language) { - return new LuceneSearcher(_path, language); + return new LuceneSearcher(_path, language, _searchFields); } } } diff --git a/Px.Search.Lucene/LuceneIndex.cs b/Px.Search.Lucene/LuceneIndex.cs index 447bba75..63bd4233 100644 --- a/Px.Search.Lucene/LuceneIndex.cs +++ b/Px.Search.Lucene/LuceneIndex.cs @@ -222,6 +222,31 @@ private static Document GetDocument(TableInformation tbl, PXMeta meta) doc.Add(new TextField(SearchConstants.SEARCH_FIELD_TIME_UNIT, tbl.TimeUnit, Field.Store.YES)); doc.Add(new TextField(SearchConstants.SEARCH_SUBJECT_CODE, tbl.SubjectCode, Field.Store.YES)); doc.Add(new StoredField(SearchConstants.SEARCH_FIELD_PATHS, GetBytes(tbl.Paths))); + + foreach (var path in tbl.Paths) + { + foreach (var level in path) + { + doc.Add(new TextField(SearchConstants.SEARCH_FIELD_LEVEL_CODE, level.Code, Field.Store.NO)); + doc.Add(new TextField(SearchConstants.SEARCH_FIELD_LEVEL_NAME, level.Text, Field.Store.NO)); + } + } + + if (!string.IsNullOrWhiteSpace(meta.MetaId)) + { + doc.Add(new TextField(SearchConstants.SEARCH_FIELD_META_ID, meta.MetaId, Field.Store.NO)); + + foreach (var variable in meta.Variables.Where(v => !string.IsNullOrWhiteSpace(v.MetaId))) + { + doc.Add(new TextField(SearchConstants.SEARCH_FIELD_META_ID, variable.MetaId, Field.Store.NO)); + } + } + + if (!string.IsNullOrWhiteSpace(meta.MainTable)) + { + doc.Add(new TextField(SearchConstants.SEARCH_FIELD_META_ID, meta.MainTable, Field.Store.NO)); + } + doc.Add(new StoredField(SearchConstants.SEARCH_AVAILABLE_LANGUAGES, string.Join("|", tbl.Languages))); if (!string.IsNullOrEmpty(meta.Synonyms)) { diff --git a/Px.Search.Lucene/LuceneSearcher.cs b/Px.Search.Lucene/LuceneSearcher.cs index cb08de88..3a8a722e 100644 --- a/Px.Search.Lucene/LuceneSearcher.cs +++ b/Px.Search.Lucene/LuceneSearcher.cs @@ -9,12 +9,14 @@ public class LuceneSearcher : ISearcher private readonly IndexSearcher _indexSearcher; private static readonly Operator _defaultOperator = Operator.AND; private readonly Analyzer _analyzer; + private readonly string[] _fieldNames; + /// /// Constructor /// /// Index directory - public LuceneSearcher(string indexDirectory, string language) + public LuceneSearcher(string indexDirectory, string language, string[] fieldNames) { if (string.IsNullOrWhiteSpace(indexDirectory)) { @@ -26,6 +28,7 @@ public LuceneSearcher(string indexDirectory, string language) IndexReader reader = DirectoryReader.Open(fsDir); _indexSearcher = new IndexSearcher(reader); _analyzer = LuceneAnalyzer.GetAnalyzer(language); + _fieldNames = fieldNames; } @@ -43,7 +46,7 @@ public SearchResultContainer Find(string? query, int pageSize, int pageNumber, i var skipRecords = pageSize * (pageNumber - 1); var searchResultContainer = new SearchResultContainer(); var searchResultList = new List(); - string[] fields = GetSearchFields(); + string[] fields = _fieldNames; Query luceneQuery; QueryParser queryParser = new MultiFieldQueryParser(LuceneAnalyzer.luceneVersion, fields, @@ -171,40 +174,5 @@ private static SearchResult GetSearchResult(Document doc) return searchResult; } - - /// - /// Get fields in index to search in - /// - /// - private static string[] GetSearchFields() - { - string[] fields; - - // Default fields - fields = new[] { SearchConstants.SEARCH_FIELD_DOCID, - SearchConstants.SEARCH_FIELD_SEARCHID, - SearchConstants.SEARCH_FIELD_UPDATED, - SearchConstants.SEARCH_FIELD_MATRIX, - SearchConstants.SEARCH_FIELD_TITLE, - SearchConstants.SEARCH_FIELD_DESCRIPTION, - SearchConstants.SEARCH_FIELD_SORTCODE, - SearchConstants.SEARCH_FIELD_CATEGORY, - SearchConstants.SEARCH_FIELD_FIRSTPERIOD, - SearchConstants.SEARCH_FIELD_LASTPERIOD, - SearchConstants.SEARCH_FIELD_VARIABLES, - SearchConstants.SEARCH_FIELD_PERIOD, - SearchConstants.SEARCH_FIELD_VALUES, - SearchConstants.SEARCH_FIELD_CODES, - SearchConstants.SEARCH_FIELD_GROUPINGS, - SearchConstants.SEARCH_FIELD_GROUPINGCODES, - SearchConstants.SEARCH_FIELD_VALUESETS, - SearchConstants.SEARCH_FIELD_VALUESETCODES, - SearchConstants.SEARCH_FIELD_DISCONTINUED, - SearchConstants.SEARCH_FIELD_TAGS - }; - - return fields; - } - } } diff --git a/Px.Search.Lucene/SearchConstants.cs b/Px.Search.Lucene/SearchConstants.cs index 5c3187ee..29baca69 100644 --- a/Px.Search.Lucene/SearchConstants.cs +++ b/Px.Search.Lucene/SearchConstants.cs @@ -27,6 +27,10 @@ public class SearchConstants public const string SEARCH_FIELD_SOURCE = "source"; public const string SEARCH_FIELD_TIME_UNIT = "timeunit"; public const string SEARCH_FIELD_PATHS = "paths"; + public const string SEARCH_FIELD_LEVEL_CODE = "levelcode"; + public const string SEARCH_FIELD_LEVEL_NAME = "levelname"; + public const string SEARCH_FIELD_META_ID = "metaid"; + public const string SEARCH_FIELD_LEGACY_ID = "legacyid"; public const string SEARCH_SUBJECT_CODE = "subjectcode"; public const string SEARCH_AVAILABLE_LANGUAGES = "languages"; } diff --git a/PxWeb.UnitTests/Search/LuceneAnalyzerTest.cs b/PxWeb.UnitTests/Search/LuceneAnalyzerTest.cs index 27fa9d7a..49280de7 100644 --- a/PxWeb.UnitTests/Search/LuceneAnalyzerTest.cs +++ b/PxWeb.UnitTests/Search/LuceneAnalyzerTest.cs @@ -21,6 +21,7 @@ private void TestSearcher(string language, string searchFor, int expectedCount) { LuceneConfigurationOptions luceneConfigurationOptions = new LuceneConfigurationOptions(); luceneConfigurationOptions.IndexDirectory = @"Database/_INDEX/"; + luceneConfigurationOptions.SearchFields = new string[] { "docid", "searchid", "updated", "matrix", "title", "description", "sortcode", "category", "firstperiod", "lastperiod", "variables", "period", "values", "codes", "groupings", "groupingcodes", "valuesets", "valuesetcodes", "discontinued", "tags" }; //seeking "C:\\repos\\github\\pxtools\\PxwebApi3\\PxWebApi\\PxWeb\\wwwroot" string pathRunning = Directory.GetCurrentDirectory(); diff --git a/PxWeb.UnitTests/Search/LuceneIndexTests.cs b/PxWeb.UnitTests/Search/LuceneIndexTests.cs index 3c246f14..5d4705cd 100644 --- a/PxWeb.UnitTests/Search/LuceneIndexTests.cs +++ b/PxWeb.UnitTests/Search/LuceneIndexTests.cs @@ -1,56 +1,63 @@ using System; +using System.Text; +using System.Text.Json; namespace PxWeb.UnitTests.Search { [TestClass] public class LuceneIndexTests { -#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. - private LuceneIndex _luceneIndex; -#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. + private string _indexRoot = string.Empty; + private LuceneIndex _luceneIndex = null!; + [TestInitialize] public void Setup() { - _luceneIndex = new LuceneIndex("testIndexDirectory"); + _indexRoot = Path.Combine(Path.GetTempPath(), "PxWeb_LuceneIndexTests", Guid.NewGuid().ToString("N")); + _luceneIndex = new LuceneIndex(_indexRoot); } + [TestCleanup] + public void Cleanup() + { + _luceneIndex.Dispose(); + + if (Directory.Exists(_indexRoot)) + { + Directory.Delete(_indexRoot, true); + } + } [TestMethod] - public void BeginWrite_ShouldCreateIndexWriter() + public void Constructor_EmptyPath_ThrowsArgumentNullException() { - // Arrange - var language = "en"; + Assert.ThrowsExactly(() => new LuceneIndex("")); + } + [TestMethod] + public void BeginWrite_ShouldCreateIndexWriter() + { try { - // Act - _luceneIndex.BeginWrite(language); - _luceneIndex.EndWrite(language); + _luceneIndex.BeginWrite("en"); + _luceneIndex.EndWrite("en"); } catch (Exception e) { - // Assert Assert.Fail(e.Message); } - - } [TestMethod] public void BeginUpdate_ShouldCreateIndexWriterAndReader() { - // Arrange - var language = "en"; - try { - // Act - _luceneIndex.BeginUpdate(language); - _luceneIndex.EndUpdate(language); + _luceneIndex.BeginUpdate("en"); + _luceneIndex.EndUpdate("en"); } catch (Exception e) { - // Assert Assert.Fail(e.Message); } } @@ -58,90 +65,49 @@ public void BeginUpdate_ShouldCreateIndexWriterAndReader() [TestMethod] public void UpdatedEntry_ShouldCreateIndexWriterAndReader() { - // Arrange - var language = "en"; - var tableInformation = new TableInformation( - "TAB001", - "Population in the world", - "Population", - "2000", - "2005", - new string[] { "TIME" }); - tableInformation.Description = "Test"; - tableInformation.SortCode = "001"; - tableInformation.Paths.Add(new Level[] { new Level("A", "Test", "A") }); - - var meta = new PXMeta(); - meta.Matrix = "TAB001"; - meta.Variables.Add(ModelStore.CreateTimeVariable("", PlacementType.Stub, 2000, 2005)); + var tableInformation = CreateValidTableInformation("TAB001"); + var meta = CreateValidMeta("TAB001"); try { - // Act - _luceneIndex.BeginUpdate(language); + _luceneIndex.BeginUpdate("en"); _luceneIndex.UpdateEntry(tableInformation, meta); - _luceneIndex.EndUpdate(language); + _luceneIndex.EndUpdate("en"); } catch (Exception e) { - // Assert Assert.Fail(e.Message); } } - [TestMethod] public void NewLucenIndex_NoPath_ShouldThrowExcpetion() { - // Assert - Assert.ThrowsExactly(() => { var index = new LuceneIndex(""); }); + Assert.ThrowsExactly(() => new LuceneIndex("")); } - [TestMethod] public void UsageWithoutOpening_ShouldThrowExcpetion() { + var tableInformation = CreateValidTableInformation("TAB001", includePaths: true); + var meta = CreateValidMeta("TAB001"); - // Arrange - - var tableInformation = new TableInformation( - "TAB001", - "Population in the world", - "Population", - "2000", - "2005", - new string[] { "TIME" }); - tableInformation.Description = "Test"; - tableInformation.SortCode = "001"; - tableInformation.Paths.Add(new Level[] { new Level("A", "Test", "A") }); - - var meta = new PXMeta(); - meta.Matrix = "TAB001"; - meta.Variables.Add(ModelStore.CreateTimeVariable("", PlacementType.Stub, 2000, 2005)); - - - // Assert.ThrowsExactly(() => _luceneIndex.AddEntry(tableInformation, meta)); Assert.ThrowsExactly(() => _luceneIndex.RemoveEntry("removable")); - - //update with path Assert.ThrowsExactly(() => _luceneIndex.UpdateEntry(tableInformation, meta)); - //update withOut path tableInformation.Paths.Clear(); Assert.ThrowsExactly(() => _luceneIndex.UpdateEntry(tableInformation, meta)); - - } [TestMethod] public void Dispose_ShouldNotThrowExcpetion() { + var tempIndex = Path.Combine(Path.GetTempPath(), "PxWeb_LuceneIndexTests", Guid.NewGuid().ToString("N")); - // Arrange try { - LuceneIndex luceneIndex2 = new LuceneIndex("testIndex2Directory"); + var luceneIndex2 = new LuceneIndex(tempIndex); luceneIndex2.Dispose(); luceneIndex2.Dispose(); } @@ -149,11 +115,265 @@ public void Dispose_ShouldNotThrowExcpetion() { Assert.Fail(); } + finally + { + if (Directory.Exists(tempIndex)) + { + Directory.Delete(tempIndex, true); + } + } } + [TestMethod] + public void BeginWrite_EmptyLanguage_ThrowsArgumentNullException() + { + Assert.ThrowsExactly(() => _luceneIndex.BeginWrite("")); + } + [TestMethod] + public void BeginUpdate_EmptyLanguage_ThrowsArgumentNullException() + { + Assert.ThrowsExactly(() => _luceneIndex.BeginUpdate("")); + } + [TestMethod] + public void BeginWrite_WhenIndexAlreadyLocked_ThrowsIOException() + { + _luceneIndex.BeginWrite("en"); - } + Assert.ThrowsExactly(() => _luceneIndex.BeginWrite("en")); + } + + [TestMethod] + public void EndWrite_And_EndUpdate_WhenNotStarted_DoNotThrow() + { + try + { + _luceneIndex.EndWrite("en"); + _luceneIndex.EndUpdate("en"); + } + catch (Exception) + { + Assert.Fail("EndWrite and EndUpdate should not throw when not started."); + } + } + + [TestMethod] + public void AddEntry_WhenWriterIsMissing_ThrowsInvalidOperationException() + { + var table = CreateValidTableInformation("TAB001"); + var meta = CreateValidMeta("TAB001"); + + Assert.ThrowsExactly(() => _luceneIndex.AddEntry(table, meta)); + } + + [TestMethod] + public void RemoveEntry_WhenWriterIsMissing_ThrowsInvalidOperationException() + { + Assert.ThrowsExactly(() => _luceneIndex.RemoveEntry("TAB001")); + } + + [TestMethod] + public void FindDocument_WhenSearcherIsMissing_ThrowsInvalidOperationException() + { + Assert.ThrowsExactly(() => _luceneIndex.FindDocument("TAB001")); + } + + [TestMethod] + public void UpdateEntry_WhenWriterMissing_AndPathsExist_ThrowsInvalidOperationException() + { + var table = CreateValidTableInformation("TAB001", includePaths: true); + var meta = CreateValidMeta("TAB001"); + + Assert.ThrowsExactly(() => _luceneIndex.UpdateEntry(table, meta)); + } + + [TestMethod] + public void UpdateEntry_WhenWriterAndSearcherMissing_AndPathsAreEmpty_ThrowsInvalidOperationException() + { + var table = CreateValidTableInformation("TAB001", includePaths: false); + var meta = CreateValidMeta("TAB001"); + + Assert.ThrowsExactly(() => _luceneIndex.UpdateEntry(table, meta)); + } + + [TestMethod] + public void AddAndFindDocument_ReturnsStoredDocument_AndMissingReturnsNull() + { + var table = CreateValidTableInformation("TAB001"); + table.Updated = new DateTime(2024, 1, 1, 12, 0, 0, DateTimeKind.Utc); + table.Discontinued = true; + + var meta = CreateValidMeta("TAB001", includeOptionalMetaFields: true); + + _luceneIndex.BeginWrite("en"); + _luceneIndex.AddEntry(table, meta); + _luceneIndex.EndWrite("en"); + + _luceneIndex.BeginUpdate("en"); + var found = _luceneIndex.FindDocument("tab001"); + var notFound = _luceneIndex.FindDocument("does-not-exist"); + _luceneIndex.EndUpdate("en"); + + Assert.IsNotNull(found); + Assert.AreEqual("TAB001", found.Get(SearchConstants.SEARCH_FIELD_DOCID)); + Assert.AreEqual("Population in the world", found.Get(SearchConstants.SEARCH_FIELD_TITLE)); + Assert.AreEqual("True", found.Get(SearchConstants.SEARCH_FIELD_DISCONTINUED)); + Assert.IsFalse(string.IsNullOrWhiteSpace(found.Get(SearchConstants.SEARCH_FIELD_UPDATED))); + Assert.IsNull(notFound); + } + + [TestMethod] + public void AddEntry_WhenRequiredDocumentDataMissing_ThrowsInvalidOperationException() + { + var table = CreateValidTableInformation("TAB001"); + table.Label = string.Empty; + + var meta = CreateValidMeta("TAB001"); + + _luceneIndex.BeginWrite("en"); + + Assert.ThrowsExactly(() => _luceneIndex.AddEntry(table, meta)); + } + + [TestMethod] + public void UpdateEntry_WithEmptyPaths_RestoresStoredPathsFromExistingDocument() + { + var original = CreateValidTableInformation("TAB001", includePaths: true); + var meta = CreateValidMeta("TAB001"); + + _luceneIndex.BeginWrite("en"); + _luceneIndex.AddEntry(original, meta); + _luceneIndex.EndWrite("en"); + + var updated = CreateValidTableInformation("TAB001", includePaths: false); + updated.Label = "Updated label"; + + _luceneIndex.BeginUpdate("en"); + _luceneIndex.UpdateEntry(updated, meta); + _luceneIndex.EndUpdate("en"); + + _luceneIndex.BeginUpdate("en"); + var found = _luceneIndex.FindDocument("tab001"); + _luceneIndex.EndUpdate("en"); + + Assert.IsNotNull(found); + var bytes = found.GetBinaryValue(SearchConstants.SEARCH_FIELD_PATHS); + Assert.IsNotNull(bytes); + + var json = Encoding.UTF8.GetString(bytes.Bytes, bytes.Offset, bytes.Length); + var restoredPaths = JsonSerializer.Deserialize>(json); + + Assert.IsNotNull(restoredPaths); + Assert.HasCount(1, restoredPaths); + Assert.AreEqual("A", restoredPaths[0][0].Code); + } + + [TestMethod] + public void UpdateEntry_WithEmptyPaths_AndNoPreviousDocument_KeepsEmptyPaths() + { + var table = CreateValidTableInformation("TAB001", includePaths: false); + var meta = CreateValidMeta("TAB001"); + + // Create an initial (empty) index so BeginUpdate can create an IndexSearcher. + _luceneIndex.BeginWrite("en"); + _luceneIndex.EndWrite("en"); + + _luceneIndex.BeginUpdate("en"); + _luceneIndex.UpdateEntry(table, meta); + _luceneIndex.EndUpdate("en"); + + _luceneIndex.BeginUpdate("en"); + var found = _luceneIndex.FindDocument("tab001"); + _luceneIndex.EndUpdate("en"); + + Assert.IsNotNull(found); + + var bytes = found.GetBinaryValue(SearchConstants.SEARCH_FIELD_PATHS); + Assert.IsNotNull(bytes); + + var json = Encoding.UTF8.GetString(bytes.Bytes, bytes.Offset, bytes.Length); + var paths = JsonSerializer.Deserialize>(json); + + Assert.IsNotNull(paths); + Assert.IsEmpty(paths); + } + [TestMethod] + public void Dispose_CanBeCalledMultipleTimes_WithActiveWriter() + { + try + { + + _luceneIndex.BeginWrite("en"); + + _luceneIndex.Dispose(); + _luceneIndex.Dispose(); + } + catch (Exception) + { + Assert.Fail("Calling Dispose multiple times should not throw exception"); + } + } + + [TestMethod] + public void GetAllTags_ReturnsConcatenatedString_AndHandlesEmptyArray() + { + var withTags = LuceneIndex.GetAllTags(new[] { "tag1", "tag2" }); + var emptyTags = LuceneIndex.GetAllTags(Array.Empty()); + + Assert.AreEqual("tag1 tag2 ", withTags); + Assert.AreEqual(string.Empty, emptyTags); + } + + private static TableInformation CreateValidTableInformation(string id, bool includePaths = true) + { + var table = new TableInformation( + id, + "Population in the world", + "Population", + "2000", + "2005", + new[] { "TIME", "SEX" }) + { + Description = "Test description", + SortCode = "001", + Source = "SCB", + TimeUnit = "A", + SubjectCode = "SUBJ", + Languages = new[] { "en", "sv" } + }; + + if (includePaths) + { + table.Paths.Add(new[] { new Level("A", "Level A", "A") }); + } + + return table; + } + + private static PXMeta CreateValidMeta(string matrix, bool includeOptionalMetaFields = false) + { + var meta = new PXMeta + { + Matrix = matrix + }; + + var timeVariable = ModelStore.CreateTimeVariable("TIME", PlacementType.Stub, 2000, 2005); + var nonTimeVariable = ModelStore.CreateClassificationVariable("sex", PlacementType.Stub, 2); + + if (includeOptionalMetaFields) + { + meta.MetaId = "META-ROOT"; + meta.MainTable = "MAIN-TABLE"; + meta.Synonyms = "population inhabitants"; + nonTimeVariable.MetaId = "META-VARIABLE"; + } + + meta.Variables.Add(timeVariable); + meta.Variables.Add(nonTimeVariable); + + return meta; + } + } } diff --git a/PxWeb/appsettings.json b/PxWeb/appsettings.json index c611f95f..63683dbd 100644 --- a/PxWeb/appsettings.json +++ b/PxWeb/appsettings.json @@ -61,7 +61,30 @@ "OmitContentsInTitle": true }, "LuceneConfiguration": { - "IndexDirectory": "Database/_INDEX/" + "IndexDirectory": "Database/_INDEX/", + "SearchFields": [ + "docid", + "searchid", + "updated", + "matrix", + "title", + "description", + "sortcode", + "category", + "firstperiod", + "lastperiod", + "variables", + "period", + "values", + "codes", + "groupings", + "groupingcodes", + "valuesets", + "valuesetcodes", + "discontinued", + "tags" + ] + }, "AdminProtection": { "IpWhitelist": ["127.0.0.1", "::1", "::ffff:127.0.0.1"], diff --git a/PxWeb/wwwroot/Database/Menu2.xml b/PxWeb/wwwroot/Database/Menu2.xml index d54795a8..8eddc4b5 100644 --- a/PxWeb/wwwroot/Database/Menu2.xml +++ b/PxWeb/wwwroot/Database/Menu2.xml @@ -86,9 +86,9 @@ 4 16412 2448 - 20250305 15:38 + 20260202 10:26 20230525 16:13 - 2025-03-05 15:38:31 + 2026-02-02 10:26:28 2023-05-25 16:13:00 2010 2016 @@ -122,11 +122,11 @@ year 1981, 1991, 2001, 3 - 46967 + 47340 5940 - 20250906 21:22 + 20260602 09:14 20230525 15:42 - 2025-09-06 21:22:02 + 2026-06-02 09:14:14 2023-05-25 15:42:00 1981 2001 @@ -198,11 +198,11 @@ år 1981, 1991, 2001, 3 - 46967 + 47340 5940 - 20250906 21:22 + 20260602 09:14 20230525 15:42 - 2025-09-06 21:22:02 + 2026-06-02 09:14:14 2023-05-25 15:42:00 1981 2001 @@ -266,9 +266,9 @@ 4 16412 2448 - 20250305 15:38 + 20260202 10:26 20230525 16:13 - 2025-03-05 15:38:31 + 2026-02-02 10:26:28 2023-05-25 16:13:00 2010 2016