From 4c452f23c546624306baeb162b55ca961b8de9aa Mon Sep 17 00:00:00 2001 From: "Aaron (Qilong)" <173288704@qq.com> Date: Thu, 12 Oct 2023 12:21:15 -0400 Subject: [PATCH 1/5] DYN-6038 Lucene search IMP (#14428) * improvements * Simply initialization logic * Skip indexing as part of DynamoModel if the index files already exist * Remove DynamoModel reference and use singleton * clean up * improvements and package manager search * Use single constructor and other code clean up * Update... * Update * Dispose all Lucene objects in the correct order * Adding DynamoModel back to test regressions * Null check * Update * IndexWriter dispose sequence only for non-RAM mode --- src/DynamoCore/Models/DynamoModel.cs | 57 ++---- src/DynamoCore/Search/NodeSearchModel.cs | 12 +- .../Utilities/LuceneSearchUtility.cs | 192 ++++++++++++++---- .../ViewModels/Core/WorkspaceViewModel.cs | 1 + .../PackageManagerSearchViewModel.cs | 15 +- .../Search/NodeAutoCompleteSearchViewModel.cs | 27 +-- .../ViewModels/Search/SearchViewModel.cs | 5 +- test/DynamoCoreTests/SearchModelTests.cs | 2 +- .../SearchViewModelTests.cs | 2 +- 9 files changed, 200 insertions(+), 113 deletions(-) diff --git a/src/DynamoCore/Models/DynamoModel.cs b/src/DynamoCore/Models/DynamoModel.cs index f955094aa41..1af53762317 100644 --- a/src/DynamoCore/Models/DynamoModel.cs +++ b/src/DynamoCore/Models/DynamoModel.cs @@ -925,16 +925,7 @@ protected DynamoModel(IStartConfiguration config) CustomNodeManager = new CustomNodeManager(NodeFactory, MigrationManager, LibraryServices); - LuceneSearch.LuceneUtilityNodeSearch = new LuceneSearchUtility(this); - - if (IsTestMode) - { - LuceneUtility.InitializeLuceneConfig(string.Empty, LuceneSearchUtility.LuceneStorage.RAM); - } - else - { - LuceneUtility.InitializeLuceneConfig(LuceneConfig.NodesIndexingDirectory); - } + LuceneSearch.LuceneUtilityNodeSearch = new LuceneSearchUtility(this, LuceneSearchUtility.DefaultNodeIndexStartConfig); InitializeCustomNodeManager(); @@ -1017,10 +1008,15 @@ protected DynamoModel(IStartConfiguration config) TraceReconciliationProcessor = this; State = DynamoModelState.StartedUIless; - // This event should only be raised at the end of this method. - DynamoReady(new ReadyParams(this)); // Write index to disk only once LuceneUtility.CommitWriterChanges(); + //Disposed writer if it is in file system mode so that the index files can be used by other processes (potentially a second Dynamo session) + if (LuceneUtility.startConfig.StorageType == LuceneSearchUtility.LuceneStorage.FILE_SYSTEM) + { + LuceneUtility.DisposeWriter(); + } + // This event should only be raised at the end of this method. + DynamoReady(new ReadyParams(this)); } /// @@ -1429,12 +1425,8 @@ public void Dispose() PreferenceSettings.MessageLogged -= LogMessage; } - //The writer have to be disposed at DynamoModel level due that we could index package-nodes as new packages are installed - LuceneUtility.DisposeWriter(); - // Lucene disposals (just if LuceneNET was initialized) - LuceneUtility.indexDir?.Dispose(); - LuceneUtility.dirReader?.Dispose(); + LuceneUtility.DisposeAll(); #if DEBUG CurrentWorkspace.NodeAdded -= CrashOnDemand.CurrentWorkspace_NodeAdded; @@ -1495,7 +1487,7 @@ private void InitializeCustomNodeManager() var iDoc = LuceneUtility.InitializeIndexDocumentForNodes(); if (searchElement != null) { - AddNodeTypeToSearchIndex(searchElement, iDoc); + LuceneUtility.AddNodeTypeToSearchIndex(searchElement, iDoc); } Action infoUpdatedHandler = null; @@ -1562,7 +1554,9 @@ private void InitializeIncludedNodes() NodeFactory.AddTypeFactoryAndLoader(outputData.Type); NodeFactory.AddAlsoKnownAs(outputData.Type, outputData.AlsoKnownAs); - SearchModel?.Add(new CodeBlockNodeSearchElement(cbnData, LibraryServices)); + var cnbNode = new CodeBlockNodeSearchElement(cbnData, LibraryServices); + SearchModel?.Add(cnbNode); + LuceneUtility.AddNodeTypeToSearchIndex(cnbNode, iDoc); var symbolSearchElement = new NodeModelSearchElement(symbolData) { @@ -1581,7 +1575,10 @@ private void InitializeIncludedNodes() }; SearchModel?.Add(symbolSearchElement); + LuceneUtility.AddNodeTypeToSearchIndex(symbolSearchElement, iDoc); + SearchModel?.Add(outputSearchElement); + LuceneUtility.AddNodeTypeToSearchIndex(outputSearchElement, iDoc); //Adding this nodes are breaking the tests (due that we have two input and two output nodes): //WhenHomeWorkspaceIsFocusedInputAndOutputNodesAreMissingFromSearch @@ -1756,7 +1753,7 @@ private void LoadNodeModels(List nodes, bool isPackageMember) // TODO: get search element some other way if (ele != null) { - AddNodeTypeToSearchIndex(ele, iDoc); + LuceneUtility.AddNodeTypeToSearchIndex(ele, iDoc); } } catch (Exception e) @@ -3286,24 +3283,6 @@ private NodeModelSearchElement AddNodeTypeToSearch(TypeLoadData typeLoadData) return node; } - /// - /// Add node information to Lucene index - /// - /// node info that will be indexed - /// Lucene document in which the node info will be indexed - internal void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc) - { - if (LuceneUtility.addedFields == null) return; - - LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), node.FullCategoryName); - LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Name), node.Name); - LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Description), node.Description); - if (node.SearchKeywords.Count > 0) LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.SearchKeywords), node.SearchKeywords.Aggregate((x, y) => x + " " + y), true, true); - LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Parameters), node.Parameters ?? string.Empty); - - LuceneUtility.writer?.AddDocument(doc); - } - /// /// Remove node information from Lucene indexing. /// @@ -3363,7 +3342,7 @@ private void AddZeroTouchNodeToSearch(FunctionDescriptor functionDescriptor, Doc { var ele = new ZeroTouchSearchElement(functionDescriptor); SearchModel?.Add(ele); - AddNodeTypeToSearchIndex(ele, iDoc); + LuceneUtility.AddNodeTypeToSearchIndex(ele, iDoc); } } diff --git a/src/DynamoCore/Search/NodeSearchModel.cs b/src/DynamoCore/Search/NodeSearchModel.cs index c0d84de2104..e828d86b509 100644 --- a/src/DynamoCore/Search/NodeSearchModel.cs +++ b/src/DynamoCore/Search/NodeSearchModel.cs @@ -21,6 +21,7 @@ using Lucene.Net.Analysis.Ru; using Lucene.Net.Analysis.Standard; using Lucene.Net.Documents; +using Lucene.Net.Index; using Lucene.Net.QueryParsers.Classic; using Lucene.Net.Search; @@ -246,9 +247,14 @@ internal IEnumerable Search(string search, LuceneSearchUtilit if (luceneSearchUtility != null) { //The DirectoryReader and IndexSearcher have to be assigned after commiting indexing changes and before executing the Searcher.Search() method, otherwise new indexed info won't be reflected - luceneSearchUtility.dirReader = luceneSearchUtility.writer?.GetReader(applyAllDeletes: true); - if (luceneSearchUtility.dirReader == null) return null; - + if (luceneSearchUtility.writer != null) + { + luceneSearchUtility.dirReader = luceneSearchUtility.writer.GetReader(applyAllDeletes: true); + } + else + { + luceneSearchUtility.dirReader = DirectoryReader.Open(luceneSearchUtility.indexDir); + } luceneSearchUtility.Searcher = new IndexSearcher(luceneSearchUtility.dirReader); string searchTerm = search.Trim(); diff --git a/src/DynamoCore/Utilities/LuceneSearchUtility.cs b/src/DynamoCore/Utilities/LuceneSearchUtility.cs index 7f8746095a5..72bebe83a61 100644 --- a/src/DynamoCore/Utilities/LuceneSearchUtility.cs +++ b/src/DynamoCore/Utilities/LuceneSearchUtility.cs @@ -3,7 +3,12 @@ using System.IO; using System.Linq; using Dynamo.Configuration; +using Dynamo.Core; +using Dynamo.Events; +using Dynamo.Logging; using Dynamo.Models; +using Dynamo.Search.SearchElements; +using Dynamo.Session; using Lucene.Net.Analysis; using Lucene.Net.Analysis.Br; using Lucene.Net.Analysis.Cjk; @@ -25,15 +30,52 @@ namespace Dynamo.Utilities { + /// + /// Lucene search utility class that will be used for indexing and searching nodes and packages + /// internal class LuceneSearchUtility { internal DynamoModel dynamoModel; + + /// + /// Index fields that were added to the document + /// internal List addedFields; + + /// + /// Lucene Directory Reader + /// internal DirectoryReader dirReader; + + /// + /// Lucene Index Directory, it can be RAMDirectory or FSDirectory + /// internal Lucene.Net.Store.Directory indexDir; + + /// + /// Lucene Index write + /// internal IndexWriter writer; - internal string directory; - internal LuceneStorage currentStorageType; + + /// + /// Start config for Lucene + /// + internal LuceneStartConfig startConfig; + + /// + /// Default start config for Lucene, it will use RAM storage type and empty directory + /// + internal static readonly LuceneStartConfig DefaultStartConfig = new LuceneStartConfig(); + + /// + /// Start config for node index, it will use file storage type and node index directory + /// + internal static readonly LuceneStartConfig DefaultNodeIndexStartConfig = new LuceneStartConfig(LuceneSearchUtility.LuceneStorage.FILE_SYSTEM, LuceneConfig.NodesIndexingDirectory); + + /// + /// Start config for package index, it will use file storage type and package index directory + /// + internal static readonly LuceneStartConfig DefaultPkgIndexStartConfig = new LuceneStartConfig(LuceneSearchUtility.LuceneStorage.FILE_SYSTEM, LuceneConfig.PackagesIndexingDirectory); public enum LuceneStorage { @@ -50,15 +92,22 @@ public enum LuceneStorage // Holds the instance for the IndexSearcher internal IndexSearcher Searcher; - internal LuceneSearchUtility(DynamoModel model) + /// + /// Constructor for LuceneSearchUtility, it will use the storage type passed as parameter + /// + /// + internal LuceneSearchUtility(DynamoModel model, LuceneStartConfig config) { dynamoModel = model; + // If under test mode, use the default StartConfig - RAM storage type and empty directory + startConfig = DynamoModel.IsTestMode? DefaultStartConfig : config; + InitializeLuceneConfig(); } /// - /// Initialize Lucene config file writer. + /// Initialize Lucene index writer based on start config. /// - internal void InitializeLuceneConfig(string dirName, LuceneStorage storageType = LuceneStorage.FILE_SYSTEM) + internal void InitializeLuceneConfig() { if (DynamoModel.IsHeadless) return; @@ -67,13 +116,9 @@ internal void InitializeLuceneConfig(string dirName, LuceneStorage storageType = DirectoryInfo luceneUserDataFolder; var userDataDir = new DirectoryInfo(dynamoModel.PathManager.UserDataDirectory); luceneUserDataFolder = userDataDir.Exists ? userDataDir : null; + string indexPath = Path.Combine(luceneUserDataFolder.FullName, LuceneConfig.Index, startConfig.Directory); - directory = dirName; - string indexPath = Path.Combine(luceneUserDataFolder.FullName, LuceneConfig.Index, dirName); - - currentStorageType = storageType; - - if (storageType == LuceneStorage.RAM) + if (startConfig.StorageType == LuceneStorage.RAM) { indexDir = new RAMDirectory(); } @@ -81,33 +126,41 @@ internal void InitializeLuceneConfig(string dirName, LuceneStorage storageType = { indexDir = FSDirectory.Open(indexPath); } - - // Create an analyzer to process the text Analyzer = CreateAnalyzerByLanguage(dynamoModel.PreferenceSettings.Locale); + CreateLuceneIndexWriter(); + } - // Initialize Lucene index writer, unless in test mode or we are using RAMDirectory for indexing info. - if (!DynamoModel.IsTestMode || currentStorageType == LuceneStorage.RAM) + /// + /// Create index writer for followup doc indexing + /// + internal void CreateLuceneIndexWriter() + { + // Create an index writer + IndexWriterConfig indexConfig = new IndexWriterConfig(LuceneConfig.LuceneNetVersion, Analyzer) + { + OpenMode = OpenMode.CREATE + }; + try + { + writer = new IndexWriter(indexDir, indexConfig); + } + catch (LockObtainFailedException ex) { try { - // Create an index writer - IndexWriterConfig indexConfig = new IndexWriterConfig(LuceneConfig.LuceneNetVersion, Analyzer) - { - OpenMode = OpenMode.CREATE - }; - - writer = new IndexWriter(indexDir, indexConfig); + writer = new IndexWriter(new RAMDirectory(), indexConfig); + (ExecutionEvents.ActiveSession.GetParameterValue(ParameterKeys.Logger) as DynamoLogger).LogError($"LuceneNET LockObtainFailedException {ex}, switching to RAM mode."); } - catch (LockObtainFailedException ex) + catch(Exception) { DisposeWriter(); - dynamoModel.Logger.LogError($"LuceneNET LockObtainFailedException {ex}"); - } - catch (Exception ex) { - dynamoModel.Logger.LogError($"LuceneNET Exception {ex}"); } } + catch (Exception ex) + { + (ExecutionEvents.ActiveSession.GetParameterValue(ParameterKeys.Logger) as DynamoLogger).LogError($"LuceneNET Exception {ex}"); + } } /// @@ -116,7 +169,7 @@ internal void InitializeLuceneConfig(string dirName, LuceneStorage storageType = /// internal Document InitializeIndexDocumentForNodes() { - if (DynamoModel.IsTestMode && currentStorageType == LuceneStorage.FILE_SYSTEM) return null; + if (DynamoModel.IsTestMode && startConfig.StorageType == LuceneStorage.FILE_SYSTEM) return null; var name = new TextField(nameof(LuceneConfig.NodeFieldsEnum.Name), string.Empty, Field.Store.YES); var fullCategory = new TextField(nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), string.Empty, Field.Store.YES); @@ -173,11 +226,11 @@ internal Document InitializeIndexDocumentForPackages() internal void SetDocumentFieldValue(Document doc, string field, string value, bool isTextField = true, bool isLast = false) { string[] indexedFields = null; - if (directory.Equals(LuceneConfig.NodesIndexingDirectory)) + if (startConfig.Directory.Equals(LuceneConfig.NodesIndexingDirectory)) { indexedFields = LuceneConfig.NodeIndexFields; } - else if (directory.Equals(LuceneConfig.PackagesIndexingDirectory)) + else if (startConfig.Directory.Equals(LuceneConfig.PackagesIndexingDirectory)) { indexedFields = LuceneConfig.PackageIndexFields; } @@ -341,21 +394,53 @@ internal Analyzer CreateAnalyzerByLanguage(string language) internal void DisposeWriter() { - //We need to check if we are not running Dynamo tests because otherwise parallel test start to fail when trying to write in the same Lucene directory location - if (!DynamoModel.IsTestMode || currentStorageType == LuceneStorage.RAM) - { - writer?.Dispose(); - writer = null; - } + writer?.Dispose(); + writer = null; + } + + /// + /// Dispose all the Lucene objects + /// + internal void DisposeAll() + { + writer?.Dispose(); + dirReader?.Dispose(); + indexDir?.Dispose(); + Analyzer?.Dispose(); } + /// + /// Commit the changes made to the Lucene index + /// internal void CommitWriterChanges() { - if (!DynamoModel.IsTestMode || currentStorageType == LuceneStorage.RAM) + //Commit the info indexed if index writer exists + writer?.Commit(); + } + + /// + /// Add node information to existing Lucene index + /// + /// node info that will be indexed + /// Lucene document in which the node info will be indexed + internal void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc) + { + if (addedFields == null) return; + + SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), node.FullCategoryName); + SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Name), node.Name); + SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Description), node.Description); + if (node.SearchKeywords.Count > 0) { - //Commit the info indexed - writer?.Commit(); + SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.SearchKeywords), node.SearchKeywords.Aggregate((x, y) => x + " " + y), true, true); } + SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Parameters), node.Parameters ?? string.Empty); + + if(writer == null) + { + CreateLuceneIndexWriter(); + } + writer?.AddDocument(doc); } } @@ -364,7 +449,7 @@ internal void CommitWriterChanges() /// public class LuceneCustomAnalyzer : Analyzer { - private LuceneVersion luceneVersion; + private readonly LuceneVersion luceneVersion; public LuceneCustomAnalyzer(LuceneVersion matchVersion) { @@ -394,4 +479,31 @@ protected override TokenStreamComponents CreateComponents(string fieldName, Text return new TokenStreamComponents(tokenizer, tok); } } + + /// + /// Start up config for Lucene indexing + /// + internal class LuceneStartConfig + { + /// + /// Lucene Index Directory name, e.g. Nodes, Packages + /// + internal string Directory { get; set; } + + /// + /// Current Lucene Index Storage type, it could be either RAM or FILE_SYSTEM + /// + internal LuceneSearchUtility.LuceneStorage StorageType { get; set; } + + /// + /// Constructor for LuceneStartConfig + /// + /// + /// + internal LuceneStartConfig(LuceneSearchUtility.LuceneStorage storageType = LuceneSearchUtility.LuceneStorage.RAM, string directory = "") + { + Directory = directory; + StorageType = storageType; + } + } } diff --git a/src/DynamoCoreWpf/ViewModels/Core/WorkspaceViewModel.cs b/src/DynamoCoreWpf/ViewModels/Core/WorkspaceViewModel.cs index 4c63be9050b..a563b44b0df 100644 --- a/src/DynamoCoreWpf/ViewModels/Core/WorkspaceViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/Core/WorkspaceViewModel.cs @@ -572,6 +572,7 @@ public override void Dispose() Errors.Clear(); Annotations.Clear(); InCanvasSearchViewModel?.Dispose(); + NodeAutoCompleteSearchViewModel.LuceneUtility?.DisposeAll(); NodeAutoCompleteSearchViewModel?.Dispose(); } diff --git a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs index f6c6583339f..72e6dde5c70 100644 --- a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs @@ -19,6 +19,7 @@ using Dynamo.Wpf.Utilities; using Greg.Responses; using Lucene.Net.Documents; +using Lucene.Net.Index; using Lucene.Net.QueryParsers.Classic; using Lucene.Net.Search; #if NETFRAMEWORK @@ -469,7 +470,7 @@ private void AddPackageToSearchIndex(PackageManagerSearchElement package, Docume LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Name), package.Name); LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Description), package.Description); - if (package.Keywords.Count() > 0) + if (package.Keywords.Length > 0) { LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.SearchKeywords), package.Keywords); } @@ -496,9 +497,8 @@ internal void InitializeLuceneForPackageManager() { if(LuceneUtility == null) { - LuceneSearch.LuceneUtilityPackageManager = new LuceneSearchUtility(PackageManagerClientViewModel.DynamoViewModel.Model); + LuceneSearch.LuceneUtilityPackageManager = new LuceneSearchUtility(PackageManagerClientViewModel.DynamoViewModel.Model, LuceneSearchUtility.DefaultPkgIndexStartConfig); } - LuceneUtility.InitializeLuceneConfig(LuceneConfig.PackagesIndexingDirectory); } /// @@ -796,7 +796,14 @@ public void RefreshAndSearchAsync() if (!DynamoModel.IsTestMode) { - LuceneUtility.dirReader = LuceneUtility.writer?.GetReader(applyAllDeletes: true); + if (LuceneUtility.writer != null) + { + LuceneUtility.dirReader = LuceneUtility.writer.GetReader(applyAllDeletes: true); + } + else + { + LuceneUtility.dirReader = DirectoryReader.Open(LuceneUtility.indexDir); + } LuceneUtility.Searcher = new IndexSearcher(LuceneUtility.dirReader); LuceneUtility.CommitWriterChanges(); diff --git a/src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs index aa2d6799aac..fb2883e2888 100644 --- a/src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs @@ -44,7 +44,7 @@ public class NodeAutoCompleteSearchViewModel : SearchViewModel private const string nodeAutocompleteMLEndpoint = "MLNodeAutocomplete"; // Lucene search utility to perform indexing operations just for NodeAutocomplete. - private LuceneSearchUtility LuceneUtility + internal LuceneSearchUtility LuceneUtility { get { @@ -682,16 +682,13 @@ internal void SearchAutoCompleteCandidates(string input) } else { - LuceneSearch.LuceneUtilityNodeAutocomplete = new LuceneSearchUtility(dynamoViewModel.Model); - - //The dirName parameter doesn't matter because we are using RAMDirectory indexing and no files are created - LuceneUtility.InitializeLuceneConfig(string.Empty, LuceneSearchUtility.LuceneStorage.RAM); + LuceneSearch.LuceneUtilityNodeAutocomplete = new LuceneSearchUtility(dynamoViewModel.Model, LuceneSearchUtility.DefaultStartConfig); //Memory indexing process for Node Autocomplete (indexing just the nodes returned by the NodeAutocomplete service so we limit the scope of the query search) foreach (var node in searchElementsCache.Select(x => x.Model)) { var doc = LuceneUtility.InitializeIndexDocumentForNodes(); - AddNodeTypeToSearchIndex(node, doc); + LuceneUtility.AddNodeTypeToSearchIndex(node, doc); } //Write the Lucene documents to memory @@ -722,24 +719,6 @@ internal void SearchAutoCompleteCandidates(string input) } } - /// - /// Add node information to Lucene index - /// - /// node info that will be indexed - /// Lucene document in which the node info will be indexed - private void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc) - { - if (LuceneUtility.addedFields == null) return; - - LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), node.FullCategoryName); - LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Name), node.Name); - LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Description), node.Description); - if (node.SearchKeywords.Count > 0) LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.SearchKeywords), node.SearchKeywords.Aggregate((x, y) => x + " " + y), true, true); - LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Parameters), node.Parameters ?? string.Empty); - - LuceneUtility.writer?.AddDocument(doc); - } - /// /// Returns a collection of node search elements for nodes /// that output a type compatible with the port type if it's an input port. diff --git a/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs index dac21d0b0c7..e6d48eaa279 100644 --- a/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs @@ -948,7 +948,10 @@ internal IEnumerable Search(string search, bool useL if (LuceneUtility != null) { var searchElements = Model.Search(search, LuceneUtility); - return searchElements.Select(MakeNodeSearchElementVM); + if (searchElements != null) + { + return searchElements.Select(MakeNodeSearchElementVM); + } } return null; } diff --git a/test/DynamoCoreTests/SearchModelTests.cs b/test/DynamoCoreTests/SearchModelTests.cs index ff1d2024795..70a5b05db25 100644 --- a/test/DynamoCoreTests/SearchModelTests.cs +++ b/test/DynamoCoreTests/SearchModelTests.cs @@ -565,7 +565,7 @@ private void AddNodeElementToSearchIndex(NodeSearchElement element) var iDoc = CurrentDynamoModel.LuceneUtility.InitializeIndexDocumentForNodes(); if (element != null) { - CurrentDynamoModel.AddNodeTypeToSearchIndex(element, iDoc); + CurrentDynamoModel.LuceneUtility.AddNodeTypeToSearchIndex(element, iDoc); } } diff --git a/test/DynamoCoreWpfTests/SearchViewModelTests.cs b/test/DynamoCoreWpfTests/SearchViewModelTests.cs index 6c9a0660ebf..7c8fb997029 100644 --- a/test/DynamoCoreWpfTests/SearchViewModelTests.cs +++ b/test/DynamoCoreWpfTests/SearchViewModelTests.cs @@ -805,7 +805,7 @@ private NodeSearchElement CreateCustomNode(string name, string category, var iDoc = ViewModel.Model.LuceneUtility.InitializeIndexDocumentForNodes(); if (element != null) { - ViewModel.Model.AddNodeTypeToSearchIndex(element, iDoc); + ViewModel.Model.LuceneUtility.AddNodeTypeToSearchIndex(element, iDoc); } From b55fe15201465d02868f8e4962fbd87197b55922 Mon Sep 17 00:00:00 2001 From: "Aaron (Qilong)" <173288704@qq.com> Date: Mon, 4 Mar 2024 14:41:44 -0500 Subject: [PATCH 2/5] Update --- src/DynamoCore/Models/DynamoModel.cs | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/src/DynamoCore/Models/DynamoModel.cs b/src/DynamoCore/Models/DynamoModel.cs index 1af53762317..d79a016e48b 100644 --- a/src/DynamoCore/Models/DynamoModel.cs +++ b/src/DynamoCore/Models/DynamoModel.cs @@ -1579,28 +1579,6 @@ private void InitializeIncludedNodes() SearchModel?.Add(outputSearchElement); LuceneUtility.AddNodeTypeToSearchIndex(outputSearchElement, iDoc); - - //Adding this nodes are breaking the tests (due that we have two input and two output nodes): - //WhenHomeWorkspaceIsFocusedInputAndOutputNodesAreMissingFromSearch - //WhenStartingDynamoInputAndOutputNodesAreNolongerMissingFromSearch - // New index process from Lucene, adding missing nodes: Code Block, Input and Output - var ele = AddNodeTypeToSearch(outputData); - if (ele != null) - { - AddNodeTypeToSearchIndex(ele, iDoc); - } - - ele = AddNodeTypeToSearch(symbolData); - if (ele != null) - { - AddNodeTypeToSearchIndex(ele, iDoc); - } - - ele = AddNodeTypeToSearch(cbnData); - if (ele != null) - { - AddNodeTypeToSearchIndex(ele, iDoc); - } } internal static bool IsDisabledPath(string packagesDirectory, IPreferences preferences) From 10b7a64e4c90564c70f71215d934545063c9208c Mon Sep 17 00:00:00 2001 From: "Aaron (Qilong)" <173288704@qq.com> Date: Fri, 27 Oct 2023 14:27:42 -0400 Subject: [PATCH 3/5] Lucene index amend improvements (#14513) * Use index writer to amend index after Dynamo Launch * Update sorting * Update * Update * Update * Make sure package loading end will already release index lock * Update Comments * Clean Up * Code clean up * clean up * Code Clean Up * regressions * revert code clean up because it affects running tests in parallel * update --- .../AssemblySharedInfo.cs | 4 +- src/DynamoCore/Search/NodeSearchModel.cs | 9 +---- .../Utilities/LuceneSearchUtility.cs | 40 +++++++++++-------- src/DynamoCoreWpf/Controls/StartPage.xaml.cs | 3 +- .../ViewModels/Core/NodeViewModel.cs | 1 - .../PackageManagerClientViewModel.cs | 2 + .../PackageManagerSearchViewModel.cs | 11 +---- .../PackageManager/PackagePathViewModel.cs | 4 +- src/DynamoUtilities/PathHelper.cs | 3 ++ .../PythonMigrationViewExtension.cs | 19 +++++---- 10 files changed, 49 insertions(+), 47 deletions(-) diff --git a/src/AssemblySharedInfoGenerator/AssemblySharedInfo.cs b/src/AssemblySharedInfoGenerator/AssemblySharedInfo.cs index dade00de0dd..38d83701bc2 100644 --- a/src/AssemblySharedInfoGenerator/AssemblySharedInfo.cs +++ b/src/AssemblySharedInfoGenerator/AssemblySharedInfo.cs @@ -45,7 +45,7 @@ // to distinguish one build from another. AssemblyFileVersion is specified // in AssemblyVersionInfo.cs so that it can be easily incremented by the // automated build process. -[assembly: AssemblyVersion("2.19.5.7897")] +[assembly: AssemblyVersion("2.19.5.7936")] // By default, the "Product version" shown in the file properties window is @@ -64,4 +64,4 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyFileVersion("2.19.5.7897")] +[assembly: AssemblyFileVersion("2.19.5.7936")] diff --git a/src/DynamoCore/Search/NodeSearchModel.cs b/src/DynamoCore/Search/NodeSearchModel.cs index e828d86b509..3917c65449a 100644 --- a/src/DynamoCore/Search/NodeSearchModel.cs +++ b/src/DynamoCore/Search/NodeSearchModel.cs @@ -247,14 +247,7 @@ internal IEnumerable Search(string search, LuceneSearchUtilit if (luceneSearchUtility != null) { //The DirectoryReader and IndexSearcher have to be assigned after commiting indexing changes and before executing the Searcher.Search() method, otherwise new indexed info won't be reflected - if (luceneSearchUtility.writer != null) - { - luceneSearchUtility.dirReader = luceneSearchUtility.writer.GetReader(applyAllDeletes: true); - } - else - { - luceneSearchUtility.dirReader = DirectoryReader.Open(luceneSearchUtility.indexDir); - } + luceneSearchUtility.dirReader = luceneSearchUtility.writer != null ? luceneSearchUtility.writer.GetReader(applyAllDeletes: true) : DirectoryReader.Open(luceneSearchUtility.indexDir); luceneSearchUtility.Searcher = new IndexSearcher(luceneSearchUtility.dirReader); string searchTerm = search.Trim(); diff --git a/src/DynamoCore/Utilities/LuceneSearchUtility.cs b/src/DynamoCore/Utilities/LuceneSearchUtility.cs index 72bebe83a61..488be6520fd 100644 --- a/src/DynamoCore/Utilities/LuceneSearchUtility.cs +++ b/src/DynamoCore/Utilities/LuceneSearchUtility.cs @@ -3,12 +3,8 @@ using System.IO; using System.Linq; using Dynamo.Configuration; -using Dynamo.Core; -using Dynamo.Events; -using Dynamo.Logging; using Dynamo.Models; using Dynamo.Search.SearchElements; -using Dynamo.Session; using Lucene.Net.Analysis; using Lucene.Net.Analysis.Br; using Lucene.Net.Analysis.Cjk; @@ -95,6 +91,7 @@ public enum LuceneStorage /// /// Constructor for LuceneSearchUtility, it will use the storage type passed as parameter /// + /// /// internal LuceneSearchUtility(DynamoModel model, LuceneStartConfig config) { @@ -134,12 +131,13 @@ internal void InitializeLuceneConfig() /// /// Create index writer for followup doc indexing /// - internal void CreateLuceneIndexWriter() + /// Index open mode for Lucene index writer + internal void CreateLuceneIndexWriter(OpenMode mode = OpenMode.CREATE) { // Create an index writer IndexWriterConfig indexConfig = new IndexWriterConfig(LuceneConfig.LuceneNetVersion, Analyzer) { - OpenMode = OpenMode.CREATE + OpenMode = mode }; try { @@ -147,19 +145,14 @@ internal void CreateLuceneIndexWriter() } catch (LockObtainFailedException ex) { - try - { - writer = new IndexWriter(new RAMDirectory(), indexConfig); - (ExecutionEvents.ActiveSession.GetParameterValue(ParameterKeys.Logger) as DynamoLogger).LogError($"LuceneNET LockObtainFailedException {ex}, switching to RAM mode."); - } - catch(Exception) - { - DisposeWriter(); - } + + DisposeWriter(); + dynamoModel.Logger.LogError($"LuceneNET LockObtainFailedException {ex}"); + } catch (Exception ex) { - (ExecutionEvents.ActiveSession.GetParameterValue(ParameterKeys.Logger) as DynamoLogger).LogError($"LuceneNET Exception {ex}"); + dynamoModel.Logger.LogError($"LuceneNET Exception {ex}"); } } @@ -392,6 +385,9 @@ internal Analyzer CreateAnalyzerByLanguage(string language) } } + /// + /// Dispose Lucene index write objects and reuse other objects + /// internal void DisposeWriter() { writer?.Dispose(); @@ -403,7 +399,7 @@ internal void DisposeWriter() /// internal void DisposeAll() { - writer?.Dispose(); + DisposeWriter(); dirReader?.Dispose(); indexDir?.Dispose(); Analyzer?.Dispose(); @@ -426,6 +422,16 @@ internal void CommitWriterChanges() internal void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc) { if (addedFields == null) return; + // During DynamoModel initialization, the index writer should still be valid here + // If the index writer is null and index not locked, it means the index writer has been disposed, e.g. DynamoModel finished initialization + // If the index writer is null and index locked, it means another Dynamo session is currently updating the search index + // Try to create a new index writer to amend the index + if (writer == null && !IndexWriter.IsLocked(this.indexDir)) + { + CreateLuceneIndexWriter(OpenMode.CREATE_OR_APPEND); + } + // If the index writer is still null, skip the indexing + if (writer == null) return; SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), node.FullCategoryName); SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Name), node.Name); diff --git a/src/DynamoCoreWpf/Controls/StartPage.xaml.cs b/src/DynamoCoreWpf/Controls/StartPage.xaml.cs index b33d05dcc47..302e6a02192 100644 --- a/src/DynamoCoreWpf/Controls/StartPage.xaml.cs +++ b/src/DynamoCoreWpf/Controls/StartPage.xaml.cs @@ -4,6 +4,7 @@ using System.Collections.Specialized; using System.Diagnostics; using System.IO; +using System.Linq; using System.Windows; using System.Windows.Controls; using System.Windows.Input; @@ -370,7 +371,7 @@ private void RefreshFileList(ObservableCollection files, IEnumerable filePaths) { files.Clear(); - foreach (var filePath in filePaths) + foreach (var filePath in filePaths.Where(x => x != null)) { try { diff --git a/src/DynamoCoreWpf/ViewModels/Core/NodeViewModel.cs b/src/DynamoCoreWpf/ViewModels/Core/NodeViewModel.cs index a043c8f6943..2bf440aae45 100644 --- a/src/DynamoCoreWpf/ViewModels/Core/NodeViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/Core/NodeViewModel.cs @@ -18,7 +18,6 @@ using Dynamo.Logging; using Dynamo.Models; using Dynamo.Selection; -using Dynamo.UI; using Dynamo.Wpf.ViewModels.Core; using Newtonsoft.Json; using Point = System.Windows.Point; diff --git a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerClientViewModel.cs b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerClientViewModel.cs index 0bdae89f0dd..0676543fd3f 100644 --- a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerClientViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerClientViewModel.cs @@ -1014,6 +1014,8 @@ internal virtual void InstallPackage(PackageDownloadHandle packageDownloadHandle } } SetPackageState(packageDownloadHandle, installPath); + // Dispose Index writer to avoid file lock after new package is installed + Search.LuceneSearch.LuceneUtilityNodeSearch.DisposeWriter(); Analytics.TrackEvent(Actions.Installed, Categories.PackageManagerOperations, $"{packageDownloadHandle?.Name}"); } catch (Exception e) diff --git a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs index 72e6dde5c70..3325e542694 100644 --- a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs @@ -796,14 +796,7 @@ public void RefreshAndSearchAsync() if (!DynamoModel.IsTestMode) { - if (LuceneUtility.writer != null) - { - LuceneUtility.dirReader = LuceneUtility.writer.GetReader(applyAllDeletes: true); - } - else - { - LuceneUtility.dirReader = DirectoryReader.Open(LuceneUtility.indexDir); - } + LuceneUtility.dirReader = LuceneUtility.writer != null ? LuceneUtility.writer.GetReader(applyAllDeletes: true) : DirectoryReader.Open(LuceneUtility.indexDir); LuceneUtility.Searcher = new IndexSearcher(LuceneUtility.dirReader); LuceneUtility.CommitWriterChanges(); @@ -1140,7 +1133,7 @@ internal IEnumerable Search(string searchT var packages = new List(); //The DirectoryReader and IndexSearcher have to be assigned after commiting indexing changes and before executing the Searcher.Search() method,otherwise new indexed info won't be reflected - LuceneUtility.dirReader = LuceneUtility.writer?.GetReader(applyAllDeletes: true); + LuceneUtility.dirReader = LuceneUtility.writer != null ? LuceneUtility.writer.GetReader(applyAllDeletes: true): DirectoryReader.Open(LuceneUtility.indexDir); if (LuceneUtility.Searcher == null && LuceneUtility.dirReader != null) { diff --git a/src/DynamoCoreWpf/ViewModels/PackageManager/PackagePathViewModel.cs b/src/DynamoCoreWpf/ViewModels/PackageManager/PackagePathViewModel.cs index adbc2ff7d43..4c0bcfb4c3b 100644 --- a/src/DynamoCoreWpf/ViewModels/PackageManager/PackagePathViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/PackageManager/PackagePathViewModel.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Collections.ObjectModel; using Dynamo.Core; @@ -266,6 +266,8 @@ private void CommitChanges(object param) packageLoader.LoadNewCustomNodesAndPackages(newPaths, customNodeManager); } packagePathsEnabled.Clear(); + // Dispose node Index writer to avoid file lock after new package path applied and packages loaded. + Search.LuceneSearch.LuceneUtilityNodeSearch.DisposeWriter(); } internal void SetPackagesScheduledState(string packagePath, bool packagePathDisabled) diff --git a/src/DynamoUtilities/PathHelper.cs b/src/DynamoUtilities/PathHelper.cs index 7a703814e71..dcc8a44548b 100644 --- a/src/DynamoUtilities/PathHelper.cs +++ b/src/DynamoUtilities/PathHelper.cs @@ -37,6 +37,9 @@ public static Exception CreateFolderIfNotExist(string folderPath) { try { + // Do not even try when folder path is null or empty. + // This usually happens when system folder dialog is initialized with empty path + if (string.IsNullOrEmpty(folderPath)) return null; // When network path is access denied, the Directory.Exits however still // return true. // EnumerateDirectories operation is additional check diff --git a/src/PythonMigrationViewExtension/PythonMigrationViewExtension.cs b/src/PythonMigrationViewExtension/PythonMigrationViewExtension.cs index 8a566f3c793..1c28c53381c 100644 --- a/src/PythonMigrationViewExtension/PythonMigrationViewExtension.cs +++ b/src/PythonMigrationViewExtension/PythonMigrationViewExtension.cs @@ -187,14 +187,17 @@ private void UnSubscribePythonNodeEvents(PythonNodeBase node) private void UnSubscribeWorkspaceEvents() { - CurrentWorkspace.RequestPackageDependencies -= PythonDependencies.AddPythonPackageDependency; - CurrentWorkspace.NodeAdded -= OnNodeAdded; - CurrentWorkspace.NodeRemoved -= OnNodeRemoved; - CurrentWorkspace.Nodes - .Where(n => n is PythonNode) - .Cast() - .ToList() - .ForEach(n => UnSubscribePythonNodeEvents(n)); + if (CurrentWorkspace != null) + { + CurrentWorkspace.RequestPackageDependencies -= PythonDependencies.AddPythonPackageDependency; + CurrentWorkspace.NodeAdded -= OnNodeAdded; + CurrentWorkspace.NodeRemoved -= OnNodeRemoved; + CurrentWorkspace.Nodes + .Where(n => n is PythonNode) + .Cast() + .ToList() + .ForEach(n => UnSubscribePythonNodeEvents(n)); + } } private void UnsubscribeEvents() From f751371acf1474364819894d78e9ba0360d196bd Mon Sep 17 00:00:00 2001 From: "Aaron (Qilong)" <173288704@qq.com> Date: Tue, 5 Mar 2024 11:03:18 -0500 Subject: [PATCH 4/5] update --- src/AssemblySharedInfoGenerator/AssemblySharedInfo.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/AssemblySharedInfoGenerator/AssemblySharedInfo.cs b/src/AssemblySharedInfoGenerator/AssemblySharedInfo.cs index 38d83701bc2..684f246c0a3 100644 --- a/src/AssemblySharedInfoGenerator/AssemblySharedInfo.cs +++ b/src/AssemblySharedInfoGenerator/AssemblySharedInfo.cs @@ -8,7 +8,7 @@ // associated with an assembly. [assembly: AssemblyCompany("Autodesk, Inc")] [assembly: AssemblyProduct("Dynamo")] -[assembly: AssemblyCopyright("Copyright © Autodesk, Inc 2023")] +[assembly: AssemblyCopyright("Copyright � Autodesk, Inc 2023")] [assembly: AssemblyTrademark("")] //In order to begin building localizable applications, set @@ -45,7 +45,7 @@ // to distinguish one build from another. AssemblyFileVersion is specified // in AssemblyVersionInfo.cs so that it can be easily incremented by the // automated build process. -[assembly: AssemblyVersion("2.19.5.7936")] +[assembly: AssemblyVersion("2.19.5.7897")] // By default, the "Product version" shown in the file properties window is @@ -64,4 +64,4 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyFileVersion("2.19.5.7936")] +[assembly: AssemblyFileVersion("2.19.5.7897")] From 664a3fdb792d7570759c18fc79aea5340ab5114b Mon Sep 17 00:00:00 2001 From: "Aaron (Qilong)" <173288704@qq.com> Date: Tue, 5 Mar 2024 11:06:16 -0500 Subject: [PATCH 5/5] Update AssemblySharedInfo.cs --- src/AssemblySharedInfoGenerator/AssemblySharedInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AssemblySharedInfoGenerator/AssemblySharedInfo.cs b/src/AssemblySharedInfoGenerator/AssemblySharedInfo.cs index 684f246c0a3..1f5effdf758 100644 --- a/src/AssemblySharedInfoGenerator/AssemblySharedInfo.cs +++ b/src/AssemblySharedInfoGenerator/AssemblySharedInfo.cs @@ -8,7 +8,7 @@ // associated with an assembly. [assembly: AssemblyCompany("Autodesk, Inc")] [assembly: AssemblyProduct("Dynamo")] -[assembly: AssemblyCopyright("Copyright � Autodesk, Inc 2023")] +[assembly: AssemblyCopyright("Copyright © Autodesk, Inc 2023")] [assembly: AssemblyTrademark("")] //In order to begin building localizable applications, set