Skip to content

Commit

Permalink
[CherryPick] Lucene search IMP (#14997)
Browse files Browse the repository at this point in the history 
* DYN-6038 Lucene search IMP (#14428)

* improvements

* Simply initialization logic

* Skip indexing as part of DynamoModel if the index files already exist

* Remove DynamoModel reference and use singleton

* clean up

* improvements and package manager search

* Use single constructor and other code clean up

* Update...

* Update

* Dispose all Lucene objects in the correct order

* Adding DynamoModel back to test regressions

* Null check

* Update

* IndexWriter dispose sequence only for non-RAM mode

* Update

* Lucene index amend improvements (#14513)

* Use index writer to amend index after Dynamo Launch

* Update sorting

* Update

* Update

* Update

* Make sure package loading end will already release index lock

* Update Comments

* Clean Up

* Code clean up

* clean up

* Code Clean Up

* regressions

* revert code clean up because it affects running tests in parallel

* update

* update

* Update AssemblySharedInfo.cs
  • Loading branch information
@QilongTang
QilongTang authored Mar 5, 2024
1 parent a1cb855 commit 29cab79
Show file tree
Hide file tree
Showing 16 changed files with 220 additions and 153 deletions.
2 changes: 1 addition & 1 deletion src/AssemblySharedInfoGenerator/AssemblySharedInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
// associated with an assembly.
[assembly: AssemblyCompany("Autodesk, Inc")]
[assembly: AssemblyProduct("Dynamo")]
[assembly: AssemblyCopyright("Copyright © Autodesk, Inc 2023")]
[assembly: AssemblyCopyright("Copyright © Autodesk, Inc 2023")]
[assembly: AssemblyTrademark("")]

//In order to begin building localizable applications, set
Expand Down
79 changes: 18 additions & 61 deletions src/DynamoCore/Models/DynamoModel.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -925,16 +925,7 @@ protected DynamoModel(IStartConfiguration config)

CustomNodeManager = new CustomNodeManager(NodeFactory, MigrationManager, LibraryServices);

LuceneSearch.LuceneUtilityNodeSearch = new LuceneSearchUtility(this);

if (IsTestMode)
{
LuceneUtility.InitializeLuceneConfig(string.Empty, LuceneSearchUtility.LuceneStorage.RAM);
}
else
{
LuceneUtility.InitializeLuceneConfig(LuceneConfig.NodesIndexingDirectory);
}
LuceneSearch.LuceneUtilityNodeSearch = new LuceneSearchUtility(this, LuceneSearchUtility.DefaultNodeIndexStartConfig);

InitializeCustomNodeManager();

Expand Down Expand Up @@ -1017,10 +1008,15 @@ protected DynamoModel(IStartConfiguration config)
TraceReconciliationProcessor = this;

State = DynamoModelState.StartedUIless;
// This event should only be raised at the end of this method.
DynamoReady(new ReadyParams(this));
// Write index to disk only once
LuceneUtility.CommitWriterChanges();
//Disposed writer if it is in file system mode so that the index files can be used by other processes (potentially a second Dynamo session)
if (LuceneUtility.startConfig.StorageType == LuceneSearchUtility.LuceneStorage.FILE_SYSTEM)
{
LuceneUtility.DisposeWriter();
}
// This event should only be raised at the end of this method.
DynamoReady(new ReadyParams(this));
}

/// <summary>
Expand Down Expand Up @@ -1429,12 +1425,8 @@ public void Dispose()
PreferenceSettings.MessageLogged -= LogMessage;
}

//The writer have to be disposed at DynamoModel level due that we could index package-nodes as new packages are installed
LuceneUtility.DisposeWriter();

// Lucene disposals (just if LuceneNET was initialized)
LuceneUtility.indexDir?.Dispose();
LuceneUtility.dirReader?.Dispose();
LuceneUtility.DisposeAll();

#if DEBUG
CurrentWorkspace.NodeAdded -= CrashOnDemand.CurrentWorkspace_NodeAdded;
Expand Down Expand Up @@ -1495,7 +1487,7 @@ private void InitializeCustomNodeManager()
var iDoc = LuceneUtility.InitializeIndexDocumentForNodes();
if (searchElement != null)
{
AddNodeTypeToSearchIndex(searchElement, iDoc);
LuceneUtility.AddNodeTypeToSearchIndex(searchElement, iDoc);
}

Action<CustomNodeInfo> infoUpdatedHandler = null;
Expand Down Expand Up @@ -1562,7 +1554,9 @@ private void InitializeIncludedNodes()
NodeFactory.AddTypeFactoryAndLoader(outputData.Type);
NodeFactory.AddAlsoKnownAs(outputData.Type, outputData.AlsoKnownAs);

SearchModel?.Add(new CodeBlockNodeSearchElement(cbnData, LibraryServices));
var cnbNode = new CodeBlockNodeSearchElement(cbnData, LibraryServices);
SearchModel?.Add(cnbNode);
LuceneUtility.AddNodeTypeToSearchIndex(cnbNode, iDoc);

var symbolSearchElement = new NodeModelSearchElement(symbolData)
{
Expand All @@ -1581,29 +1575,10 @@ private void InitializeIncludedNodes()
};

SearchModel?.Add(symbolSearchElement);
SearchModel?.Add(outputSearchElement);
LuceneUtility.AddNodeTypeToSearchIndex(symbolSearchElement, iDoc);

//Adding this nodes are breaking the tests (due that we have two input and two output nodes):
//WhenHomeWorkspaceIsFocusedInputAndOutputNodesAreMissingFromSearch
//WhenStartingDynamoInputAndOutputNodesAreNolongerMissingFromSearch
// New index process from Lucene, adding missing nodes: Code Block, Input and Output
var ele = AddNodeTypeToSearch(outputData);
if (ele != null)
{
AddNodeTypeToSearchIndex(ele, iDoc);
}

ele = AddNodeTypeToSearch(symbolData);
if (ele != null)
{
AddNodeTypeToSearchIndex(ele, iDoc);
}

ele = AddNodeTypeToSearch(cbnData);
if (ele != null)
{
AddNodeTypeToSearchIndex(ele, iDoc);
}
SearchModel?.Add(outputSearchElement);
LuceneUtility.AddNodeTypeToSearchIndex(outputSearchElement, iDoc);
}

internal static bool IsDisabledPath(string packagesDirectory, IPreferences preferences)
Expand Down Expand Up @@ -1756,7 +1731,7 @@ private void LoadNodeModels(List<TypeLoadData> nodes, bool isPackageMember)
// TODO: get search element some other way
if (ele != null)
{
AddNodeTypeToSearchIndex(ele, iDoc);
LuceneUtility.AddNodeTypeToSearchIndex(ele, iDoc);
}
}
catch (Exception e)
Expand Down Expand Up @@ -3286,24 +3261,6 @@ private NodeModelSearchElement AddNodeTypeToSearch(TypeLoadData typeLoadData)
return node;
}

/// <summary>
/// Add node information to Lucene index
/// </summary>
/// <param name="node">node info that will be indexed</param>
/// <param name="doc">Lucene document in which the node info will be indexed</param>
internal void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc)
{
if (LuceneUtility.addedFields == null) return;

LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), node.FullCategoryName);
LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Name), node.Name);
LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Description), node.Description);
if (node.SearchKeywords.Count > 0) LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.SearchKeywords), node.SearchKeywords.Aggregate((x, y) => x + " " + y), true, true);
LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Parameters), node.Parameters ?? string.Empty);

LuceneUtility.writer?.AddDocument(doc);
}

/// <summary>
/// Remove node information from Lucene indexing.
/// </summary>
Expand Down Expand Up @@ -3363,7 +3320,7 @@ private void AddZeroTouchNodeToSearch(FunctionDescriptor functionDescriptor, Doc
{
var ele = new ZeroTouchSearchElement(functionDescriptor);
SearchModel?.Add(ele);
AddNodeTypeToSearchIndex(ele, iDoc);
LuceneUtility.AddNodeTypeToSearchIndex(ele, iDoc);
}
}

Expand Down
5 changes: 2 additions & 3 deletions src/DynamoCore/Search/NodeSearchModel.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
using Lucene.Net.Analysis.Ru;
using Lucene.Net.Analysis.Standard;
using Lucene.Net.Documents;
using Lucene.Net.Index;
using Lucene.Net.QueryParsers.Classic;
using Lucene.Net.Search;

Expand Down Expand Up @@ -246,9 +247,7 @@ internal IEnumerable<NodeSearchElement> Search(string search, LuceneSearchUtilit
if (luceneSearchUtility != null)
{
//The DirectoryReader and IndexSearcher have to be assigned after commiting indexing changes and before executing the Searcher.Search() method, otherwise new indexed info won't be reflected
luceneSearchUtility.dirReader = luceneSearchUtility.writer?.GetReader(applyAllDeletes: true);
if (luceneSearchUtility.dirReader == null) return null;

luceneSearchUtility.dirReader = luceneSearchUtility.writer != null ? luceneSearchUtility.writer.GetReader(applyAllDeletes: true) : DirectoryReader.Open(luceneSearchUtility.indexDir);
luceneSearchUtility.Searcher = new IndexSearcher(luceneSearchUtility.dirReader);

string searchTerm = search.Trim();
Expand Down
Loading

0 comments on commit 29cab79

Please sign in to comment.