Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[CherryPick] Lucene search IMP #14997

Merged
merged 5 commits into from
Mar 5, 2024
Merged

[CherryPick] Lucene search IMP #14997

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4c452f2
DYN-6038 Lucene search IMP (#14428)
QilongTang Oct 12, 2023
b55fe15
Update
QilongTang Mar 4, 2024
10b7a64
Lucene index amend improvements (#14513)
QilongTang Oct 27, 2023
f751371
update
QilongTang Mar 5, 2024
664a3fd
Update AssemblySharedInfo.cs
QilongTang Mar 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/AssemblySharedInfoGenerator/AssemblySharedInfo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
// to distinguish one build from another. AssemblyFileVersion is specified
// in AssemblyVersionInfo.cs so that it can be easily incremented by the
// automated build process.
[assembly: AssemblyVersion("2.19.5.7897")]
[assembly: AssemblyVersion("2.19.5.7936")]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove this file commit.



// By default, the "Product version" shown in the file properties window is
Expand All @@ -64,4 +64,4 @@
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyFileVersion("2.19.5.7897")]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it's ok to modify this file?

[assembly: AssemblyFileVersion("2.19.5.7936")]
79 changes: 18 additions & 61 deletions src/DynamoCore/Models/DynamoModel.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -925,16 +925,7 @@ protected DynamoModel(IStartConfiguration config)

CustomNodeManager = new CustomNodeManager(NodeFactory, MigrationManager, LibraryServices);

LuceneSearch.LuceneUtilityNodeSearch = new LuceneSearchUtility(this);

if (IsTestMode)
{
LuceneUtility.InitializeLuceneConfig(string.Empty, LuceneSearchUtility.LuceneStorage.RAM);
}
else
{
LuceneUtility.InitializeLuceneConfig(LuceneConfig.NodesIndexingDirectory);
}
LuceneSearch.LuceneUtilityNodeSearch = new LuceneSearchUtility(this, LuceneSearchUtility.DefaultNodeIndexStartConfig);

InitializeCustomNodeManager();

Expand Down Expand Up @@ -1017,10 +1008,15 @@ protected DynamoModel(IStartConfiguration config)
TraceReconciliationProcessor = this;

State = DynamoModelState.StartedUIless;
// This event should only be raised at the end of this method.
DynamoReady(new ReadyParams(this));
// Write index to disk only once
LuceneUtility.CommitWriterChanges();
//Disposed writer if it is in file system mode so that the index files can be used by other processes (potentially a second Dynamo session)
if (LuceneUtility.startConfig.StorageType == LuceneSearchUtility.LuceneStorage.FILE_SYSTEM)
{
LuceneUtility.DisposeWriter();
}
// This event should only be raised at the end of this method.
DynamoReady(new ReadyParams(this));
}

/// <summary>
Expand Down Expand Up @@ -1429,12 +1425,8 @@ public void Dispose()
PreferenceSettings.MessageLogged -= LogMessage;
}

//The writer have to be disposed at DynamoModel level due that we could index package-nodes as new packages are installed
LuceneUtility.DisposeWriter();

// Lucene disposals (just if LuceneNET was initialized)
LuceneUtility.indexDir?.Dispose();
LuceneUtility.dirReader?.Dispose();
LuceneUtility.DisposeAll();

#if DEBUG
CurrentWorkspace.NodeAdded -= CrashOnDemand.CurrentWorkspace_NodeAdded;
Expand Down Expand Up @@ -1495,7 +1487,7 @@ private void InitializeCustomNodeManager()
var iDoc = LuceneUtility.InitializeIndexDocumentForNodes();
if (searchElement != null)
{
AddNodeTypeToSearchIndex(searchElement, iDoc);
LuceneUtility.AddNodeTypeToSearchIndex(searchElement, iDoc);
}

Action<CustomNodeInfo> infoUpdatedHandler = null;
Expand Down Expand Up @@ -1562,7 +1554,9 @@ private void InitializeIncludedNodes()
NodeFactory.AddTypeFactoryAndLoader(outputData.Type);
NodeFactory.AddAlsoKnownAs(outputData.Type, outputData.AlsoKnownAs);

SearchModel?.Add(new CodeBlockNodeSearchElement(cbnData, LibraryServices));
var cnbNode = new CodeBlockNodeSearchElement(cbnData, LibraryServices);
SearchModel?.Add(cnbNode);
LuceneUtility.AddNodeTypeToSearchIndex(cnbNode, iDoc);

var symbolSearchElement = new NodeModelSearchElement(symbolData)
{
Expand All @@ -1581,29 +1575,10 @@ private void InitializeIncludedNodes()
};

SearchModel?.Add(symbolSearchElement);
SearchModel?.Add(outputSearchElement);
LuceneUtility.AddNodeTypeToSearchIndex(symbolSearchElement, iDoc);

//Adding this nodes are breaking the tests (due that we have two input and two output nodes):
//WhenHomeWorkspaceIsFocusedInputAndOutputNodesAreMissingFromSearch
//WhenStartingDynamoInputAndOutputNodesAreNolongerMissingFromSearch
// New index process from Lucene, adding missing nodes: Code Block, Input and Output
var ele = AddNodeTypeToSearch(outputData);
if (ele != null)
{
AddNodeTypeToSearchIndex(ele, iDoc);
}

ele = AddNodeTypeToSearch(symbolData);
if (ele != null)
{
AddNodeTypeToSearchIndex(ele, iDoc);
}

ele = AddNodeTypeToSearch(cbnData);
if (ele != null)
{
AddNodeTypeToSearchIndex(ele, iDoc);
}
SearchModel?.Add(outputSearchElement);
LuceneUtility.AddNodeTypeToSearchIndex(outputSearchElement, iDoc);
}

internal static bool IsDisabledPath(string packagesDirectory, IPreferences preferences)
Expand Down Expand Up @@ -1756,7 +1731,7 @@ private void LoadNodeModels(List<TypeLoadData> nodes, bool isPackageMember)
// TODO: get search element some other way
if (ele != null)
{
AddNodeTypeToSearchIndex(ele, iDoc);
LuceneUtility.AddNodeTypeToSearchIndex(ele, iDoc);
}
}
catch (Exception e)
Expand Down Expand Up @@ -3286,24 +3261,6 @@ private NodeModelSearchElement AddNodeTypeToSearch(TypeLoadData typeLoadData)
return node;
}

/// <summary>
/// Add node information to Lucene index
/// </summary>
/// <param name="node">node info that will be indexed</param>
/// <param name="doc">Lucene document in which the node info will be indexed</param>
internal void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc)
{
if (LuceneUtility.addedFields == null) return;

LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), node.FullCategoryName);
LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Name), node.Name);
LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Description), node.Description);
if (node.SearchKeywords.Count > 0) LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.SearchKeywords), node.SearchKeywords.Aggregate((x, y) => x + " " + y), true, true);
LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Parameters), node.Parameters ?? string.Empty);

LuceneUtility.writer?.AddDocument(doc);
}

/// <summary>
/// Remove node information from Lucene indexing.
/// </summary>
Expand Down Expand Up @@ -3363,7 +3320,7 @@ private void AddZeroTouchNodeToSearch(FunctionDescriptor functionDescriptor, Doc
{
var ele = new ZeroTouchSearchElement(functionDescriptor);
SearchModel?.Add(ele);
AddNodeTypeToSearchIndex(ele, iDoc);
LuceneUtility.AddNodeTypeToSearchIndex(ele, iDoc);
}
}

Expand Down
5 changes: 2 additions & 3 deletions src/DynamoCore/Search/NodeSearchModel.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
using Lucene.Net.Analysis.Ru;
using Lucene.Net.Analysis.Standard;
using Lucene.Net.Documents;
using Lucene.Net.Index;
using Lucene.Net.QueryParsers.Classic;
using Lucene.Net.Search;

Expand Down Expand Up @@ -246,9 +247,7 @@ internal IEnumerable<NodeSearchElement> Search(string search, LuceneSearchUtilit
if (luceneSearchUtility != null)
{
//The DirectoryReader and IndexSearcher have to be assigned after commiting indexing changes and before executing the Searcher.Search() method, otherwise new indexed info won't be reflected
luceneSearchUtility.dirReader = luceneSearchUtility.writer?.GetReader(applyAllDeletes: true);
if (luceneSearchUtility.dirReader == null) return null;

luceneSearchUtility.dirReader = luceneSearchUtility.writer != null ? luceneSearchUtility.writer.GetReader(applyAllDeletes: true) : DirectoryReader.Open(luceneSearchUtility.indexDir);
luceneSearchUtility.Searcher = new IndexSearcher(luceneSearchUtility.dirReader);

string searchTerm = search.Trim();
Expand Down
Loading
Loading