public IEnumerable <(string FilePath, DateTime LastWriteTimeUtc)> GetAllIndexedCodeSource() { return(CodeIndexPool.SearchWithSpecificFields( new MatchAllDocsQuery(), int.MaxValue, nameof(CodeSource.LastWriteTimeUtc), nameof(CodeSource.FilePath)).Select(u => (u.Get(nameof(CodeSource.FilePath)), new DateTime(u.GetField(nameof(CodeSource.LastWriteTimeUtc)).GetInt64Value() ?? throw new ArgumentException(nameof(CodeSource.LastWriteTimeUtc)))))); }
public void DeleteAllIndex() { Log.LogInformation($"{Name}: Delete All Index start"); CodeIndexPool.DeleteAllIndex(); HintIndexPool.DeleteAllIndex(); Log.LogInformation($"{Name}: Delete All Index finished"); }
public void Dispose() { if (!IsDisposing) { IsDisposing = true; CodeIndexPool.Dispose(); HintIndexPool.Dispose(); } }
void RenameIndex(Document document, string oldFilePath, string nowFilePath) { var pathField = document.Get(nameof(CodeSource.FilePath)); var nowPath = pathField.Replace(oldFilePath, nowFilePath); document.RemoveField(nameof(CodeSource.FilePath)); document.RemoveField(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix); document.Add(new TextField(nameof(CodeSource.FilePath), nowPath, Field.Store.YES)); document.Add(new StringField(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, nowPath, Field.Store.YES)); CodeIndexPool.UpdateIndex(new Term(nameof(CodeSource.CodePK), document.Get(nameof(CodeSource.CodePK))), document); }
public bool DeleteIndex(string fileOrFolderPath) { try { var query = new BooleanQuery { { new TermQuery(GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), fileOrFolderPath)), Occur.SHOULD }, { new PrefixQuery(GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), fileOrFolderPath + Path.DirectorySeparatorChar)), Occur.SHOULD } }; CodeIndexPool.DeleteIndex(query, out var documentsBeenDeleted); if (documentsBeenDeleted.Length >= 1) { var wordsNeedToRemove = new HashSet <string>(); foreach (var document in documentsBeenDeleted) { AddHintWords(wordsNeedToRemove, GetCodeSourceFromDocument(document).Content); } Log.LogInformation($"{Name}: Find {wordsNeedToRemove.Count} Delete Candidates Words With Path {fileOrFolderPath}"); if (documentsBeenDeleted.Length > 1) { Log.LogInformation($"{Name}: Find {documentsBeenDeleted.Length} Documents With Path {fileOrFolderPath} To Delete"); } foreach (var needToDeleteWord in wordsNeedToRemove) { if (!CodeIndexPool.Exists(new TermQuery(new Term(GetCaseSensitiveField(nameof(CodeSource.Content)), needToDeleteWord)))) { HintIndexPool.DeleteIndex(new Term(nameof(CodeWord.Word), needToDeleteWord)); } } } else { Log.LogWarning($"{Name}: Find No Documents To Delete For {fileOrFolderPath}"); } Log.LogInformation($"{Name}: Delete index For {fileOrFolderPath} finished"); return(true); } catch (Exception ex) { Log.LogError($"{Name}: Delete index for {fileOrFolderPath} failed, exception: " + ex); return(false); } }
public IndexBuildResults CreateIndex(FileInfo fileInfo) { try { if (fileInfo.Exists) { var source = CodeSource.GetCodeSource(fileInfo, FilesContentHelper.ReadAllText(fileInfo.FullName)); var words = new HashSet <string>(); AddHintWords(words, source.Content); var doc = IndexBuilderHelper.GetDocumentFromSource(source); CodeIndexPool.UpdateIndex(GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), source.FilePath), doc); foreach (var word in words) { HintIndexPool.UpdateIndex(new Term(nameof(CodeWord.Word), word), new Document { new StringField(nameof(CodeWord.Word), word, Field.Store.YES), new StringField(nameof(CodeWord.WordLower), word.ToLowerInvariant(), Field.Store.YES) }); } Log.LogInformation($"{Name}: Create index For {source.FilePath} finished"); } return(IndexBuildResults.Successful); } catch (Exception ex) { Log.LogError($"{Name}: Create index for {fileInfo.FullName} failed, exception: " + ex); if (ex is IOException) { return(IndexBuildResults.FailedWithIOException); } else if (ex is OperationCanceledException) { throw; } return(IndexBuildResults.FailedWithError); } }
public bool RenameFolderIndexes(string oldFolderPath, string nowFolderPath, CancellationToken cancellationToken) { try { var documents = CodeIndexPool.Search(new PrefixQuery(GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), oldFolderPath)), int.MaxValue); foreach (var document in documents) { cancellationToken.ThrowIfCancellationRequested(); RenameIndex(document, oldFolderPath, nowFolderPath); } Log.LogInformation($"{Name}: Rename folder index from {oldFolderPath} to {nowFolderPath} successful, documents count: {documents.Length}"); return(true); } catch (Exception ex) { Log.LogError($"{Name}: Rename folder index from {oldFolderPath} to {nowFolderPath} failed, exception: " + ex); return(false); } }
public IndexBuildResults RenameFileIndex(string oldFilePath, string nowFilePath) { try { var documents = CodeIndexPool.Search(new TermQuery(GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), oldFilePath)), 1); if (documents.Length == 1) { RenameIndex(documents[0], oldFilePath, nowFilePath); Log.LogInformation($"{Name}: Rename file index from {oldFilePath} to {nowFilePath} successful"); return(IndexBuildResults.Successful); } if (documents.Length == 0) { Log.LogInformation($"{Name}: Rename file index failed, unable to find any document from {oldFilePath}, possible template file renamed, fallback to create index."); return(CreateIndex(new FileInfo(nowFilePath))); } Log.LogWarning($"{Name}: Rename file index from {oldFilePath} to {nowFilePath} failed, unable to find one document, there are {documents.Length} document(s) founded"); return(IndexBuildResults.FailedWithError); } catch (Exception ex) { Log.LogError($"{Name}: Rename file index from {oldFilePath} to {nowFilePath} failed, exception: " + ex); if (ex is IOException) { return(IndexBuildResults.FailedWithIOException); } return(IndexBuildResults.FailedWithError); } }
public void Commit() { CodeIndexPool.Commit(); HintIndexPool.Commit(); }
public IndexBuildResults UpdateIndex(FileInfo fileInfo, CancellationToken cancellationToken) { try { if (fileInfo.Exists) { cancellationToken.ThrowIfCancellationRequested(); var source = CodeSource.GetCodeSource(fileInfo, FilesContentHelper.ReadAllText(fileInfo.FullName)); var words = new HashSet <string>(); AddHintWords(words, source.Content); var doc = IndexBuilderHelper.GetDocumentFromSource(source); CodeIndexPool.UpdateIndex(GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), source.FilePath), doc, out var rawDocuments); if (rawDocuments.Length >= 1) { var rawWords = new HashSet <string>(); AddHintWords(rawWords, GetCodeSourceFromDocument(rawDocuments[0]).Content); var wordsNeedToRemove = rawWords.Except(words).ToArray(); var wordsNeedToAdd = words.Except(rawWords); words = wordsNeedToAdd.ToHashSet(); Log.LogInformation($"{Name}: Find {wordsNeedToRemove.Length} Delete Candidates Words, {words.Count} Update Candidates Words With Path {source.FilePath}"); if (rawDocuments.Length > 1) { Log.LogError($"{Name}: Find {rawDocuments.Length} Documents With Path {source.FilePath} To Update"); } foreach (var needToDeleteWord in wordsNeedToRemove) { if (!CodeIndexPool.Exists(new TermQuery(new Term(GetCaseSensitiveField(nameof(CodeSource.Content)), needToDeleteWord)))) { HintIndexPool.DeleteIndex(new Term(nameof(CodeWord.Word), needToDeleteWord)); } } } else { Log.LogError($"{Name}: Find 0 Document To Update With Path {source.FilePath}, Create New Index"); } foreach (var word in words) { HintIndexPool.UpdateIndex(new Term(nameof(CodeWord.Word), word), new Document { new StringField(nameof(CodeWord.Word), word, Field.Store.YES), new StringField(nameof(CodeWord.WordLower), word.ToLowerInvariant(), Field.Store.YES) }); } Log.LogInformation($"{Name}: Update index For {source.FilePath} finished"); } return(IndexBuildResults.Successful); } catch (Exception ex) { Log.LogError($"{Name}: Update index for {fileInfo.FullName} failed, exception: " + ex); if (ex is IOException) { return(IndexBuildResults.FailedWithIOException); } else if (ex is OperationCanceledException) { throw; } return(IndexBuildResults.FailedWithError); } }
void BuildIndex(bool needCommit, bool triggerMerge, bool applyAllDeletes, List <Document> codeDocuments, HashSet <string> newHintWords, CancellationToken cancellationToken, bool brandNewBuild) { cancellationToken.ThrowIfCancellationRequested(); Log.LogInformation($"{Name}: Build code index start, documents count {codeDocuments.Count}"); Parallel.ForEach( codeDocuments, () => new List <Document>(), (codeDocument, status, documentLists) => { documentLists.Add(codeDocument); return(documentLists); }, documentLists => { if (documentLists.Count > 0) { CodeIndexPool.BuildIndex(documentLists, needCommit, triggerMerge, applyAllDeletes); } }); Log.LogInformation($"{Name}: Build code index finished"); Log.LogInformation($"{Name}: Build {(brandNewBuild ? "brand New" : "exist")} hint index start, documents count {newHintWords.Count}"); if (brandNewBuild) { Parallel.ForEach( newHintWords, () => new List <Document>(), (word, status, documentLists) => { documentLists.Add(new Document { new StringField(nameof(CodeWord.Word), word, Field.Store.YES), new StringField(nameof(CodeWord.WordLower), word.ToLowerInvariant(), Field.Store.YES) }); return(documentLists); }, documentLists => { if (documentLists.Count > 0) { HintIndexPool.BuildIndex(documentLists, needCommit, triggerMerge, applyAllDeletes); } }); } else { Parallel.ForEach(newHintWords, word => { HintIndexPool.UpdateIndex(new Term(nameof(CodeWord.Word), word), new Document { new StringField(nameof(CodeWord.Word), word, Field.Store.YES), new StringField(nameof(CodeWord.WordLower), word.ToLowerInvariant(), Field.Store.YES) }); }); if (needCommit || triggerMerge || applyAllDeletes) { HintIndexPool.Commit(); } } Log.LogInformation($"{Name}: Build hint index finished"); }