public List <FileInfo> BuildIndexByBatch(IEnumerable <FileInfo> fileInfos, bool needCommit, bool triggerMerge, bool applyAllDeletes, CancellationToken cancellationToken, bool brandNewBuild, int batchSize = 10000) { cancellationToken.ThrowIfCancellationRequested(); fileInfos.RequireNotNull(nameof(fileInfos)); batchSize.RequireRange(nameof(batchSize), int.MaxValue, 50); var codeDocuments = new List <Document>(); var wholeWords = new HashSet <string>(); var newHintWords = new HashSet <string>(); var failedIndexFiles = new List <FileInfo>(); try { foreach (var fileInfo in fileInfos) { cancellationToken.ThrowIfCancellationRequested(); try { if (fileInfo.Exists) { var source = CodeSource.GetCodeSource(fileInfo, FilesContentHelper.ReadAllText(fileInfo.FullName)); AddHintWords(newHintWords, wholeWords, source.Content); var doc = IndexBuilderHelper.GetDocumentFromSource(source); codeDocuments.Add(doc); Log.LogInformation($"{Name}: Add index for {source.FilePath}"); } } catch (Exception ex) { failedIndexFiles.Add(fileInfo); Log.LogError($"{Name}: Add index for {fileInfo.FullName} failed, exception: " + ex); } if (codeDocuments.Count >= batchSize) { BuildIndex(needCommit, triggerMerge, applyAllDeletes, codeDocuments, newHintWords, cancellationToken, brandNewBuild); codeDocuments.Clear(); newHintWords.Clear(); } } if (codeDocuments.Count > 0) { BuildIndex(needCommit, triggerMerge, applyAllDeletes, codeDocuments, newHintWords, cancellationToken, brandNewBuild); } return(failedIndexFiles); } finally { wholeWords.Clear(); newHintWords.Clear(); codeDocuments.Clear(); } }
public IndexBuildResults CreateIndex(FileInfo fileInfo) { try { if (fileInfo.Exists) { var source = CodeSource.GetCodeSource(fileInfo, FilesContentHelper.ReadAllText(fileInfo.FullName)); var words = new HashSet <string>(); AddHintWords(words, source.Content); var doc = IndexBuilderHelper.GetDocumentFromSource(source); CodeIndexPool.UpdateIndex(GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), source.FilePath), doc); foreach (var word in words) { HintIndexPool.UpdateIndex(new Term(nameof(CodeWord.Word), word), new Document { new StringField(nameof(CodeWord.Word), word, Field.Store.YES), new StringField(nameof(CodeWord.WordLower), word.ToLowerInvariant(), Field.Store.YES) }); } Log.LogInformation($"{Name}: Create index For {source.FilePath} finished"); } return(IndexBuildResults.Successful); } catch (Exception ex) { Log.LogError($"{Name}: Create index for {fileInfo.FullName} failed, exception: " + ex); if (ex is IOException) { return(IndexBuildResults.FailedWithIOException); } else if (ex is OperationCanceledException) { throw; } return(IndexBuildResults.FailedWithError); } }
public IndexBuildResults UpdateIndex(FileInfo fileInfo, CancellationToken cancellationToken) { try { if (fileInfo.Exists) { cancellationToken.ThrowIfCancellationRequested(); var source = CodeSource.GetCodeSource(fileInfo, FilesContentHelper.ReadAllText(fileInfo.FullName)); var words = new HashSet <string>(); AddHintWords(words, source.Content); var doc = IndexBuilderHelper.GetDocumentFromSource(source); CodeIndexPool.UpdateIndex(GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), source.FilePath), doc, out var rawDocuments); if (rawDocuments.Length >= 1) { var rawWords = new HashSet <string>(); AddHintWords(rawWords, GetCodeSourceFromDocument(rawDocuments[0]).Content); var wordsNeedToRemove = rawWords.Except(words).ToArray(); var wordsNeedToAdd = words.Except(rawWords); words = wordsNeedToAdd.ToHashSet(); Log.LogInformation($"{Name}: Find {wordsNeedToRemove.Length} Delete Candidates Words, {words.Count} Update Candidates Words With Path {source.FilePath}"); if (rawDocuments.Length > 1) { Log.LogError($"{Name}: Find {rawDocuments.Length} Documents With Path {source.FilePath} To Update"); } foreach (var needToDeleteWord in wordsNeedToRemove) { if (!CodeIndexPool.Exists(new TermQuery(new Term(GetCaseSensitiveField(nameof(CodeSource.Content)), needToDeleteWord)))) { HintIndexPool.DeleteIndex(new Term(nameof(CodeWord.Word), needToDeleteWord)); } } } else { Log.LogError($"{Name}: Find 0 Document To Update With Path {source.FilePath}, Create New Index"); } foreach (var word in words) { HintIndexPool.UpdateIndex(new Term(nameof(CodeWord.Word), word), new Document { new StringField(nameof(CodeWord.Word), word, Field.Store.YES), new StringField(nameof(CodeWord.WordLower), word.ToLowerInvariant(), Field.Store.YES) }); } Log.LogInformation($"{Name}: Update index For {source.FilePath} finished"); } return(IndexBuildResults.Successful); } catch (Exception ex) { Log.LogError($"{Name}: Update index for {fileInfo.FullName} failed, exception: " + ex); if (ex is IOException) { return(IndexBuildResults.FailedWithIOException); } else if (ex is OperationCanceledException) { throw; } return(IndexBuildResults.FailedWithError); } }