Exemplo n.º 1
0
        public List <FileInfo> BuildIndexByBatch(IEnumerable <FileInfo> fileInfos, bool needCommit, bool triggerMerge, bool applyAllDeletes, CancellationToken cancellationToken, bool brandNewBuild, int batchSize = 10000)
        {
            cancellationToken.ThrowIfCancellationRequested();
            fileInfos.RequireNotNull(nameof(fileInfos));
            batchSize.RequireRange(nameof(batchSize), int.MaxValue, 50);

            var codeDocuments    = new List <Document>();
            var wholeWords       = new HashSet <string>();
            var newHintWords     = new HashSet <string>();
            var failedIndexFiles = new List <FileInfo>();

            try
            {
                foreach (var fileInfo in fileInfos)
                {
                    cancellationToken.ThrowIfCancellationRequested();

                    try
                    {
                        if (fileInfo.Exists)
                        {
                            var source = CodeSource.GetCodeSource(fileInfo, FilesContentHelper.ReadAllText(fileInfo.FullName));

                            AddHintWords(newHintWords, wholeWords, source.Content);

                            var doc = IndexBuilderHelper.GetDocumentFromSource(source);
                            codeDocuments.Add(doc);

                            Log.LogInformation($"{Name}: Add index for {source.FilePath}");
                        }
                    }
                    catch (Exception ex)
                    {
                        failedIndexFiles.Add(fileInfo);
                        Log.LogError($"{Name}: Add index for {fileInfo.FullName} failed, exception: " + ex);
                    }

                    if (codeDocuments.Count >= batchSize)
                    {
                        BuildIndex(needCommit, triggerMerge, applyAllDeletes, codeDocuments, newHintWords, cancellationToken, brandNewBuild);
                        codeDocuments.Clear();
                        newHintWords.Clear();
                    }
                }

                if (codeDocuments.Count > 0)
                {
                    BuildIndex(needCommit, triggerMerge, applyAllDeletes, codeDocuments, newHintWords, cancellationToken, brandNewBuild);
                }

                return(failedIndexFiles);
            }
            finally
            {
                wholeWords.Clear();
                newHintWords.Clear();
                codeDocuments.Clear();
            }
        }
Exemplo n.º 2
0
        public IndexBuildResults CreateIndex(FileInfo fileInfo)
        {
            try
            {
                if (fileInfo.Exists)
                {
                    var source = CodeSource.GetCodeSource(fileInfo, FilesContentHelper.ReadAllText(fileInfo.FullName));

                    var words = new HashSet <string>();
                    AddHintWords(words, source.Content);

                    var doc = IndexBuilderHelper.GetDocumentFromSource(source);
                    CodeIndexPool.UpdateIndex(GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), source.FilePath), doc);

                    foreach (var word in words)
                    {
                        HintIndexPool.UpdateIndex(new Term(nameof(CodeWord.Word), word), new Document
                        {
                            new StringField(nameof(CodeWord.Word), word, Field.Store.YES),
                            new StringField(nameof(CodeWord.WordLower), word.ToLowerInvariant(), Field.Store.YES)
                        });
                    }

                    Log.LogInformation($"{Name}: Create index For {source.FilePath} finished");
                }

                return(IndexBuildResults.Successful);
            }
            catch (Exception ex)
            {
                Log.LogError($"{Name}: Create index for {fileInfo.FullName} failed, exception: " + ex);

                if (ex is IOException)
                {
                    return(IndexBuildResults.FailedWithIOException);
                }
                else if (ex is OperationCanceledException)
                {
                    throw;
                }

                return(IndexBuildResults.FailedWithError);
            }
        }
Exemplo n.º 3
0
        public IndexBuildResults UpdateIndex(FileInfo fileInfo, CancellationToken cancellationToken)
        {
            try
            {
                if (fileInfo.Exists)
                {
                    cancellationToken.ThrowIfCancellationRequested();

                    var source = CodeSource.GetCodeSource(fileInfo, FilesContentHelper.ReadAllText(fileInfo.FullName));

                    var words = new HashSet <string>();
                    AddHintWords(words, source.Content);

                    var doc = IndexBuilderHelper.GetDocumentFromSource(source);
                    CodeIndexPool.UpdateIndex(GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), source.FilePath), doc, out var rawDocuments);

                    if (rawDocuments.Length >= 1)
                    {
                        var rawWords = new HashSet <string>();
                        AddHintWords(rawWords, GetCodeSourceFromDocument(rawDocuments[0]).Content);

                        var wordsNeedToRemove = rawWords.Except(words).ToArray();
                        var wordsNeedToAdd    = words.Except(rawWords);
                        words = wordsNeedToAdd.ToHashSet();

                        Log.LogInformation($"{Name}: Find {wordsNeedToRemove.Length} Delete Candidates Words, {words.Count} Update Candidates Words With Path {source.FilePath}");

                        if (rawDocuments.Length > 1)
                        {
                            Log.LogError($"{Name}: Find {rawDocuments.Length} Documents With Path {source.FilePath} To Update");
                        }

                        foreach (var needToDeleteWord in wordsNeedToRemove)
                        {
                            if (!CodeIndexPool.Exists(new TermQuery(new Term(GetCaseSensitiveField(nameof(CodeSource.Content)), needToDeleteWord))))
                            {
                                HintIndexPool.DeleteIndex(new Term(nameof(CodeWord.Word), needToDeleteWord));
                            }
                        }
                    }
                    else
                    {
                        Log.LogError($"{Name}: Find 0 Document To Update With Path {source.FilePath}, Create New Index");
                    }

                    foreach (var word in words)
                    {
                        HintIndexPool.UpdateIndex(new Term(nameof(CodeWord.Word), word), new Document
                        {
                            new StringField(nameof(CodeWord.Word), word, Field.Store.YES),
                            new StringField(nameof(CodeWord.WordLower), word.ToLowerInvariant(), Field.Store.YES)
                        });
                    }

                    Log.LogInformation($"{Name}: Update index For {source.FilePath} finished");
                }

                return(IndexBuildResults.Successful);
            }
            catch (Exception ex)
            {
                Log.LogError($"{Name}: Update index for {fileInfo.FullName} failed, exception: " + ex);

                if (ex is IOException)
                {
                    return(IndexBuildResults.FailedWithIOException);
                }
                else if (ex is OperationCanceledException)
                {
                    throw;
                }

                return(IndexBuildResults.FailedWithError);
            }
        }