// This method gets called by the runtime. Use this method to add services to the container. // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 public void ConfigureServices(IServiceCollection services) { services.AddMvc(); services.AddRazorPages(); services.AddServerSideBlazor(); services.AddSingleton <WeatherForecastService>(); services.AddSingleton <ILog>(new NLogger()); services.AddScoped <Storage>(); config = new CodeIndexConfiguration(); Configuration.GetSection("CodeIndex").Bind(config); services.AddSingleton(config); // Server Side Blazor doesn't register HttpClient by default if (!services.Any(x => x.ServiceType == typeof(HttpClient))) { // Setup HttpClient for server side in a client side compatible fashion services.AddScoped(s => { // Creating the URI helper needs to wait until the JS Runtime is initialized, so defer it. var uriHelper = s.GetRequiredService <NavigationManager>(); return(new HttpClient { BaseAddress = new Uri(uriHelper.BaseUri) }); }); } }
static void Main(string[] args) { var config = new CodeIndexConfiguration { MonitorFolder = @"D:\TestFolder\CodeFolder", LuceneIndex = @"D:\TestFolder\Index", ExcludedExtensions = ".DLL|.PBD", ExcludedPaths = "\\DEBUG\\|\\RELEASE\\|\\RELEASES\\|\\BIN\\|\\OBJ\\|\\DEBUGPUBLIC\\", IncludedExtensions = ".CS|.XML|.XAML|.JS|.TXT", SaveIntervalSeconds = 300 }; var logger = new NLogger(); var initializer = new IndexInitializer(logger); var maintainer = new CodeFilesIndexMaintainer(config, logger); maintainer.StartWatch(); initializer.InitializeIndex(config, out var failedIndexFiles); maintainer.SetInitalizeFinishedToTrue(failedIndexFiles); Console.WriteLine("Initialize complete"); Console.WriteLine("Start monitoring, press any key to stop"); Console.ReadLine(); Console.WriteLine("Stop"); maintainer.Dispose(); }
static void BuildIndex(CodeIndexConfiguration config, bool triggerMerge, bool applyAllDeletes, List <Document> documents, bool needFlush, ILog log) { log?.Info($"Build index start, documents count {documents.Count}"); LucenePool.BuildIndex(config.LuceneIndexForHint, triggerMerge, applyAllDeletes, documents, needFlush); LucenePool.SaveResultsAndClearLucenePool(config.LuceneIndexForHint); log?.Info($"Build index finished"); }
public void TestProperties() { var config = new CodeIndexConfiguration() { IsInLinux = true, LocalUrl = "http://localhost:1234", LuceneIndex = "AAA/BBB", MaximumResults = 234, ManagerUsers = new[] { new UserInfo { Id = 1, Password = "******", UserName = "******" } } }; Assert.Multiple(() => { Assert.AreEqual(true, config.IsInLinux); Assert.AreEqual("http://localhost:1234", config.LocalUrl); Assert.AreEqual("AAA/BBB", config.LuceneIndex); Assert.AreEqual(234, config.MaximumResults); CollectionAssert.AreEquivalent(new[] { new UserInfo { Id = 1, Password = "******", UserName = "******" } }, config.ManagerUsers); }); }
public void TestProperties() { var config = new CodeIndexConfiguration() { ExcludedExtensions = ".cs|.dll|.EXE", ExcludedPaths = "AAA|ccc", IncludedExtensions = ".txt|.temp", IsInLinux = true, LocalUrl = "http://localhost:1234", LuceneIndex = "AAA/BBB", MaxContentHighlightLength = 3030, MonitorFolder = "DDD/EEE", MonitorFolderRealPath = "EEE", OpenIDEUriFormat = "ABC{SSS}", SaveIntervalSeconds = 123 }; Assert.Multiple(() => { Assert.AreEqual(".cs|.dll|.EXE", config.ExcludedExtensions); Assert.AreEqual("AAA|ccc", config.ExcludedPaths); Assert.AreEqual(".txt|.temp", config.IncludedExtensions); Assert.AreEqual(true, config.IsInLinux); Assert.AreEqual("http://localhost:1234", config.LocalUrl); Assert.AreEqual("AAA/BBB", config.LuceneIndex); Assert.AreEqual(3030, config.MaxContentHighlightLength); Assert.AreEqual("DDD/EEE", config.MonitorFolder); Assert.AreEqual("EEE", config.MonitorFolderRealPath); Assert.AreEqual("ABC{SSS}", config.OpenIDEUriFormat); Assert.AreEqual(123, config.SaveIntervalSeconds); CollectionAssert.AreEquivalent(new[] { ".cs", ".dll", ".EXE" }, config.ExcludedExtensionsArray); CollectionAssert.AreEquivalent(new[] { "AAA", "ccc" }, config.ExcludedPathsArray); CollectionAssert.AreEquivalent(new[] { ".txt", ".temp" }, config.IncludedExtensionsArray); }); }
static void UpdateHintWordsAndSaveIndex(CodeIndexConfiguration config, string[] words, ILog log, int batchSize = -1, bool needSaveIndex = true) { var totalUpdate = 0; log?.Info($"Update hint index start, words count {words.Length}"); foreach (var word in words) { var document = new Document { new StringField(nameof(CodeWord.Word), word, Field.Store.YES), new StringField(nameof(CodeWord.WordLower), word.ToLowerInvariant(), Field.Store.YES) }; LucenePool.UpdateIndex(config.LuceneIndexForHint, new Lucene.Net.Index.Term(nameof(CodeWord.Word), word), document); totalUpdate++; if (needSaveIndex && batchSize > 0 && totalUpdate > batchSize) { totalUpdate = 0; LucenePool.SaveResultsAndClearLucenePool(config.LuceneIndexForHint); } } if (needSaveIndex && batchSize > 0 && totalUpdate > 0) { LucenePool.SaveResultsAndClearLucenePool(config.LuceneIndexForHint); } log?.Info($"Update hint index finished"); }
public InitManagement(string monitorFolder, CodeIndexConfiguration codeIndexConfiguration, bool initFiles = false, int maxContentHighlightLength = Constants.DefaultMaxContentHighlightLength) { indexConfig = new IndexConfig { IndexName = "ABC", MonitorFolder = monitorFolder, MaxContentHighlightLength = maxContentHighlightLength }; if (initFiles) { var fileName1 = Path.Combine(monitorFolder, "A.txt"); var fileName2 = Path.Combine(monitorFolder, "B.txt"); var fileName3 = Path.Combine(monitorFolder, "C.txt"); File.AppendAllText(fileName1, "ABCD"); File.AppendAllText(fileName2, "ABCD EFGH"); File.AppendAllText(fileName3, "ABCD EFGH IJKL" + Environment.NewLine + "ABCD ABCE"); } management = new IndexManagement(codeIndexConfiguration, log1); management.AddIndex(indexConfig); var maintainer = management.GetIndexMaintainerWrapperAndInitializeIfNeeded(indexConfig.Pk); // Wait initialized finished while (maintainer.Result.Status == IndexStatus.Initializing_ComponentInitializeFinished || maintainer.Result.Status == IndexStatus.Initialized) { Thread.Sleep(100); } }
public IndexMaintainerWrapper(IndexConfig indexConfig, CodeIndexConfiguration codeIndexConfiguration, ILogger log) { indexConfig.RequireNotNull(nameof(indexConfig)); codeIndexConfiguration.RequireNotNull(nameof(codeIndexConfiguration)); log.RequireNotNull(nameof(log)); Maintainer = new IndexMaintainer(indexConfig, codeIndexConfiguration, log); IndexConfig = indexConfig; }
public static void BuildIndexByBatch(CodeIndexConfiguration config, bool triggerMerge, bool applyAllDeletes, bool needFlush, IEnumerable <FileInfo> fileInfos, bool deleteExistIndex, ILog log, out List <FileInfo> failedIndexFiles, int batchSize = 1000, bool needHint = true) { config.RequireNotNull(nameof(config)); fileInfos.RequireNotNull(nameof(fileInfos)); batchSize.RequireRange(nameof(batchSize), int.MaxValue, 50); var needDeleteExistIndex = deleteExistIndex && IndexExists(config.LuceneIndexForCode); var documents = new List <Document>(); failedIndexFiles = new List <FileInfo>(); foreach (var fileInfo in fileInfos) { try { if (fileInfo.Exists) { var source = CodeSource.GetCodeSource(fileInfo, FilesContentHelper.ReadAllText(fileInfo.FullName)); if (needDeleteExistIndex) { DeleteIndex(config.LuceneIndexForCode, new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, source.FilePath)); } if (needHint) { WordsHintBuilder.AddWords(WordSegmenter.GetWords(source.Content)); } var doc = GetDocumentFromSource(source); documents.Add(doc); log?.Info($"Add index For {source.FilePath}"); } } catch (Exception ex) { failedIndexFiles.Add(fileInfo); log?.Error($"Add index for {fileInfo.FullName} failed, exception: " + ex.ToString()); } if (documents.Count >= batchSize) { BuildIndex(config, triggerMerge, applyAllDeletes, documents, needFlush, log); documents.Clear(); } } if (documents.Count > 0) { BuildIndex(config, triggerMerge, applyAllDeletes, documents, needFlush, log); } }
public IndexManagement(CodeIndexConfiguration codeIndexConfiguration, ILogger <IndexManagement> log) { codeIndexConfiguration.RequireNotNull(nameof(codeIndexConfiguration)); log.RequireNotNull(nameof(log)); MaintainerPool = new ConcurrentDictionary <Guid, IndexMaintainerWrapper>(); CodeIndexConfiguration = codeIndexConfiguration; Log = log; ConfigMaintainer = new ConfigIndexMaintainer(codeIndexConfiguration, log); InitializeMaintainerPool(); }
public CodeFilesIndexMaintainer(CodeIndexConfiguration config, ILog log) { config.RequireNotNull(nameof(config)); config.ExcludedPathsArray.RequireNotNull(nameof(config.ExcludedPathsArray)); config.SaveIntervalSeconds.RequireRange(nameof(config.SaveIntervalSeconds), 3600, 1); this.config = config; excludedExtensions = config.ExcludedExtensionsArray.Select(u => u.ToUpperInvariant()).ToArray(); excludedPaths = FilePathHelper.GetPaths(config.ExcludedPathsArray, config.IsInLinux); saveIntervalSeconds = config.SaveIntervalSeconds; includedExtensions = config.IncludedExtensionsArray?.Select(u => u.ToUpperInvariant()).ToArray() ?? Array.Empty <string>(); this.log = log; tokenSource = new CancellationTokenSource(); }
public static void InitIndexFolderIfNeeded(CodeIndexConfiguration config, ILog log) { if (!System.IO.Directory.Exists(config.LuceneIndexForCode)) { log?.Info($"Create index folder {config.LuceneIndexForCode}"); System.IO.Directory.CreateDirectory(config.LuceneIndexForCode); } if (!System.IO.Directory.Exists(config.LuceneIndexForHint)) { log?.Info($"Create index folder {config.LuceneIndexForHint}"); System.IO.Directory.CreateDirectory(config.LuceneIndexForHint); } }
public static void DeleteAllIndex(CodeIndexConfiguration config) { config.RequireNotNull(nameof(config)); if (IndexExists(config.LuceneIndexForCode)) { LucenePool.DeleteAllIndex(config.LuceneIndexForCode); } if (IndexExists(config.LuceneIndexForHint)) { LucenePool.DeleteAllIndex(config.LuceneIndexForHint); } }
public IndexMaintainer(IndexConfig indexConfig, CodeIndexConfiguration codeIndexConfiguration, ILogger log) { indexConfig.RequireNotNull(nameof(indexConfig)); codeIndexConfiguration.RequireNotNull(nameof(codeIndexConfiguration)); log.RequireNotNull(nameof(log)); IndexConfig = indexConfig; CodeIndexConfiguration = codeIndexConfiguration; Log = log; Status = IndexStatus.Idle; ExcludedExtensions = indexConfig.ExcludedExtensionsArray.Select(u => u.ToUpperInvariant()).ToArray(); ExcludedPaths = FilePathHelper.GetPaths(indexConfig.ExcludedPathsArray, codeIndexConfiguration.IsInLinux); IncludedExtensions = indexConfig.IncludedExtensionsArray.Select(u => u.ToUpperInvariant()).ToArray() ?? Array.Empty <string>(); TokenSource = new CancellationTokenSource(); }
public ConfigIndexMaintainer(CodeIndexConfiguration codeIndexConfiguration, ILogger log) { codeIndexConfiguration.RequireNotNull(nameof(codeIndexConfiguration)); log.RequireNotNull(nameof(log)); CodeIndexConfiguration = codeIndexConfiguration; Log = log; var folder = Path.Combine(codeIndexConfiguration.LuceneIndex, CodeIndexConfiguration.ConfigurationIndexFolder); if (!Directory.Exists(folder)) { Log.LogInformation($"Create Configuraion index folder {folder}"); Directory.CreateDirectory(folder); } ConfigIndexBuilder = new ConfigIndexBuilder(folder); }
public async Task <ClientLoginModel> Login([FromServices] CodeIndexConfiguration codeIndexConfiguration, ClientLoginModel loginModel) { if (string.IsNullOrWhiteSpace(loginModel.Captcha) || HttpContext.Session.GetString(nameof(ClientLoginModel.Captcha)) != loginModel.Captcha.ToLowerInvariant()) { loginModel.Status = LoginStatus.Failed; loginModel.Message = "Wrong captcha"; } else { var user = codeIndexConfiguration.ManagerUsers?.FirstOrDefault(u => u.UserName == loginModel.UserName && u.Password == loginModel.Password); if (user == null) { loginModel.Status = LoginStatus.Failed; loginModel.Message = "Wrong username or password"; } else { loginModel.Status = LoginStatus.Succesful; var claims = new[] { new Claim(ClaimTypes.NameIdentifier, user.Id.ToString()), new Claim(ClaimTypes.Name, user.UserName) }; var identity = new ClaimsIdentity(claims, CookieAuthenticationDefaults.AuthenticationScheme); var principal = new ClaimsPrincipal(identity); var properties = new AuthenticationProperties(); if (loginModel.Persist) { properties.IsPersistent = true; properties.ExpiresUtc = DateTime.UtcNow.AddMonths(12); } await HttpContext.SignInAsync(CookieAuthenticationDefaults.AuthenticationScheme, principal, properties); } } HttpContext.Session.Remove(nameof(ClientLoginModel.Captcha)); return(loginModel); }
public static void BuildIndexByBatch(CodeIndexConfiguration config, bool triggerMerge, bool applyAllDeletes, bool needFlush, ILog log, bool firstInitialize, int batchSize = 10000) { config.RequireNotNull(nameof(config)); batchSize.RequireRange(nameof(batchSize), int.MaxValue, 50); var documents = new List <Document>(); log?.Info($"Start build hint words for {config.LuceneIndexForHint}"); if (firstInitialize) { foreach (var word in Words) { documents.Add(new Document { new StringField(nameof(CodeWord.Word), word, Field.Store.YES), new StringField(nameof(CodeWord.WordLower), word.ToLowerInvariant(), Field.Store.YES) }); if (documents.Count > batchSize) { BuildIndex(config, triggerMerge, applyAllDeletes, documents, needFlush, log); documents.Clear(); } } if (documents.Count > 0) { BuildIndex(config, triggerMerge, applyAllDeletes, documents, needFlush, log); } } else { UpdateHintWordsAndSaveIndex(config, Words.ToArray(), log, batchSize); } Words.Clear(); log?.Info($"Finished build hint words for {config.LuceneIndexForHint}"); }
public static void BuildIndex(CodeIndexConfiguration config, bool triggerMerge, bool applyAllDeletes, bool needFlush, IEnumerable <CodeSource> codeSources, bool deleteExistIndex = true, ILog log = null) { config.RequireNotNull(nameof(config)); codeSources.RequireNotNull(nameof(codeSources)); var needDeleteExistIndex = deleteExistIndex && IndexExists(config.LuceneIndexForCode); var documents = new List <Document>(); foreach (var source in codeSources) { if (needDeleteExistIndex) { DeleteIndex(config.LuceneIndexForCode, new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, source.FilePath)); } var doc = GetDocumentFromSource(source); documents.Add(doc); log?.Info($"Add index For {source.FilePath}"); } BuildIndex(config, triggerMerge, applyAllDeletes, documents, needFlush, log); }
public CodeFilesIndexMaintainerForTest(CodeIndexConfiguration config, ILog log) : base(config, log) { }
// This method gets called by the runtime. Use this method to add services to the container. // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 public void ConfigureServices(IServiceCollection services) { #region Config swagger // Register the Swagger services services.AddSwaggerDocument(config => { config.Title = "CodeIndex Server API"; }); #endregion services.AddSession(options => { options.IdleTimeout = TimeSpan.FromMinutes(60); options.Cookie.HttpOnly = true; options.Cookie.IsEssential = true; }); services.AddAuthentication(CookieAuthenticationDefaults.AuthenticationScheme).AddCookie(options => { options.Events = new CookieAuthenticationEvents { OnRedirectToLogin = ctx => { if (ctx.Request.Path.StartsWithSegments("/api")) { ctx.Response.StatusCode = (int)HttpStatusCode.Unauthorized; } else { ctx.Response.Redirect(ctx.RedirectUri); } return(Task.FromResult(0)); } }; }); services.AddMvc(); services.AddRazorPages(); services.AddServerSideBlazor(); services.AddSingleton <IndexManagement>(); services.AddScoped <Storage>(); services.AddSingleton <CodeIndexSearcher>(); services.AddSingleton <SearchService>(); config = new CodeIndexConfiguration(); Configuration.GetSection("CodeIndex").Bind(config); services.AddSingleton(config); // Server Side Blazor doesn't register HttpClient by default if (services.All(x => x.ServiceType != typeof(HttpClient))) { // Setup HttpClient for server side in a client side compatible fashion services.AddScoped(s => { // Creating the URI helper needs to wait until the JS Runtime is initialized, so defer it. var uriHelper = s.GetRequiredService <NavigationManager>(); return(new HttpClient { BaseAddress = new Uri(uriHelper.BaseUri) }); }); } }
public IndexMaintainerForTest(IndexConfig indexConfig, CodeIndexConfiguration codeIndexConfiguration, ILogger <IndexMaintainerForTest> log, AutoResetEvent resetEvent) : base(indexConfig, codeIndexConfiguration, log) { ResetEvent = resetEvent; }
public void InitializeIndex(CodeIndexConfiguration config, out List <FileInfo> failedIndexFiles, bool forceDeleteAllIndex = false) { config.RequireNotNull(nameof(config)); log?.Info($"Initialize start for {config.LuceneIndex}"); var allFiles = FilesFetcher.FetchAllFiles(config.MonitorFolder, config.ExcludedExtensionsArray, config.ExcludedPathsArray, includedExtensions: config.IncludedExtensionsArray, isInLinux: config.IsInLinux).ToList(); List <FileInfo> needToBuildIndex = null; var firstInitialize = true; CodeIndexBuilder.InitIndexFolderIfNeeded(config, log); if (CodeIndexBuilder.IndexExists(config.LuceneIndexForCode)) { if (forceDeleteAllIndex) { log?.Info("Delete exist index"); CodeIndexBuilder.DeleteAllIndex(config); } else { firstInitialize = false; log?.Info("Compare index difference"); var allCodeSource = CodeIndexBuilder.GetAllIndexedCodeSource(config.LuceneIndexForCode); needToBuildIndex = new List <FileInfo>(); foreach (var codeSource in allCodeSource) { var fileInfo = allFiles.FirstOrDefault(u => u.FullName == codeSource.FilePath); if (fileInfo != null) { if (fileInfo.LastWriteTimeUtc != codeSource.LastWriteTimeUtc) { log?.Info($"File {fileInfo.FullName} modified"); CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, CodeFilesIndexMaintainer.GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), codeSource.FilePath)); needToBuildIndex.Add(fileInfo); } allFiles.Remove(fileInfo); } else { log?.Info($"File {codeSource.FilePath} deleted"); CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, CodeFilesIndexMaintainer.GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), codeSource.FilePath)); } } foreach (var needToCreateFiles in allFiles) { log?.Info($"Found new file {needToCreateFiles.FullName}"); needToBuildIndex.Add(needToCreateFiles); } } } CodeIndexBuilder.BuildIndexByBatch(config, true, true, true, needToBuildIndex ?? allFiles, false, log, out failedIndexFiles); LucenePool.SaveResultsAndClearLucenePool(config.LuceneIndexForCode); WordsHintBuilder.BuildIndexByBatch(config, true, true, true, log, firstInitialize); log?.Info($"Initialize finished for {config.LuceneIndex}"); }
public static void UpdateWordsHint(CodeIndexConfiguration config, string[] words, ILog log) { words = words.Where(HasValidLength).Distinct().ToArray(); UpdateHintWordsAndSaveIndex(config, words, log, needSaveIndex: false); }
public SearchService(CodeIndexConfiguration codeIndexConfiguration, ILogger <SearchService> log, CodeIndexSearcher codeIndexSearcher) { CodeIndexConfiguration = codeIndexConfiguration; Log = log; CodeIndexSearcher = codeIndexSearcher; }
public static void SaveResultsAndClearLucenePool(CodeIndexConfiguration config) { SaveResultsAndClearLucenePool(config.LuceneIndexForCode); SaveResultsAndClearLucenePool(config.LuceneIndexForHint); }
public LuceneController(CodeIndexConfiguration codeIndexConfiguration, ILog log) { this.codeIndexConfiguration = codeIndexConfiguration; this.log = log; }