// This method gets called by the runtime. Use this method to add services to the container.
// For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940
public void ConfigureServices(IServiceCollection services)
{
services.AddMvc();
services.AddRazorPages();
services.AddServerSideBlazor();
services.AddSingleton <WeatherForecastService>();
services.AddSingleton <ILog>(new NLogger());
services.AddScoped <Storage>();
config = new CodeIndexConfiguration();
Configuration.GetSection("CodeIndex").Bind(config);
services.AddSingleton(config);
// Server Side Blazor doesn't register HttpClient by default
if (!services.Any(x => x.ServiceType == typeof(HttpClient)))
{
// Setup HttpClient for server side in a client side compatible fashion
services.AddScoped(s =>
{
// Creating the URI helper needs to wait until the JS Runtime is initialized, so defer it.
var uriHelper = s.GetRequiredService <NavigationManager>();
return(new HttpClient
{
BaseAddress = new Uri(uriHelper.BaseUri)
});
});
}
}
static void Main(string[] args)
{
var config = new CodeIndexConfiguration
{
MonitorFolder = @"D:\TestFolder\CodeFolder",
LuceneIndex = @"D:\TestFolder\Index",
ExcludedExtensions = ".DLL|.PBD",
ExcludedPaths = "\\DEBUG\\|\\RELEASE\\|\\RELEASES\\|\\BIN\\|\\OBJ\\|\\DEBUGPUBLIC\\",
IncludedExtensions = ".CS|.XML|.XAML|.JS|.TXT",
SaveIntervalSeconds = 300
};
var logger = new NLogger();
var initializer = new IndexInitializer(logger);
var maintainer = new CodeFilesIndexMaintainer(config, logger);
maintainer.StartWatch();
initializer.InitializeIndex(config, out var failedIndexFiles);
maintainer.SetInitalizeFinishedToTrue(failedIndexFiles);
Console.WriteLine("Initialize complete");
Console.WriteLine("Start monitoring, press any key to stop");
Console.ReadLine();
Console.WriteLine("Stop");
maintainer.Dispose();
}
static void BuildIndex(CodeIndexConfiguration config, bool triggerMerge, bool applyAllDeletes, List <Document> documents, bool needFlush, ILog log)
{
log?.Info($"Build index start, documents count {documents.Count}");
LucenePool.BuildIndex(config.LuceneIndexForHint, triggerMerge, applyAllDeletes, documents, needFlush);
LucenePool.SaveResultsAndClearLucenePool(config.LuceneIndexForHint);
log?.Info($"Build index finished");
}
public void TestProperties()
{
var config = new CodeIndexConfiguration()
{
IsInLinux = true,
LocalUrl = "http://localhost:1234",
LuceneIndex = "AAA/BBB",
MaximumResults = 234,
ManagerUsers = new[]
{
new UserInfo
{
Id = 1,
Password = "******",
UserName = "******"
}
}
};
Assert.Multiple(() =>
{
Assert.AreEqual(true, config.IsInLinux);
Assert.AreEqual("http://localhost:1234", config.LocalUrl);
Assert.AreEqual("AAA/BBB", config.LuceneIndex);
Assert.AreEqual(234, config.MaximumResults);
CollectionAssert.AreEquivalent(new[] { new UserInfo
{
Id = 1,
Password = "******",
UserName = "******"
} }, config.ManagerUsers);
});
}
public void TestProperties()
{
var config = new CodeIndexConfiguration()
{
ExcludedExtensions = ".cs|.dll|.EXE",
ExcludedPaths = "AAA|ccc",
IncludedExtensions = ".txt|.temp",
IsInLinux = true,
LocalUrl = "http://localhost:1234",
LuceneIndex = "AAA/BBB",
MaxContentHighlightLength = 3030,
MonitorFolder = "DDD/EEE",
MonitorFolderRealPath = "EEE",
OpenIDEUriFormat = "ABC{SSS}",
SaveIntervalSeconds = 123
};
Assert.Multiple(() =>
{
Assert.AreEqual(".cs|.dll|.EXE", config.ExcludedExtensions);
Assert.AreEqual("AAA|ccc", config.ExcludedPaths);
Assert.AreEqual(".txt|.temp", config.IncludedExtensions);
Assert.AreEqual(true, config.IsInLinux);
Assert.AreEqual("http://localhost:1234", config.LocalUrl);
Assert.AreEqual("AAA/BBB", config.LuceneIndex);
Assert.AreEqual(3030, config.MaxContentHighlightLength);
Assert.AreEqual("DDD/EEE", config.MonitorFolder);
Assert.AreEqual("EEE", config.MonitorFolderRealPath);
Assert.AreEqual("ABC{SSS}", config.OpenIDEUriFormat);
Assert.AreEqual(123, config.SaveIntervalSeconds);
CollectionAssert.AreEquivalent(new[] { ".cs", ".dll", ".EXE" }, config.ExcludedExtensionsArray);
CollectionAssert.AreEquivalent(new[] { "AAA", "ccc" }, config.ExcludedPathsArray);
CollectionAssert.AreEquivalent(new[] { ".txt", ".temp" }, config.IncludedExtensionsArray);
});
}
static void UpdateHintWordsAndSaveIndex(CodeIndexConfiguration config, string[] words, ILog log, int batchSize = -1, bool needSaveIndex = true)
{
var totalUpdate = 0;
log?.Info($"Update hint index start, words count {words.Length}");
foreach (var word in words)
{
var document = new Document
{
new StringField(nameof(CodeWord.Word), word, Field.Store.YES),
new StringField(nameof(CodeWord.WordLower), word.ToLowerInvariant(), Field.Store.YES)
};
LucenePool.UpdateIndex(config.LuceneIndexForHint, new Lucene.Net.Index.Term(nameof(CodeWord.Word), word), document);
totalUpdate++;
if (needSaveIndex && batchSize > 0 && totalUpdate > batchSize)
{
totalUpdate = 0;
LucenePool.SaveResultsAndClearLucenePool(config.LuceneIndexForHint);
}
}
if (needSaveIndex && batchSize > 0 && totalUpdate > 0)
{
LucenePool.SaveResultsAndClearLucenePool(config.LuceneIndexForHint);
}
log?.Info($"Update hint index finished");
}
public InitManagement(string monitorFolder, CodeIndexConfiguration codeIndexConfiguration, bool initFiles = false, int maxContentHighlightLength = Constants.DefaultMaxContentHighlightLength)
{
indexConfig = new IndexConfig
{
IndexName = "ABC",
MonitorFolder = monitorFolder,
MaxContentHighlightLength = maxContentHighlightLength
};
if (initFiles)
{
var fileName1 = Path.Combine(monitorFolder, "A.txt");
var fileName2 = Path.Combine(monitorFolder, "B.txt");
var fileName3 = Path.Combine(monitorFolder, "C.txt");
File.AppendAllText(fileName1, "ABCD");
File.AppendAllText(fileName2, "ABCD EFGH");
File.AppendAllText(fileName3, "ABCD EFGH IJKL" + Environment.NewLine + "ABCD ABCE");
}
management = new IndexManagement(codeIndexConfiguration, log1);
management.AddIndex(indexConfig);
var maintainer = management.GetIndexMaintainerWrapperAndInitializeIfNeeded(indexConfig.Pk);
// Wait initialized finished
while (maintainer.Result.Status == IndexStatus.Initializing_ComponentInitializeFinished || maintainer.Result.Status == IndexStatus.Initialized)
{
Thread.Sleep(100);
}
}
public IndexMaintainerWrapper(IndexConfig indexConfig, CodeIndexConfiguration codeIndexConfiguration, ILogger log)
{
indexConfig.RequireNotNull(nameof(indexConfig));
codeIndexConfiguration.RequireNotNull(nameof(codeIndexConfiguration));
log.RequireNotNull(nameof(log));
Maintainer = new IndexMaintainer(indexConfig, codeIndexConfiguration, log);
IndexConfig = indexConfig;
}
public static void BuildIndexByBatch(CodeIndexConfiguration config, bool triggerMerge, bool applyAllDeletes, bool needFlush, IEnumerable <FileInfo> fileInfos, bool deleteExistIndex, ILog log, out List <FileInfo> failedIndexFiles, int batchSize = 1000, bool needHint = true)
{
config.RequireNotNull(nameof(config));
fileInfos.RequireNotNull(nameof(fileInfos));
batchSize.RequireRange(nameof(batchSize), int.MaxValue, 50);
var needDeleteExistIndex = deleteExistIndex && IndexExists(config.LuceneIndexForCode);
var documents = new List <Document>();
failedIndexFiles = new List <FileInfo>();
foreach (var fileInfo in fileInfos)
{
try
{
if (fileInfo.Exists)
{
var source = CodeSource.GetCodeSource(fileInfo, FilesContentHelper.ReadAllText(fileInfo.FullName));
if (needDeleteExistIndex)
{
DeleteIndex(config.LuceneIndexForCode, new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, source.FilePath));
}
if (needHint)
{
WordsHintBuilder.AddWords(WordSegmenter.GetWords(source.Content));
}
var doc = GetDocumentFromSource(source);
documents.Add(doc);
log?.Info($"Add index For {source.FilePath}");
}
}
catch (Exception ex)
{
failedIndexFiles.Add(fileInfo);
log?.Error($"Add index for {fileInfo.FullName} failed, exception: " + ex.ToString());
}
if (documents.Count >= batchSize)
{
BuildIndex(config, triggerMerge, applyAllDeletes, documents, needFlush, log);
documents.Clear();
}
}
if (documents.Count > 0)
{
BuildIndex(config, triggerMerge, applyAllDeletes, documents, needFlush, log);
}
}
public IndexManagement(CodeIndexConfiguration codeIndexConfiguration, ILogger <IndexManagement> log)
{
codeIndexConfiguration.RequireNotNull(nameof(codeIndexConfiguration));
log.RequireNotNull(nameof(log));
MaintainerPool = new ConcurrentDictionary <Guid, IndexMaintainerWrapper>();
CodeIndexConfiguration = codeIndexConfiguration;
Log = log;
ConfigMaintainer = new ConfigIndexMaintainer(codeIndexConfiguration, log);
InitializeMaintainerPool();
}
public CodeFilesIndexMaintainer(CodeIndexConfiguration config, ILog log)
{
config.RequireNotNull(nameof(config));
config.ExcludedPathsArray.RequireNotNull(nameof(config.ExcludedPathsArray));
config.SaveIntervalSeconds.RequireRange(nameof(config.SaveIntervalSeconds), 3600, 1);
this.config = config;
excludedExtensions = config.ExcludedExtensionsArray.Select(u => u.ToUpperInvariant()).ToArray();
excludedPaths = FilePathHelper.GetPaths(config.ExcludedPathsArray, config.IsInLinux);
saveIntervalSeconds = config.SaveIntervalSeconds;
includedExtensions = config.IncludedExtensionsArray?.Select(u => u.ToUpperInvariant()).ToArray() ?? Array.Empty <string>();
this.log = log;
tokenSource = new CancellationTokenSource();
}
public static void InitIndexFolderIfNeeded(CodeIndexConfiguration config, ILog log)
{
if (!System.IO.Directory.Exists(config.LuceneIndexForCode))
{
log?.Info($"Create index folder {config.LuceneIndexForCode}");
System.IO.Directory.CreateDirectory(config.LuceneIndexForCode);
}
if (!System.IO.Directory.Exists(config.LuceneIndexForHint))
{
log?.Info($"Create index folder {config.LuceneIndexForHint}");
System.IO.Directory.CreateDirectory(config.LuceneIndexForHint);
}
}
public static void DeleteAllIndex(CodeIndexConfiguration config)
{
config.RequireNotNull(nameof(config));
if (IndexExists(config.LuceneIndexForCode))
{
LucenePool.DeleteAllIndex(config.LuceneIndexForCode);
}
if (IndexExists(config.LuceneIndexForHint))
{
LucenePool.DeleteAllIndex(config.LuceneIndexForHint);
}
}
public IndexMaintainer(IndexConfig indexConfig, CodeIndexConfiguration codeIndexConfiguration, ILogger log)
{
indexConfig.RequireNotNull(nameof(indexConfig));
codeIndexConfiguration.RequireNotNull(nameof(codeIndexConfiguration));
log.RequireNotNull(nameof(log));
IndexConfig = indexConfig;
CodeIndexConfiguration = codeIndexConfiguration;
Log = log;
Status = IndexStatus.Idle;
ExcludedExtensions = indexConfig.ExcludedExtensionsArray.Select(u => u.ToUpperInvariant()).ToArray();
ExcludedPaths = FilePathHelper.GetPaths(indexConfig.ExcludedPathsArray, codeIndexConfiguration.IsInLinux);
IncludedExtensions = indexConfig.IncludedExtensionsArray.Select(u => u.ToUpperInvariant()).ToArray() ?? Array.Empty <string>();
TokenSource = new CancellationTokenSource();
}
public ConfigIndexMaintainer(CodeIndexConfiguration codeIndexConfiguration, ILogger log)
{
codeIndexConfiguration.RequireNotNull(nameof(codeIndexConfiguration));
log.RequireNotNull(nameof(log));
CodeIndexConfiguration = codeIndexConfiguration;
Log = log;
var folder = Path.Combine(codeIndexConfiguration.LuceneIndex, CodeIndexConfiguration.ConfigurationIndexFolder);
if (!Directory.Exists(folder))
{
Log.LogInformation($"Create Configuraion index folder {folder}");
Directory.CreateDirectory(folder);
}
ConfigIndexBuilder = new ConfigIndexBuilder(folder);
}
public async Task <ClientLoginModel> Login([FromServices] CodeIndexConfiguration codeIndexConfiguration, ClientLoginModel loginModel)
{
if (string.IsNullOrWhiteSpace(loginModel.Captcha) || HttpContext.Session.GetString(nameof(ClientLoginModel.Captcha)) != loginModel.Captcha.ToLowerInvariant())
{
loginModel.Status = LoginStatus.Failed;
loginModel.Message = "Wrong captcha";
}
else
{
var user = codeIndexConfiguration.ManagerUsers?.FirstOrDefault(u => u.UserName == loginModel.UserName && u.Password == loginModel.Password);
if (user == null)
{
loginModel.Status = LoginStatus.Failed;
loginModel.Message = "Wrong username or password";
}
else
{
loginModel.Status = LoginStatus.Succesful;
var claims = new[]
{
new Claim(ClaimTypes.NameIdentifier, user.Id.ToString()),
new Claim(ClaimTypes.Name, user.UserName)
};
var identity = new ClaimsIdentity(claims, CookieAuthenticationDefaults.AuthenticationScheme);
var principal = new ClaimsPrincipal(identity);
var properties = new AuthenticationProperties();
if (loginModel.Persist)
{
properties.IsPersistent = true;
properties.ExpiresUtc = DateTime.UtcNow.AddMonths(12);
}
await HttpContext.SignInAsync(CookieAuthenticationDefaults.AuthenticationScheme, principal, properties);
}
}
HttpContext.Session.Remove(nameof(ClientLoginModel.Captcha));
return(loginModel);
}
public static void BuildIndexByBatch(CodeIndexConfiguration config, bool triggerMerge, bool applyAllDeletes, bool needFlush, ILog log, bool firstInitialize, int batchSize = 10000)
{
config.RequireNotNull(nameof(config));
batchSize.RequireRange(nameof(batchSize), int.MaxValue, 50);
var documents = new List <Document>();
log?.Info($"Start build hint words for {config.LuceneIndexForHint}");
if (firstInitialize)
{
foreach (var word in Words)
{
documents.Add(new Document
{
new StringField(nameof(CodeWord.Word), word, Field.Store.YES),
new StringField(nameof(CodeWord.WordLower), word.ToLowerInvariant(), Field.Store.YES)
});
if (documents.Count > batchSize)
{
BuildIndex(config, triggerMerge, applyAllDeletes, documents, needFlush, log);
documents.Clear();
}
}
if (documents.Count > 0)
{
BuildIndex(config, triggerMerge, applyAllDeletes, documents, needFlush, log);
}
}
else
{
UpdateHintWordsAndSaveIndex(config, Words.ToArray(), log, batchSize);
}
Words.Clear();
log?.Info($"Finished build hint words for {config.LuceneIndexForHint}");
}
public static void BuildIndex(CodeIndexConfiguration config, bool triggerMerge, bool applyAllDeletes, bool needFlush, IEnumerable <CodeSource> codeSources, bool deleteExistIndex = true, ILog log = null)
{
config.RequireNotNull(nameof(config));
codeSources.RequireNotNull(nameof(codeSources));
var needDeleteExistIndex = deleteExistIndex && IndexExists(config.LuceneIndexForCode);
var documents = new List <Document>();
foreach (var source in codeSources)
{
if (needDeleteExistIndex)
{
DeleteIndex(config.LuceneIndexForCode, new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, source.FilePath));
}
var doc = GetDocumentFromSource(source);
documents.Add(doc);
log?.Info($"Add index For {source.FilePath}");
}
BuildIndex(config, triggerMerge, applyAllDeletes, documents, needFlush, log);
}
public CodeFilesIndexMaintainerForTest(CodeIndexConfiguration config, ILog log) : base(config, log)
{
}
// This method gets called by the runtime. Use this method to add services to the container.
// For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940
public void ConfigureServices(IServiceCollection services)
{
#region Config swagger
// Register the Swagger services
services.AddSwaggerDocument(config =>
{
config.Title = "CodeIndex Server API";
});
#endregion
services.AddSession(options =>
{
options.IdleTimeout = TimeSpan.FromMinutes(60);
options.Cookie.HttpOnly = true;
options.Cookie.IsEssential = true;
});
services.AddAuthentication(CookieAuthenticationDefaults.AuthenticationScheme).AddCookie(options =>
{
options.Events = new CookieAuthenticationEvents
{
OnRedirectToLogin = ctx =>
{
if (ctx.Request.Path.StartsWithSegments("/api"))
{
ctx.Response.StatusCode = (int)HttpStatusCode.Unauthorized;
}
else
{
ctx.Response.Redirect(ctx.RedirectUri);
}
return(Task.FromResult(0));
}
};
});
services.AddMvc();
services.AddRazorPages();
services.AddServerSideBlazor();
services.AddSingleton <IndexManagement>();
services.AddScoped <Storage>();
services.AddSingleton <CodeIndexSearcher>();
services.AddSingleton <SearchService>();
config = new CodeIndexConfiguration();
Configuration.GetSection("CodeIndex").Bind(config);
services.AddSingleton(config);
// Server Side Blazor doesn't register HttpClient by default
if (services.All(x => x.ServiceType != typeof(HttpClient)))
{
// Setup HttpClient for server side in a client side compatible fashion
services.AddScoped(s =>
{
// Creating the URI helper needs to wait until the JS Runtime is initialized, so defer it.
var uriHelper = s.GetRequiredService <NavigationManager>();
return(new HttpClient
{
BaseAddress = new Uri(uriHelper.BaseUri)
});
});
}
}
public IndexMaintainerForTest(IndexConfig indexConfig, CodeIndexConfiguration codeIndexConfiguration, ILogger <IndexMaintainerForTest> log, AutoResetEvent resetEvent) : base(indexConfig, codeIndexConfiguration, log)
{
ResetEvent = resetEvent;
}
public void InitializeIndex(CodeIndexConfiguration config, out List <FileInfo> failedIndexFiles, bool forceDeleteAllIndex = false)
{
config.RequireNotNull(nameof(config));
log?.Info($"Initialize start for {config.LuceneIndex}");
var allFiles = FilesFetcher.FetchAllFiles(config.MonitorFolder, config.ExcludedExtensionsArray, config.ExcludedPathsArray, includedExtensions: config.IncludedExtensionsArray, isInLinux: config.IsInLinux).ToList();
List <FileInfo> needToBuildIndex = null;
var firstInitialize = true;
CodeIndexBuilder.InitIndexFolderIfNeeded(config, log);
if (CodeIndexBuilder.IndexExists(config.LuceneIndexForCode))
{
if (forceDeleteAllIndex)
{
log?.Info("Delete exist index");
CodeIndexBuilder.DeleteAllIndex(config);
}
else
{
firstInitialize = false;
log?.Info("Compare index difference");
var allCodeSource = CodeIndexBuilder.GetAllIndexedCodeSource(config.LuceneIndexForCode);
needToBuildIndex = new List <FileInfo>();
foreach (var codeSource in allCodeSource)
{
var fileInfo = allFiles.FirstOrDefault(u => u.FullName == codeSource.FilePath);
if (fileInfo != null)
{
if (fileInfo.LastWriteTimeUtc != codeSource.LastWriteTimeUtc)
{
log?.Info($"File {fileInfo.FullName} modified");
CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, CodeFilesIndexMaintainer.GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), codeSource.FilePath));
needToBuildIndex.Add(fileInfo);
}
allFiles.Remove(fileInfo);
}
else
{
log?.Info($"File {codeSource.FilePath} deleted");
CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, CodeFilesIndexMaintainer.GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), codeSource.FilePath));
}
}
foreach (var needToCreateFiles in allFiles)
{
log?.Info($"Found new file {needToCreateFiles.FullName}");
needToBuildIndex.Add(needToCreateFiles);
}
}
}
CodeIndexBuilder.BuildIndexByBatch(config, true, true, true, needToBuildIndex ?? allFiles, false, log, out failedIndexFiles);
LucenePool.SaveResultsAndClearLucenePool(config.LuceneIndexForCode);
WordsHintBuilder.BuildIndexByBatch(config, true, true, true, log, firstInitialize);
log?.Info($"Initialize finished for {config.LuceneIndex}");
}
public static void UpdateWordsHint(CodeIndexConfiguration config, string[] words, ILog log)
{
words = words.Where(HasValidLength).Distinct().ToArray();
UpdateHintWordsAndSaveIndex(config, words, log, needSaveIndex: false);
}
public SearchService(CodeIndexConfiguration codeIndexConfiguration, ILogger <SearchService> log, CodeIndexSearcher codeIndexSearcher)
{
CodeIndexConfiguration = codeIndexConfiguration;
Log = log;
CodeIndexSearcher = codeIndexSearcher;
}
public static void SaveResultsAndClearLucenePool(CodeIndexConfiguration config)
{
SaveResultsAndClearLucenePool(config.LuceneIndexForCode);
SaveResultsAndClearLucenePool(config.LuceneIndexForHint);
}
public LuceneController(CodeIndexConfiguration codeIndexConfiguration, ILog log)
{
this.codeIndexConfiguration = codeIndexConfiguration;
this.log = log;
}