public static void Start()
{
if (RunSettings.DocumentNumber == -1)
{
var files = FileTreeTools.GetFileLinkRelations(RunSettings.DatasetPath);
RunSettings.DocumentNumber = files.Count;
}
var parallelOptions = new ParallelOptions()
{
MaxDegreeOfParallelism = RunSettings.MaxDegreeOfParallelism
};
Parallel.ForEach(Master.GetGroupedWordsBatch(), parallelOptions, Slave.BuildReducedReverseIndex);
Console.WriteLine($"Building a database index on {DbClient.ReducedReverseIndexCollectionName} collection");
CreateDbIndex();
}
public static void InitializeSettings()
{
DbClient.ClearDatabase();
var projectDir = Environment.CurrentDirectory.Contains("Debug\\netcoreapp") ?
Path.GetFullPath(Path.Combine(Environment.CurrentDirectory, @"..\..\..\")) :
Environment.CurrentDirectory;
var siteDir = Path.Combine(projectDir, "riwdataset");
RunSettings.DatasetPath = siteDir;
RunSettings.LogType = LogType.Debug;
RunSettings.DocumentBatchSize = 5;
RunSettings.GroupedWordsBatchSize = 300; //used for calculating idf
RunSettings.FilePaths = FileTreeTools.GetFileLinkRelations(RunSettings.DatasetPath);
}
