public static void Start() { if (RunSettings.DocumentNumber == -1) { var files = FileTreeTools.GetFileLinkRelations(RunSettings.DatasetPath); RunSettings.DocumentNumber = files.Count; } var parallelOptions = new ParallelOptions() { MaxDegreeOfParallelism = RunSettings.MaxDegreeOfParallelism }; Parallel.ForEach(Master.GetGroupedWordsBatch(), parallelOptions, Slave.BuildReducedReverseIndex); Console.WriteLine($"Building a database index on {DbClient.ReducedReverseIndexCollectionName} collection"); CreateDbIndex(); }
public static void InitializeSettings() { DbClient.ClearDatabase(); var projectDir = Environment.CurrentDirectory.Contains("Debug\\netcoreapp") ? Path.GetFullPath(Path.Combine(Environment.CurrentDirectory, @"..\..\..\")) : Environment.CurrentDirectory; var siteDir = Path.Combine(projectDir, "riwdataset"); RunSettings.DatasetPath = siteDir; RunSettings.LogType = LogType.Debug; RunSettings.DocumentBatchSize = 5; RunSettings.GroupedWordsBatchSize = 300; //used for calculating idf RunSettings.FilePaths = FileTreeTools.GetFileLinkRelations(RunSettings.DatasetPath); }