public static void ExtartFilesDataIdioms() { //var fileJson = File.ReadAllText(AppDomain.CurrentDomain.BaseDirectory + "\\Titles.json"); var titleList = new ResourceService().GetAllIsPrimary(); var files = Directory.GetFiles(_path); var taskList = new List <Task>(); foreach (var file in files) { taskList.Add(Task.Factory.StartNew(() => { var fi = new FileInfo(file); var id = int.Parse(fi.Name.Split('.').FirstOrDefault()); var currentWord = titleList.FirstOrDefault(x => x.ID == id); if (currentWord != null) { var oxfordWords = new List <OxfordWord>(); var doc = new HtmlDocument(); doc.Load(file, Encoding.UTF8); oxfordWords.AddRange(GetWords("Phrasal verbs", id, 2, doc)); oxfordWords.AddRange(GetWords("Idioms", id, 3, doc)); var other = GetWords("All matches", id, 4, doc); var newOther = other.Where(x => !oxfordWords.Select(c => c.Word).Contains(x.Word)).ToList(); oxfordWords.AddRange(newOther); if (oxfordWords.Count > 0) { Console.WriteLine($"{currentWord.Word}\t{oxfordWords.Count}"); //\t{ oxfordWords.Select(x => x.Word).Aggregate((x, y) => x + "," + y)} //"); using (var service = new ResourceService()) service.BatchInsertOxfordWord(oxfordWords.Where(x => x.Word != currentWord.Word).ToList()); } } })); if (taskList.Count > 10) { Task.WaitAll(taskList.ToArray()); taskList.Clear(); } } Task.WaitAll(taskList.ToArray()); taskList.Clear(); // Console.WriteLine( // $"{titleList.Count(x => !string.IsNullOrEmpty(x.ProductionYear) && !string.IsNullOrEmpty(x.AgeRating))}"); // File.WriteAllText(AppDomain.CurrentDomain.BaseDirectory + "\\Titles.json", JsonConvert.SerializeObject(titleList), Encoding.UTF8); }
public static void ExtartFilesDataAutoComplete() { var titleList = new ResourceService().GetAllIsPrimary(); var files = Directory.GetFiles(_path); var taskList = new List <Task>(); foreach (var file in files) { taskList.Add(Task.Factory.StartNew(() => { var fi = new FileInfo(file); var id = int.Parse(fi.Name.Split('.').FirstOrDefault()); var word = titleList.FirstOrDefault(x => x.ID == id); var content = File.ReadAllText(file); var list = JsonConvert.DeserializeObject <AutoCompleteModel>(content); if (list.results != null && list.results.Count > 0) { var words = new List <OxfordWord>(); Console.WriteLine( $"Word Id: {word.ID}\t Family: {list.results.Select(x => x.searchtext).Aggregate((x, y) => x + "," + y)}"); words.AddRange(list.results.Select(x => new OxfordWord { WordId = id, Type = 1, Word = x.searchtext }).ToList()); using (var service = new ResourceService()) service.BatchInsertOxfordWord(words); } })); if (taskList.Count > 10) { Task.WaitAll(taskList.ToArray()); taskList.Clear(); } } Task.WaitAll(taskList.ToArray()); taskList.Clear(); // Console.WriteLine( // $"{titleList.Count(x => !string.IsNullOrEmpty(x.ProductionYear) && !string.IsNullOrEmpty(x.AgeRating))}"); // File.WriteAllText(AppDomain.CurrentDomain.BaseDirectory + "\\Titles.json", JsonConvert.SerializeObject(titleList), Encoding.UTF8); }