private static BoligaDBEntities AddToContext(BoligaDBEntities context, BoligaProperty entity, int count, int commitCount, bool recreateContext) { context.Set <BoligaProperty>().Add(entity); if (count % commitCount == 0) { context.SaveChanges(); if (recreateContext) { context.Dispose(); context = new BoligaDBEntities(); context.Configuration.AutoDetectChangesEnabled = false; } } return(context); }
private static async Task <bool> GetSearchPage(int pageNumber) { using (var client = new HttpClient(new RetryHandler(new HttpClientHandler()))) { var uri = new Uri($"{_startUrl}?page={pageNumber}"); var response = await client.GetAsync(uri); var responseString = await response.Content.ReadAsStringAsync(); HtmlDocument htmlDoc = new HtmlDocument { OptionFixNestedTags = true }; htmlDoc.LoadHtml(responseString); List <int> newPageNumbers = GetNewSearchPageNumbers(htmlDoc); if (newPageNumbers == null) { return(true); } else { ConcurrentDictionary <string, BoligaProperty> properties = new ConcurrentDictionary <string, BoligaProperty>(); Parallel.ForEach(GetProrertyUrls(htmlDoc), url => { try { properties.TryAdd(url, GetPropertyData(url)); } catch (Exception) { // ignored } }); htmlDoc = null; GC.Collect(); using (TransactionScope scope = new TransactionScope()) { BoligaDBEntities context = null; try { context = new BoligaDBEntities(); context.Configuration.AutoDetectChangesEnabled = false; int count = 0; foreach (var p in properties) { ++count; context = AddToContext(context, p.Value, count, 100, true); _counter += 1; _pbar.Tick("Properties processed " + _counter); } context.SaveChanges(); } finally { context?.Dispose(); } scope.Complete(); } properties.Clear(); foreach (int pn in newPageNumbers) { await GetSearchPage(pn); } } } return(true); }