private static BoligaDBEntities AddToContext(BoligaDBEntities context, BoligaProperty entity, int count, int commitCount, bool recreateContext)
{
context.Set <BoligaProperty>().Add(entity);
if (count % commitCount == 0)
{
context.SaveChanges();
if (recreateContext)
{
context.Dispose();
context = new BoligaDBEntities();
context.Configuration.AutoDetectChangesEnabled = false;
}
}
return(context);
}
private static async Task <bool> GetSearchPage(int pageNumber)
{
using (var client = new HttpClient(new RetryHandler(new HttpClientHandler())))
{
var uri = new Uri($"{_startUrl}?page={pageNumber}");
var response = await client.GetAsync(uri);
var responseString = await response.Content.ReadAsStringAsync();
HtmlDocument htmlDoc = new HtmlDocument {
OptionFixNestedTags = true
};
htmlDoc.LoadHtml(responseString);
List <int> newPageNumbers = GetNewSearchPageNumbers(htmlDoc);
if (newPageNumbers == null)
{
return(true);
}
else
{
ConcurrentDictionary <string, BoligaProperty> properties = new ConcurrentDictionary <string, BoligaProperty>();
Parallel.ForEach(GetProrertyUrls(htmlDoc), url =>
{
try
{
properties.TryAdd(url, GetPropertyData(url));
}
catch (Exception)
{
// ignored
}
});
htmlDoc = null;
GC.Collect();
using (TransactionScope scope = new TransactionScope())
{
BoligaDBEntities context = null;
try
{
context = new BoligaDBEntities();
context.Configuration.AutoDetectChangesEnabled = false;
int count = 0;
foreach (var p in properties)
{
++count;
context = AddToContext(context, p.Value, count, 100, true);
_counter += 1;
_pbar.Tick("Properties processed " + _counter);
}
context.SaveChanges();
}
finally
{
context?.Dispose();
}
scope.Complete();
}
properties.Clear();
foreach (int pn in newPageNumbers)
{
await GetSearchPage(pn);
}
}
}
return(true);
}
