private void DownloadPage(ITargetBlock<Parse> targetAscTransform)
{
foreach (var q in _dataAccess.GetEmailQueries().FindAll(x => x.Email == true))
{
var cars = _dataAccess.GetHtmlToScrape(GenSqlStatementForHtmlParsing(q.Id));
var emailss = cars.Where(e => e.HtmlDownloaded == false);
foreach (var car in cars)
{
var parse = new Parse
{
CarId = car.Id,
Link = car.Link,
UserAgent = _dataAccess.GetRandomUserAgent()
};
targetAscTransform.Post(parse);
}
}
}
private void ProcessDataBuffer(Parse ascDataSet)
{
#region commented
//if (ascDataSet.CarId %3 == 0)
//{
// // Testing if this will delay all the other data processing
// var c = new CryptoRandom();
// var st = c.Next(45000, 90000);
// var tast = LoadAsync(ascDataSet.CarId, ascDataSet.Link, ascDataSet.UserAgent, CancellationToken.None);
// Thread.Sleep(st);
// Console.WriteLine("{0} divide by 3 slept from {1}", ascDataSet.Link, st);
//}
//else
//{
// if (ascDataSet.CarId %2 == 0)
// {
// var c = new CryptoRandom();
// var st = c.Next(45000, 90000);
// var tast = LoadAsync(ascDataSet.CarId, ascDataSet.Link, ascDataSet.UserAgent, CancellationToken.None);
// Thread.Sleep(st);
// Console.WriteLine("{0} even slept from {1}", ascDataSet.Link, st);
// }
// else
// {
// var c = new CryptoRandom();
// var st = c.Next(45000, 90000);
// var tast = LoadAsync(ascDataSet.CarId, ascDataSet.Link, ascDataSet.UserAgent, CancellationToken.None);
// Thread.Sleep(st);
// Console.WriteLine("{0} odd slept from {1}", ascDataSet.Link, st);
// }
//}
#endregion
var c = new CryptoRandom();
var st = c.Next(45000, 90000);
var tast = LoadAsync(ascDataSet.CarId, ascDataSet.Link, ascDataSet.UserAgent, CancellationToken.None);
Thread.Sleep(st);
Console.WriteLine("{0} slept from {1}", ascDataSet.Link, st);
}
// Demonstrates the consumption end of the producer and consumer pattern.
public async Task<Parse> AscTransConsumerAsync(ISourceBlock<Parse> source)
{
// counter to track the number of items that are processed
Int64 count = 0;
var parse = new Parse();
var actionBlock = new ActionBlock<Parse>(
data =>
{
ProcessDataBuffer(data);
// count has to be accessed in a thread-safe manner
// be careful about using Interlocked,
// for more complicated computations, locking might be more appropriate
Interlocked.Increment(ref count);
},
// some small constant might be better than Unbounded, depedning on circumstances
new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = 5}); //DataflowBlockOptions.Unbounded});
source.LinkTo(actionBlock, new DataflowLinkOptions {PropagateCompletion = true});
// this assumes source will be completed when done,
// you need to call ascbuffer.Complete() after AscBufferProducer() for this
await actionBlock.Completion;
return parse;
}
private static void ProcessDataBuffer(Parse ascDataSet)
{
var c = new CryptoRandom();
var st = c.Next(45000, 90000);
var tast = LoadAsync(ascDataSet.CarId, ascDataSet.Link, ascDataSet.UserAgent, CancellationToken.None);
Thread.Sleep(st);
Debug.WriteLine("{0} slept from {1}", ascDataSet.Link, st);
}