private async void GetProductListAsynchronously(List <Product> productList)
{
var scrapper = new OLXScrapper();
var request = new ProductRequest {
Keyword = "3ds"
};
object threadLock = new object();
var taskList = new List <Task <ProductResponse> >();
for (var ctr = 1; ctr <= 3; ++ctr)
{
lock (threadLock)
{
request.Page += 1;
taskList.Add(scrapper.GetProductListAsync(request));
}
}
var result = await Task.WhenAll(taskList);
foreach (var productResponse in result)
{
productList.AddRange(productResponse.ProductList);
}
}
public void TestLoadingProductListFromFile()
{
var scrapper = new OLXScrapper();
var products = scrapper.LoadProductListFromFile("c:/olx.txt");
Assert.IsTrue(products != null && products.AvailablePages > 0);
}
private async void GetProductListAsync(ProductResponse response)
{
var scrapper = new OLXScrapper();
var request = new ProductRequest {
Keyword = "ps4"
};
response = await scrapper.GetProductListAsync(request);
}
public void TestGettingProductList()
{
var scrapper = new OLXScrapper();
var request = new ProductRequest {
Keyword = "ps4"
};
var products = scrapper.GetProductList(request);
Assert.IsTrue(products != null && products.ProductList.Any());
}
public async Task <IEnumerable <ProductResponse> > GetProductList(IEnumerable <ProductRequest> requests)
{
try
{
if (!requests.Any())
{
return(null);
}
var threadLock = new object();
var list = new List <ProductResponse>();
var taskList = new List <Task <ProductResponse> >();
var carousellScrapper = new CarousellScrapper();
var olxScrapper = new OLXScrapper();
foreach (var request in requests)
{
if (request.Source == ProductSource.Carousell)
{
taskList.Add(Task.Run(() =>
{
var result = String.IsNullOrEmpty(request.NextPage)
? carousellScrapper.GetProductList(request)
: carousellScrapper.GetProductList(request.NextPage);
lock (threadLock)
{
list.Add(result);
}
return(result);
}));
}
if (request.Source == ProductSource.Olx)
{
var olxRequest = new ProductRequest {
Keyword = request.Keyword, Page = request.Page
};
taskList.Add(Task.Run(() =>
{
var result = olxScrapper.GetProductList(olxRequest);
lock (threadLock)
{
list.Add(result);
}
return(result);
}));
}
}
Task.WaitAll(taskList.ToArray());
return(list);
}
catch (AggregateException arrException)
{
throw arrException;
}
catch (Exception ex)
{
throw ex;
}
}
