private async static void digikala_7_AddProductToMongo()
{
for (short i = 0; i < 120; i++)
{
List <DigikalaPageBaseDTO> getAll = new List <DigikalaPageBaseDTO>();
List <BasePage> pages = new List <BasePage>();
using (IWebsiteCrawler digikala = new DigikalaHelper())
{
getAll = (await digikala.GetAllBasePage <GetAllBasePageDigikalaResult>()).BasePages;
Console.WriteLine($"list total {getAll.Count()}");
pages = getAll.Select(x => new BasePage
{
Id = x._id,
Loc = x.Loc
}).ToList();
getAll.Clear();
}
long x = 0;
short errorCount = 0;
foreach (var item in pages)
{
try
{
var _s = DateTime.Now;
DigikalaProductDTO product = null;
using (IWebsiteCrawler digikala = new DigikalaHelper())
{
product = await digikala.GetProduct <DigikalaProductDTO>(item.Loc);
}
if (product == null)
{
int dkp = getDKPWithUrl(item.Loc);
Console.WriteLine($"Try Again , DKP - {dkp} Wait: {1000} Secs");
System.Threading.Thread.Sleep(1000 * errorCount);
using (IWebsiteCrawler digikala = new DigikalaHelper())
{
product = await digikala.GetProduct <DigikalaProductDTO>(item.Loc);
}
}
var _t = Math.Round((DateTime.Now - _s).TotalSeconds, 2);
if (product != null)
{
ProductTemp prd = new ProductTemp();
prd.BasePage = item;
prd.DigikalaProduct = product;
digikala_SaveProductBatch(prd);
Console.WriteLine($"S{i}, {++x} = DKP-{product.DKP} , Comment={(product.Comments != null ? product.Comments.Count() + "+ " : "0 ")} , in {_t} Secs ");
if (x % 5 == 0)
{
Console.WriteLine("--------------");
System.Threading.Thread.Sleep(100);
}
if (x % 100 == 0)
{
System.Threading.Thread.Sleep(500);
Console.Clear();
}
errorCount = 0;
}
else
{
if (errorCount < 3)
{
errorCount += 1;
}
int dkp = getDKPWithUrl(item.Loc);
Console.WriteLine($"{++x} = DKP-{dkp} , Wait: {1000 * errorCount} Secs , *** Error *** ,");
System.Threading.Thread.Sleep(1000 * errorCount);
}
}
catch (Exception ex)
{
if (errorCount < 3)
{
errorCount += 1;
}
int dkp = getDKPWithUrl(item.Loc);
Console.WriteLine($"{++x} = DKP-{dkp} , Wait: {1000 * errorCount} Secs , *** Error *** Problem");
using (ILoger Logger = new MongoDBLoggerHelper())
{
LogDTO log = new LogDTO()
{
_id = ObjectId.GenerateNewId().ToString(),
DateTime = DateTime.Now,
Description = ex.Message.ToString(),
ProjectId = (int)ProjectNames.Console,
Url = item.Loc,
MethodName = "Digikala - Console App",
Title = "Get Product Error - " + dkp
};
Logger.AddLog(log);
}
System.Threading.Thread.Sleep(1000 * errorCount);
}
}
}
digikala_SaveProductBatch(null, true);
}
