static void Main(string[] args)
{
for (char c = 'a'; c <= 'z'; ++c)
{
alph.Add(c);
}
alph.AddRange(Enumerable.Range(1, 9).Select(x => x.ToString()[0]).ToArray());
HtmlReaderManager hrm = new HtmlReaderManager();
LimitedConcurrencyLevelTaskScheduler lcts =
new LimitedConcurrencyLevelTaskScheduler(200);
TaskFactory factory = new TaskFactory(lcts);
var tasks = new List<Task>();
int counter = 1;
foreach (string url in GetNextUrl())
{
Options opt = new Options();
opt.Url = url;
opt.counter = counter;
tasks.Add(factory.StartNew((opts) =>
{
bool isDownloaded = false;
int tryCount = 0;
do
{
Options options = (Options) opts;
string targetUrl = "http://prntscr.com/1" + options.Url;
try
{
hrm.Get(targetUrl);
string html = hrm.Html;
HtmlAgilityPack.HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(html);
string imgUrl = doc.DocumentNode.SelectSingleNode("//meta[@name='twitter:image:src']").Attributes["content"].Value;
if (!string.IsNullOrEmpty(imgUrl))
{
ImageDownloader.DownloadRemoteImageFile(imgUrl, "images/" + Guid.NewGuid() + ".jpeg");
Console.WriteLine(options.counter + ". Скачиваем " + imgUrl);
}
else
Console.WriteLine(options.counter + ". " + imgUrl + " пуст");
break;
}
catch (Exception ex)
{
tryCount++;
if (ex.ToString().StartsWith("System.Net.WebException: The remote server returned an error: (503) Server Unavailable.") && tryCount<20)
{
Random s = new Random();
int randomVal = s.Next(1000, 10000);
Thread.Sleep(randomVal);
Console.WriteLine(options.counter + ". ждём " + targetUrl +" ("+tryCount+" раз)");
}
else
{
Console.WriteLine(options.counter + ". ошибка " + targetUrl);
isDownloaded = true;
}
}
} while (!isDownloaded);
}, opt, TaskCreationOptions.LongRunning));
if (counter%200 == 0)
Task.WaitAll(tasks.ToArray());
tasks.Clear();
counter++;
}
Console.ReadLine();
}
static void Main(string[] args)
{
for (char c = 'a'; c <= 'z'; ++c)
{
alph.Add(c);
}
alph.AddRange(Enumerable.Range(1, 9).Select(x => x.ToString()[0]).ToArray());
HtmlReaderManager hrm = new HtmlReaderManager();
LimitedConcurrencyLevelTaskScheduler lcts =
new LimitedConcurrencyLevelTaskScheduler(200);
TaskFactory factory = new TaskFactory(lcts);
var tasks = new List <Task>();
int counter = 1;
foreach (string url in GetNextUrl())
{
Options opt = new Options();
opt.Url = url;
opt.counter = counter;
tasks.Add(factory.StartNew((opts) =>
{
bool isDownloaded = false;
int tryCount = 0;
do
{
Options options = (Options)opts;
string targetUrl = "http://prntscr.com/1" + options.Url;
try
{
hrm.Get(targetUrl);
string html = hrm.Html;
HtmlAgilityPack.HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(html);
string imgUrl = doc.DocumentNode.SelectSingleNode("//meta[@name='twitter:image:src']").Attributes["content"].Value;
if (!string.IsNullOrEmpty(imgUrl))
{
ImageDownloader.DownloadRemoteImageFile(imgUrl, "images/" + Guid.NewGuid() + ".jpeg");
Console.WriteLine(options.counter + ". Скачиваем " + imgUrl);
}
else
{
Console.WriteLine(options.counter + ". " + imgUrl + " пуст");
}
break;
}
catch (Exception ex)
{
tryCount++;
if (ex.ToString().StartsWith("System.Net.WebException: The remote server returned an error: (503) Server Unavailable.") && tryCount < 20)
{
Random s = new Random();
int randomVal = s.Next(1000, 10000);
Thread.Sleep(randomVal);
Console.WriteLine(options.counter + ". ждём " + targetUrl + " (" + tryCount + " раз)");
}
else
{
Console.WriteLine(options.counter + ". ошибка " + targetUrl);
isDownloaded = true;
}
}
} while (!isDownloaded);
}, opt, TaskCreationOptions.LongRunning));
if (counter % 200 == 0)
{
Task.WaitAll(tasks.ToArray());
}
tasks.Clear();
counter++;
}
Console.ReadLine();
}