public void Download()
{
Chapter = SerieTestData.Serie.Chapters.ElementAtOrDefault(Index - 1);
PageCount = -1;
URL = Chapter.URL;
Chapter.State = ChapterState.Waiting;
Limiter.BeginChapter(Chapter);
try
{
Chapter.DownloadPagesList();
}
finally
{
Limiter.EndChapter(Chapter);
}
PageCount = Chapter.Pages.Count;
Title = Chapter.Title;
foreach (var page in Pages)
{
if (page.Index > PageCount)
{
page.Index = -1;
page.Name += " - index out of range";
continue;
}
page.Download();
}
}
public void Download()
{
Page = ChapterTestData.Chapter.Pages.ElementAtOrDefault(Index - 1);
URL = Page.URL;
Index = -1;
Limiter.BeginChapter(Page.Chapter);
try
{
var stream = Page.GetImageStream();
Image = System.Drawing.Image.FromStream(stream);
ImageURL = Page.ImageURL;
stream.Position = 0;
Hash = TomanuExtensions.Utils.Hash.CalculateSHA256(stream);
}
finally
{
Limiter.EndChapter(Page.Chapter);
}
if (Page.State == PageState.Error)
{
throw new Exception("Downloading page error");
}
Index = Page.Index;
Name = Page.Name;
}
public void _RandomTestAll()
{
Dictionary <Server, int> serie_chapters = new Dictionary <Server, int>();
Dictionary <Server, int> chapter_pageslist = new Dictionary <Server, int>();
Dictionary <Server, int> chapter_images = new Dictionary <Server, int>();
DateTime last_report = DateTime.Now;
TimeSpan report_delta = new TimeSpan(0, 15, 0);
int errors = 0;
m_pi = new ProgressIndicator("_RandomTestAll");
Object locker = new Object();
WriteLine("------------------------------------------------------------------------------");
DownloadManager.Instance.MangaSettings.MaximumConnectionsPerServer = 4;
DownloadManager.Instance.MangaSettings.SleepAfterEachDownloadMS = 0;
foreach (var server in DownloadManager.Instance.Servers)
{
serie_chapters[server] = 0;
chapter_pageslist[server] = 0;
chapter_images[server] = 0;
}
if (File.Exists(TestBase.GetTestFilePath(FILE_EXCEPTION)))
{
m_exceptions = new HashSet <string>(
File.ReadAllLines(TestBase.GetTestFilePath(FILE_EXCEPTION)));
WriteLine("Exceptions: {0}", m_exceptions.Count);
}
else
{
WriteLine("Exceptions: no file");
}
Action <bool> report = (force) =>
{
lock (locker)
{
if (!force)
{
if (DateTime.Now - last_report < report_delta)
{
return;
}
}
last_report = DateTime.Now;
}
WriteLine("");
WriteLine("Report ({0}):", DateTime.Now);
foreach (var server in DownloadManager.Instance.Servers)
{
WriteLine("Server: {0}, Series: {1}, Chapters: {2}, Pages and images: {3}",
server.Name, serie_chapters[server], chapter_pageslist[server], chapter_images[server]);
}
WriteLine("Errors: {0}", errors);
WriteLine("");
};
Parallel.ForEach(
DownloadManager.Instance.Servers,
new ParallelOptions()
{
MaxDegreeOfParallelism = DownloadManager.Instance.Servers.Count()
},
(server, state) =>
{
//if (!server.Name.Contains("Fox"))
// return;
for (; ;)
{
WriteLine("{0} - Downloading series", server);
server.State = ServerState.Waiting;
server.DownloadSeries();
if (server.State == ServerState.Error)
{
WriteLineError("ERROR - {0} {1} - Error while downloading series from server",
server.Name, server.URL);
continue;
}
else if (server.Series.Count == 0)
{
WriteLineError("ERROR - {0} {1} - Server have no series",
server.Name, server.URL);
continue;
}
else
{
WriteLine("{0} - Downloaded series", server);
break;
}
}
});
DownloadManager.Instance.MangaSettings.MaximumConnectionsPerServer = 1;
DownloadManager.Instance.MangaSettings.SleepAfterEachDownloadMS = 4000;
Parallel.ForEach(
DownloadManager.Instance.Servers,
new ParallelOptions()
{
MaxDegreeOfParallelism = DownloadManager.Instance.Servers.Count()
},
(server, state) =>
{
//if (!server.Name.Contains("Fox"))
// return;
Parallel.ForEach(
TakeRandom(server.Series, 0.3),
new ParallelOptions()
{
MaxDegreeOfParallelism = server.Crawler.MaxConnectionsPerServer
},
serie =>
{
WriteLine("{0} - Downloading chapters", serie);
serie.State = SerieState.Waiting;
serie.DownloadChapters();
serie_chapters[server]++;
if (serie.State == SerieState.Error)
{
WriteLineError("ERROR - {0} {1} - Error while downloading chapters from serie",
serie, serie.URL);
errors++;
}
else
{
WriteLine("{0} - Downloaded chapters", serie);
}
Parallel.ForEach(TakeRandom(serie.Chapters, 0.1),
new ParallelOptions()
{
MaxDegreeOfParallelism = server.Crawler.MaxConnectionsPerServer
},
(chapter) =>
{
WriteLine("{0} - Downloading pages list", chapter);
try
{
chapter.State = ChapterState.Waiting;
Limiter.BeginChapter(chapter);
try
{
chapter.DownloadPagesList();
}
finally
{
Limiter.EndChapter(chapter);
}
chapter_pageslist[server]++;
WriteLine("{0} - Downloaded pages list", chapter);
}
catch
{
WriteLineError("ERROR - {0} {1} - Exception while downloading pages from chapter",
chapter, chapter.URL);
errors++;
}
Parallel.ForEach(TakeRandom(chapter.Pages, 0.1),
new ParallelOptions()
{
MaxDegreeOfParallelism = chapter.Crawler.MaxConnectionsPerServer
},
(page) =>
{
WriteLine("{0} - Downloading image", page);
Limiter.BeginChapter(chapter);
try
{
MemoryStream stream = null;
try
{
page.GetImageURL();
try
{
stream = page.GetImageStream();
if (stream.Length == 0)
{
WriteLineErrorForPage(
"ERROR - {0} {1} - Image stream is zero size for page",
page);
errors++;
}
else
{
try
{
System.Drawing.Image.FromStream(stream);
WriteLine("{0} - Downloaded image", page);
}
catch
{
WriteLineErrorForPage(
"ERROR - {0} {1} - Exception while creating image from stream for page",
page);
errors++;
}
}
}
catch
{
WriteLineErrorForPage(
"ERROR - {0} {1} - Exception while downloading image from page",
page);
errors++;
}
}
catch
{
WriteLineErrorForPage(
"ERROR - {0} {1} - Exception while detecting image url",
page);
errors++;
}
}
finally
{
Limiter.EndChapter(chapter);
}
chapter_images[server]++;
report(false);
});
});
});
});
}
