internal HtmlDocument DownloadDocument(Serie a_serie, string a_url = null)
{
return(DownloadDocument(
(a_url == null) ? a_serie.URL : a_url,
() => a_serie.State = SerieState.Downloading,
() => Limiter.Aquire(a_serie),
() => Limiter.Release(a_serie)));
}
internal HtmlDocument DownloadDocument(Page a_page, string a_url = null)
{
return(DownloadDocument(
(a_url == null) ? a_page.URL : a_url,
() => a_page.State = PageState.Downloading,
() => Limiter.Aquire(a_page),
() => Limiter.Release(a_page),
a_page.Chapter.Token));
}
internal HtmlDocument DownloadDocument(Chapter a_chapter, string a_url = null)
{
return(DownloadDocument(
(a_url == null) ? a_chapter.URL : a_url,
() => a_chapter.State = ChapterState.DownloadingPagesList,
() => Limiter.Aquire(a_chapter),
() => Limiter.Release(a_chapter),
a_chapter.Token));
}
internal virtual MemoryStream GetImageStream(Page a_page)
{
return(DownloadWithRetry(() =>
{
try
{
Limiter.Aquire(a_page);
HttpWebRequest myReq = (HttpWebRequest)WebRequest.Create(
Uri.EscapeUriString(a_page.ImageURL));
myReq.UserAgent = DownloadManager.Instance.MangaSettings.UserAgent;
myReq.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
myReq.Referer = Uri.EscapeUriString(a_page.URL);
byte[] buffer = new byte[4 * 1024];
MemoryStream mem_stream = new MemoryStream();
using (Stream image_stream = myReq.GetResponse().GetResponseStream())
{
for (;;)
{
int readed = image_stream.Read(buffer, 0, buffer.Length);
if (readed == 0)
{
break;
}
a_page.Chapter.Token.ThrowIfCancellationRequested();
mem_stream.Write(buffer, 0, readed);
}
}
Thread.Sleep(DownloadManager.Instance.MangaSettings.SleepAfterEachDownloadMS);
mem_stream.Position = 0;
return mem_stream;
}
finally
{
Limiter.Release(a_page);
}
}));
}
internal void DownloadPagesAndImages()
{
try
{
Limiter.BeginChapter(this);
try
{
DownloadPagesList();
var names = Pages.Select(p => p.Name);
var sorted_names = Pages.Select(p => p.Name).OrderBy(n => n, new NaturalOrderStringComparer());
bool error = false;
PageNamingStrategy pns = DownloadManager.Instance.MangaSettings.PageNamingStrategy;
if (pns == PageNamingStrategy.IndexToPreserveOrder)
{
if (!names.SequenceEqual(sorted_names))
{
pns = PageNamingStrategy.AlwaysUseIndex;
}
}
else if (pns == PageNamingStrategy.PrefixToPreserverOrder)
{
if (!names.SequenceEqual(sorted_names))
{
pns = PageNamingStrategy.AlwaysUsePrefix;
}
}
for (int i = 0; i < Pages.Count; i++)
{
Pages[i].LimiterOrder = Catalog.NextID();
Debug.Assert(Pages[i].Index == i + 1);
}
Parallel.ForEach(new SequentialPartitioner <Page>(Pages),
new ParallelOptions()
{
MaxDegreeOfParallelism = Crawler.MaxConnectionsPerServer
},
(page, state) =>
{
try
{
page.DownloadAndSavePageImage(pns);
Catalog.Save(this);
}
catch (OperationCanceledException)
{
state.Break();
}
catch (Exception ex2)
{
Loggers.MangaCrawler.Error(String.Format(
"Exception #1, chapter: {0} state: {1}",
this, State), ex2);
error = true;
}
}
);
Token.ThrowIfCancellationRequested();
if (PagesDownloaded != Pages.Count)
{
State = ChapterState.Error;
}
else if (Pages.Any(p => p.State != PageState.Downloaded))
{
State = ChapterState.Error;
}
else if (error)
{
State = ChapterState.Error;
}
Catalog.Save(this);
if (DownloadManager.Instance.MangaSettings.UseCBZ)
{
if (State != ChapterState.Error)
{
CreateCBZ();
}
}
Visited = true;
}
finally
{
Limiter.EndChapter(this);
}
}
catch (OperationCanceledException)
{
Debug.Assert(State == ChapterState.Cancelling);
Debug.Assert(m_cancellation_token_source.IsCancellationRequested);
State = ChapterState.Cancelled;
}
catch (Exception ex1)
{
Loggers.MangaCrawler.Error(String.Format(
"Exception #2, chapter: {0} state: {1}", this, State), ex1);
State = ChapterState.Error;
try
{
DownloadManager.Instance.DownloadChapters(Serie, true);
}
catch (Exception ex2)
{
Loggers.MangaCrawler.Error(String.Format(
"Exception #3, chapter: {0} state: {1}", this, State), ex2);
}
}
finally
{
lock (m_state_lock)
{
if ((State != ChapterState.Error) && (State != ChapterState.Cancelled))
{
Debug.Assert(
(State == ChapterState.DownloadingPages) ||
(State == ChapterState.Zipping));
State = ChapterState.Downloaded;
}
}
}
Catalog.Save(this);
}
