public static ThreadWatcher Create(ThreadWatcherConfig config)
{
SiteHelper siteHelper = SiteHelper.CreateByUrl(config.PageUrl);
return(new ThreadWatcher(siteHelper, config));
}
private void Check()
{
try {
lock (_settingsSync) {
_nextCheckWorkItem = null;
_checkFinishedEvent.Reset();
_isWaiting = false;
if (!_hasInitialized)
{
_pageList = new List <PageInfo> {
new PageInfo(PageUrl)
};
_imageDiskFileNames = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
_completedImages = new Dictionary <string, DownloadInfo>(StringComparer.OrdinalIgnoreCase);
_completedThumbs = new Dictionary <string, DownloadInfo>(StringComparer.OrdinalIgnoreCase);
CreateDirectory(_threadDownloadDirectory);
_hasInitialized = true;
}
}
string threadDir = ThreadDownloadDirectory;
string imageDir = ImageDownloadDirectory;
string thumbDir = ThumbnailDownloadDirectory;
Queue <ImageInfo> pendingImages = new Queue <ImageInfo>();
Queue <ThumbnailInfo> pendingThumbs = new Queue <ThumbnailInfo>();
if (imageDir != _previousImageDir)
{
// If the image directory changed, recalculate the maximum filename length. This
// affects the generated filenames, so rescan the files as well.
_imageFileNameLengthLimit = null;
_previousImageDir = imageDir;
_imageDiskFileNames.Clear();
_completedImages.Clear();
_completedThumbs.Clear();
}
foreach (PageInfo pageInfo in _pageList)
{
// Reset the fresh flag on all of the pages before downloading starts so that
// they're valid even if stopping before all the pages have been downloaded
pageInfo.IsFresh = false;
}
int pageIndex = 0;
bool anyPageSkipped = false;
OnDownloadStatus(new DownloadStatusEventArgs(DownloadType.Page, 0, _pageList.Count));
while (pageIndex < _pageList.Count && !IsStopping)
{
string saveFileName = PageBaseFileName + (pageIndex == 0 ? "" : $"_{pageIndex + 1}") + ".html";
HtmlParser pageParser = null;
PageInfo pageInfo = _pageList[pageIndex];
pageInfo.Path = Path.Combine(threadDir, saveFileName);
ManualResetEvent downloadEndEvent = new ManualResetEvent(false);
void DownloadEndCallback(DownloadResult result, string content, DateTime?lastModifiedTime, Encoding encoding)
{
if (result == DownloadResult.Completed)
{
pageInfo.IsFresh = true;
pageParser = new HtmlParser(content);
pageInfo.CacheTime = lastModifiedTime;
pageInfo.Encoding = encoding;
pageInfo.ReplaceList = Settings.SaveThumbnails ? new List <ReplaceInfo>() : null;
}
downloadEndEvent.Set();
}
DownloadPageAsync(pageInfo.Path, pageInfo.Url, PageAuth, _anyPendingRetries ? null : pageInfo.CacheTime, DownloadEndCallback);
downloadEndEvent.WaitOne();
downloadEndEvent.Close();
if (pageParser == null)
{
anyPageSkipped = true;
}
else
{
SiteHelper.SetParameters(this, pageParser);
GetFilesResult getFilesResult = SiteHelper.GetFiles(pageInfo.ReplaceList);
if (_completedImages.Count == 0)
{
if (getFilesResult.Images.Count != 0)
{
CreateDirectory(imageDir);
_imageFileNameLengthLimit ??= GetFileNameLengthLimit(imageDir);
}
if (getFilesResult.Thumbnails.Count != 0)
{
CreateDirectory(thumbDir);
}
foreach (ImageInfo image in getFilesResult.Images)
{
string fileName = GetUniqueFileName(image.GetEffectiveFileName(UseOriginalFileNames, _imageFileNameLengthLimit.Value), _imageDiskFileNames, true);
string path = Path.Combine(imageDir, fileName);
if (File.Exists(path))
{
_imageDiskFileNames.Add(fileName);
_completedImages[image.FileName] = new DownloadInfo {
FileName = fileName,
Skipped = false
};
}
}
foreach (ThumbnailInfo thumb in getFilesResult.Thumbnails)
{
string path = Path.Combine(thumbDir, thumb.FileName);
if (File.Exists(path))
{
_completedThumbs[thumb.FileName] = new DownloadInfo {
FileName = thumb.FileName,
Skipped = false
};
}
}
}
HashSet <string> pendingImageFileNames = new HashSet <string>(_completedImages.Comparer);
foreach (ImageInfo image in getFilesResult.Images)
{
if (_completedImages.ContainsKey(image.FileName))
{
continue;
}
if (!pendingImageFileNames.Add(image.FileName))
{
continue;
}
pendingImages.Enqueue(image);
}
HashSet <string> pendingThumbFileNames = new HashSet <string>(_completedThumbs.Comparer);
foreach (ThumbnailInfo thumb in getFilesResult.Thumbnails)
{
if (_completedThumbs.ContainsKey(thumb.FileName))
{
continue;
}
if (!pendingThumbFileNames.Add(thumb.FileName))
{
continue;
}
pendingThumbs.Enqueue(thumb);
}
string nextPageUrl = SiteHelper.GetNextPageUrl();
if (!String.IsNullOrEmpty(nextPageUrl))
{
PageInfo nextPageInfo = new PageInfo(nextPageUrl);
if (pageIndex == _pageList.Count - 1)
{
_pageList.Add(nextPageInfo);
}
else if (_pageList[pageIndex + 1].Url != nextPageUrl)
{
_pageList[pageIndex + 1] = nextPageInfo;
}
}
else if (pageIndex < _pageList.Count - 1)
{
_pageList.RemoveRange(pageIndex + 1, _pageList.Count - (pageIndex + 1));
}
}
pageIndex++;
OnDownloadStatus(new DownloadStatusEventArgs(DownloadType.Page, pageIndex, _pageList.Count));
}
if (!anyPageSkipped)
{
_anyPendingRetries = false;
}
MillisecondsUntilNextCheck = CheckIntervalSeconds * 1000;
if (pendingImages.Count != 0 && !IsStopping)
{
LastImageOn = DateTime.UtcNow;
OnFoundNewImage(EventArgs.Empty);
List <ManualResetEvent> downloadEndEvents = new List <ManualResetEvent>();
int completedImageCount = 0;
foreach (KeyValuePair <string, DownloadInfo> item in _completedImages)
{
if (!item.Value.Skipped)
{
completedImageCount++;
}
}
int totalImageCount = completedImageCount + pendingImages.Count;
OnDownloadStatus(new DownloadStatusEventArgs(DownloadType.Image, completedImageCount, totalImageCount));
while (pendingImages.Count != 0 && !IsStopping)
{
ImageInfo image = pendingImages.Dequeue();
string fileName = GetUniqueFileName(image.GetEffectiveFileName(UseOriginalFileNames, _imageFileNameLengthLimit.Value), _imageDiskFileNames);
string path = Path.Combine(imageDir, fileName);
HashType hashType = Settings.VerifyImageHashes ? image.HashType : HashType.None;
ManualResetEvent downloadEndEvent = new ManualResetEvent(false);
void DownloadEndCallback(DownloadResult result)
{
if (result == DownloadResult.Completed || result == DownloadResult.Skipped)
{
lock (_completedImages) {
_completedImages[image.FileName] = new DownloadInfo {
FileName = fileName,
Skipped = (result == DownloadResult.Skipped)
};
if (result == DownloadResult.Completed)
{
completedImageCount++;
}
else if (result == DownloadResult.Skipped)
{
totalImageCount--;
}
OnDownloadStatus(new DownloadStatusEventArgs(DownloadType.Image, completedImageCount, totalImageCount));
}
}
if (result == DownloadResult.Skipped || result == DownloadResult.RetryLater)
{
lock (_imageDiskFileNames) {
_imageDiskFileNames.Remove(fileName);
if (result == DownloadResult.RetryLater)
{
_anyPendingRetries = true;
}
}
}
downloadEndEvent.Set();
}
downloadEndEvents.Add(downloadEndEvent);
DownloadFileAsync(path, image.Url, ImageAuth, image.Referer, hashType, image.Hash, DownloadEndCallback);
}
foreach (ManualResetEvent downloadEndEvent in downloadEndEvents)
{
downloadEndEvent.WaitOne();
downloadEndEvent.Close();
}
}
if (Settings.SaveThumbnails)
{
if (pendingThumbs.Count != 0 && !IsStopping)
{
List <ManualResetEvent> downloadEndEvents = new List <ManualResetEvent>();
int completedThumbCount = 0;
foreach (KeyValuePair <string, DownloadInfo> item in _completedThumbs)
{
if (!item.Value.Skipped)
{
completedThumbCount++;
}
}
int totalThumbCount = completedThumbCount + pendingThumbs.Count;
OnDownloadStatus(new DownloadStatusEventArgs(DownloadType.Thumbnail, completedThumbCount, totalThumbCount));
while (pendingThumbs.Count != 0 && !IsStopping)
{
ThumbnailInfo thumb = pendingThumbs.Dequeue();
string path = Path.Combine(thumbDir, thumb.FileName);
ManualResetEvent downloadEndEvent = new ManualResetEvent(false);
void DownloadEndCallback(DownloadResult result)
{
if (result == DownloadResult.Completed || result == DownloadResult.Skipped)
{
lock (_completedThumbs) {
_completedThumbs[thumb.FileName] = new DownloadInfo {
FileName = thumb.FileName,
Skipped = (result == DownloadResult.Skipped)
};
if (result == DownloadResult.Completed)
{
completedThumbCount++;
}
else if (result == DownloadResult.Skipped)
{
totalThumbCount--;
}
OnDownloadStatus(new DownloadStatusEventArgs(DownloadType.Thumbnail, completedThumbCount, totalThumbCount));
}
}
downloadEndEvent.Set();
}
downloadEndEvents.Add(downloadEndEvent);
DownloadFileAsync(path, thumb.Url, PageAuth, thumb.Referer, HashType.None, null, DownloadEndCallback);
}
foreach (ManualResetEvent downloadEndEvent in downloadEndEvents)
{
downloadEndEvent.WaitOne();
downloadEndEvent.Close();
}
}
if (!IsStopping || StopReason != StopReason.IOError)
{
foreach (PageInfo pageInfo in _pageList)
{
if (!pageInfo.IsFresh)
{
continue;
}
HtmlParser htmlParser = new HtmlParser(File.ReadAllText(pageInfo.Path, pageInfo.Encoding));
for (int i = 0; i < pageInfo.ReplaceList.Count; i++)
{
ReplaceInfo replace = pageInfo.ReplaceList[i];
DownloadInfo downloadInfo = null;
string GetRelativeDownloadPath(string fileDownloadDir) =>
General.GetRelativeFilePath(Path.Combine(fileDownloadDir, downloadInfo.FileName), threadDir).Replace(Path.DirectorySeparatorChar, '/');
if (replace.Type == ReplaceType.ImageLinkHref && _completedImages.TryGetValue(replace.Tag, out downloadInfo))
{
replace.Value = "href=\"" + HttpUtility.HtmlAttributeEncode(GetRelativeDownloadPath(imageDir)) + "\"";
}
if (replace.Type == ReplaceType.ImageSrc && _completedThumbs.TryGetValue(replace.Tag, out downloadInfo))
{
replace.Value = "src=\"" + HttpUtility.HtmlAttributeEncode(GetRelativeDownloadPath(thumbDir)) + "\"";
}
}
General.AddOtherReplaces(htmlParser, pageInfo.Url, pageInfo.ReplaceList);
using (StreamWriter sw = new StreamWriter(pageInfo.Path, false, pageInfo.Encoding)) {
General.WriteReplacedString(htmlParser.PreprocessedHtml, pageInfo.ReplaceList, sw);
}
if (htmlParser.FindEndTag("html") != null && File.Exists(pageInfo.Path + ".bak"))
{
try { File.Delete(pageInfo.Path + ".bak"); }
catch { }
}
}
}
}
if (OneTimeDownload)
{
Stop(StopReason.DownloadComplete);
}
}
catch {
Stop(StopReason.Other);
}
if (IsThreadDownloadDirectoryPendingRename)
{
TryRenameThreadDownloadDirectory(true);
}
lock (_settingsSync) {
_checkFinishedEvent.Set();
if (!IsStopping)
{
_nextCheckWorkItem = _workScheduler.AddItem(NextCheckTicks, Check, PageHost);
_isWaiting = MillisecondsUntilNextCheck > 0;
}
}
if (IsStopping)
{
OnStopStatus(EventArgs.Empty);
}
else if (IsWaiting)
{
OnWaitStatus(EventArgs.Empty);
}
}