private void clientList_DownloadFileCompleted(object sender, AsyncCompletedEventArgs e)
{
tsProgressBar.Visible = false;
UpdateLog("clientList_DownloadFileCompleted", "Download Complete: " + e.UserState);
txtListFile.Text = saveFileDialog1.FileName;
gbxSearch.Enabled = true;
gbxList.Enabled = true;
btnGet.Enabled = true;
btnListCancel.Enabled = false;
if (chkAutoLoadList.Checked)
{
DanbooruPostDaoOption option = new DanbooruPostDaoOption()
{
Provider = _currProvider,
Url = txtListFile.Text,
Referer = _clientList.Referer,
Query = txtListFile.Text.Split('\\').Last(),
SearchTags = "",
BlacklistedTags = TagBlacklist,
BlacklistedTagsRegex = TagBlacklistRegex,
BlacklistedTagsUseRegex = chkBlacklistTagsUseRegex.Checked,
IgnoredTags = TagIgnore,
IgnoredTagsRegex = TagIgnoreRegex,
IgnoredTagsUseRegex = chkIgnoreTagsUseRegex.Checked
};
DanbooruPostDao newPosts = new DanbooruPostDao(option);
LoadList(newPosts);
}
_isLoadingList = false;
}
//int currFileRetry = 0;
private void backgroundWorker1_DoWork(object sender, DoWorkEventArgs e)
{
BackgroundWorker worker = sender as BackgroundWorker;
object[] param = (object[])e.Argument;
Stream ms = (Stream)param[0];
DanbooruPostDaoOption option = (DanbooruPostDaoOption)param[1];
DanbooruPostDao newPosts = new DanbooruPostDao(ms, option);
e.Result = newPosts;
}
private static void PostProcess(DanbooruPostDaoOption option, DanbooruPost post)
{
post.TagsEntity = Helper.ParseTags(post.Tags, option.Provider);
if (option.BlacklistedTagsUseRegex)
{
post.Hidden = IsBlacklisted(post, option.BlacklistedTagsRegex);
}
else
{
post.Hidden = IsBlacklisted(post, option.BlacklistedTags);
}
post.Query = option.Query;
post.SearchTags = option.SearchTags;
post.Provider = option.Provider;
post.CreatedAtDateTime = DanbooruPostDao.ParseDateTime(post.CreatedAt, option.Provider);
}
/// <summary>
/// Parse the post details after added to the download list or from batch job.
/// </summary>
/// <param name="post"></param>
/// <param name="postHtml"></param>
/// <param name="overideTagParsing"></param>
/// <returns></returns>
public static DanbooruPost ParsePost(DanbooruPost post, string postHtml, bool overideTagParsing)
{
try
{
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(postHtml);
string file_url = "";
string sample_url = "";
// reparse tags with type
if (overideTagParsing)
{
ReparseTags(post, doc);
}
// Flash Game or bmp
if (post.PreviewUrl.EndsWith("download-preview.png"))
{
var links = doc.DocumentNode.SelectNodes("//a");
foreach (var link in links)
{
// flash
if (link.InnerText == "Save this file (right click and save as)")
{
file_url = Helper.FixUrl(link.GetAttributeValue("href", ""), isHttps(post.Provider));
// http://cs.sankakucomplex.com/data/f6/23/f623ea7559ef39d96ebb0ca7530586b8.swf
post.MD5 = file_url.Substring(file_url.LastIndexOf("/") + 1);
post.MD5 = post.MD5.Substring(0, 32);
break;
}
// bmp
if (link.InnerText == "Download")
{
file_url = Helper.FixUrl(link.GetAttributeValue("href", ""), isHttps(post.Provider));
break;
}
}
}
else
{
var lowresElement = doc.DocumentNode.SelectSingleNode("//a[@id='lowres']");
if (lowresElement != null)
{
sample_url = Helper.FixUrl(lowresElement.GetAttributeValue("href", ""), isHttps(post.Provider));
}
var highresElement = doc.DocumentNode.SelectSingleNode("//a[@id='highres']");
if (highresElement != null)
{
file_url = Helper.FixUrl(highresElement.GetAttributeValue("href", ""), isHttps(post.Provider));
}
}
post.FileUrl = file_url;
if (!string.IsNullOrWhiteSpace(file_url) && string.IsNullOrWhiteSpace(sample_url))
{
sample_url = file_url;
}
post.SampleUrl = sample_url;
// Created datetime
post.CreatedAt = "N/A";
post.CreatedAtDateTime = DateTime.MinValue;
try
{
var lis = doc.DocumentNode.SelectNodes("//div[@id='post-view']//div[@id='stats']//li");
foreach (var item in lis)
{
if (item.InnerText.Contains("Posted:"))
{
var links = item.SelectNodes("//div[@id='post-view']//div[@id='stats']//li//a");
foreach (var link in links)
{
if (link.Attributes.Contains("href") && link.Attributes["href"].Value.Contains("?tags=date"))
{
post.CreatedAt = link.Attributes["title"].Value;
post.CreatedAtDateTime = DanbooruPostDao.ParseDateTime(post.CreatedAt, post.Provider);
break;
}
}
break;
}
}
}
catch (Exception ex)
{
Program.Logger.Error("Unable to parse date", ex);
}
return(post);
}
catch (Exception ex)
{
string filename = "Dump for Post " + post.Id + post.Provider.Name + " Query " + post.Query + ".txt";
bool result = Helper.DumpRawData(postHtml, filename);
if (!result)
{
Program.Logger.Error("Failed to dump rawdata to: " + filename, ex);
}
throw;
}
}
private void clientList_DownloadDataCompleted(object sender, DownloadDataCompletedEventArgs e)
{
try
{
Program.Logger.Debug("Download list completed");
tsProgressBar.Visible = false;
MemoryStream ms = new MemoryStream(e.Result);
DanbooruPostDaoOption option = new DanbooruPostDaoOption()
{
Provider = _currProvider,
//Url = txtListFile.Text,
Referer = _clientList.Referer,
Query = txtQuery.Text,
SearchTags = txtTags.Text,
BlacklistedTags = TagBlacklist,
BlacklistedTagsRegex = TagBlacklistRegex,
BlacklistedTagsUseRegex = chkBlacklistTagsUseRegex.Checked,
IgnoredTags = TagIgnore,
IgnoredTagsRegex = TagIgnoreRegex,
IgnoredTagsUseRegex = chkIgnoreTagsUseRegex.Checked
};
tsStatus.Text = "Loading downloaded list...";
tsProgressBar.Style = ProgressBarStyle.Marquee;
tsProgressBar.Visible = true;
_isLoadingList = true;
backgroundWorker1 = new BackgroundWorker();
backgroundWorker1.DoWork += new DoWorkEventHandler(backgroundWorker1_DoWork);
backgroundWorker1.RunWorkerCompleted += new RunWorkerCompletedEventHandler(backgroundWorker1_RunWorkerCompleted);
backgroundWorker1.RunWorkerAsync(new object[] { ms, option });
}
catch (Exception ex)
{
var message = ex.Message;
if (ex.InnerException != null)
{
message = ex.InnerException.Message;
var wex = ex.InnerException as System.Net.WebException;
if (wex != null && wex.Status == WebExceptionStatus.ProtocolError)
{
var status = wex.Response != null ? wex.Response.Headers["Status"] : null;
var response = wex.Response as HttpWebResponse;
if (status == "403" || (response != null && response.StatusCode == HttpStatusCode.Forbidden))
{
message += Environment.NewLine + "Please check your login information.";
}
else
{
using (var responseStream = wex.Response.GetResponseStream())
{
if (responseStream != null)
{
DanbooruPostDaoOption option = new DanbooruPostDaoOption()
{
Provider = _currProvider,
//Url = "",
Referer = "",
Query = "",
SearchTags = "",
BlacklistedTags = TagBlacklist,
BlacklistedTagsRegex = TagBlacklistRegex,
BlacklistedTagsUseRegex = chkBlacklistTagsUseRegex.Checked,
IgnoredTags = TagIgnore,
IgnoredTagsRegex = TagIgnoreRegex,
IgnoredTagsUseRegex = chkIgnoreTagsUseRegex.Checked
};
var resp = new DanbooruPostDao(responseStream, option);
message = "Server Message: " + resp.ResponseMessage;
if (status != "200")
{
message += "\nStatus Code: " + wex.Status.ToString() + " (" + status + ")";
}
}
}
}
}
}
MessageBox.Show(message, "Download List");
Program.Logger.Error(message, ex);
UpdateLog("clientList_DownloadDataCompleted", "Error: " + (ex.InnerException == null ? ex.Message : ex.InnerException.Message), ex);
UpdateLog("clientList_DownloadDataCompleted", "Referer: " + _clientList.Referer);
chkAutoLoadNext.Checked = false;
btnGet.Enabled = true;
btnListCancel.Enabled = false;
_isLoadingList = false;
}
}
public static DanbooruPost ParsePost(DanbooruPost post, string postHtml)
{
HtmlDocument doc = new HtmlDocument();
if (String.IsNullOrEmpty(postHtml))
{
throw new Exception("No post html!");
}
doc.LoadHtml(postHtml);
string file_url = "";
string sample_url = "";
// Flash Game or bmp
// TODO: need to change the preview url
if (post.PreviewUrl == "http://chan.sankakucomplex.com/download-preview.png")
{
var links = doc.DocumentNode.SelectNodes("//a");
foreach (var link in links)
{
// flash
if (link.InnerText == "Save this flash (right click and save)")
{
file_url = link.GetAttributeValue("href", "");
break;
}
// bmp
if (link.InnerText == "Download")
{
file_url = link.GetAttributeValue("href", "");
break;
}
}
}
else
{
var image = doc.DocumentNode.SelectSingleNode("//img[@id='image']");
if (image != null)
{
sample_url = image.GetAttributeValue("src", "");
}
var links = doc.DocumentNode.SelectNodes("//a");
foreach (var link in links)
{
if (link.InnerText == "Original image")
{
file_url = link.GetAttributeValue("href", "");
break;
}
}
}
post.FileUrl = file_url;
if (!string.IsNullOrWhiteSpace(file_url) && string.IsNullOrWhiteSpace(sample_url))
{
sample_url = file_url;
}
post.SampleUrl = sample_url;
// parse datetime
// TODO: untested
var statsLIs = doc.DocumentNode.SelectNodes("//div[@id='stats']//li");
foreach (var item in statsLIs)
{
if (item.InnerHtml.StartsWith("Posted: "))
{
post.CreatedAt = System.Text.RegularExpressions.Regex.Match(item.InnerHtml, "Posted: (.*)<").Value;
post.CreatedAtDateTime = DanbooruPostDao.ParseDateTime(post.CreatedAt, post.Provider);
break;
}
}
return(post);
}