public void GetProxy(string url) // GetProxy downloads the URL site data via HTTP and uses regex to extract proxies
{
MyWebClient wc = new MyWebClient();
try
{
string data = wc.DownloadString(url);
string pattern = @"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\:\d{1,5}";
MatchCollection matches = Regex.Matches(data, pattern);
if (matches.Count > 0)
{
string prox = "";
foreach (Match match in matches)
{
prox = prox + match.Groups[0].Value + Environment.NewLine;
count++;
}
LABEL_NumberOfProxies.Text = count.ToString();
TB_Proxies.AppendText(prox);
goodURLS.Add(url);
}
wc.Dispose();
}
catch (Exception)
{
wc.Dispose();
}
}
public override string GetPageString(int page, int count, string keyWord, System.Net.IWebProxy proxy)
{
Login(proxy);
//http://mjv-art.org/pictures/view_posts/0?lang=en
string url = SiteUrl + "/pictures/view_posts/" + (page - 1) + "?lang=en";
MyWebClient web = new MyWebClient();
web.Proxy = proxy;
web.Headers["Cookie"] = sessionId;
web.Encoding = Encoding.UTF8;
if (keyWord.Length > 0)
{
//http://mjv-art.org/pictures/view_posts/0?search_tag=suzumiya%20haruhi&order_by=date&ldate=0&lang=en
url = SiteUrl + "/pictures/view_posts/" + (page - 1) + "?search_tag=" + keyWord + "&order_by=date&ldate=0&lang=en";
}
string pageString = web.DownloadString(url);
web.Dispose();
return(pageString);
}
public override string GetPageString(int page, int count, string keyWord, IWebProxy proxy)
{
//if (page > 1000) throw new Exception("页码过大，若需浏览更多图片请使用关键词限定范围");
Login(proxy);
//http://www.pixiv.net/new_illust.php?p=2
string url = SiteUrl + "/new_illust.php?p=" + page;
MyWebClient web = new MyWebClient();
web.Proxy = proxy;
web.Headers["Cookie"] = cookie;
web.Encoding = Encoding.UTF8;
if (keyWord.Length > 0)
{
//http://www.pixiv.net/search.php?s_mode=s_tag&word=hatsune&order=date_d&p=2
url = SiteUrl + "/search.php?s_mode=s_tag"
+ (srcType == PixivSrcType.TagFull ? "_full" : "")
+ "&word=" + keyWord + "&order=date_d&p=" + page;
}
if (srcType == PixivSrcType.Author)
{
int memberId = 0;
if (keyWord.Trim().Length == 0 || !int.TryParse(keyWord.Trim(), out memberId))
{
throw new Exception("必须在关键词中指定画师 id；若需要使用标签进行搜索请使用 www.pixiv.net [TAG]");
}
//member id
url = SiteUrl + "/member_illust.php?id=" + memberId + "&p=" + page;
}
else if (srcType == PixivSrcType.Day)
{
url = SiteUrl + "/ranking.php?mode=daily&p=" + page;
}
else if (srcType == PixivSrcType.Week)
{
url = SiteUrl + "/ranking.php?mode=weekly&p=" + page;
}
else if (srcType == PixivSrcType.Month)
{
url = SiteUrl + "/ranking.php?mode=monthly&p=" + page;
}
string pageString = web.DownloadString(url);
web.Dispose();
return(pageString);
}
private void DownloadFile(object t)
{
WebClient wc = new MyWebClient(timerOut);
try
{
wc.DownloadFile(_url, _toFile);
}
catch { }
finally
{
wc.Dispose();
}
}
/// <summary>
/// get images sync
/// </summary>
//public List<Img> GetImages(int page, int count, string keyWord, int maskScore, int maskRes, ViewedID lastViewed, bool maskViewed, System.Net.IWebProxy proxy, bool showExplicit)
//{
// return GetImages(GetPageString(page, count, keyWord, proxy), maskScore, maskRes, lastViewed, maskViewed, proxy, showExplicit);
//}
public override string GetPageString(int page, int count, string keyWord, System.Net.IWebProxy proxy)
{
Login(proxy);
string url = SiteUrl + "/?p=" + page;
MyWebClient web = new MyWebClient();
web.Proxy = proxy;
web.Headers["Cookie"] = sessionId;
web.Encoding = Encoding.UTF8;
if (keyWord.Length > 0)
{
//先使用关键词搜索，然后HTTP 301返回实际地址
//http://www.zerochan.net/search?q=tony+taka
url = SiteUrl + "/search?q=" + keyWord;
System.Net.HttpWebRequest req = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url);
req.UserAgent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)";
req.Proxy = proxy;
req.Timeout = 8000;
req.Method = "GET";
//prevent 301
req.AllowAutoRedirect = false;
System.Net.WebResponse rsp = req.GetResponse();
//http://www.zerochan.net/Tony+Taka?p=1
//HTTP 301然后返回实际地址
string location = rsp.Headers["Location"];
rsp.Close();
if (location != null && location.Length > 0)
{
//非完整地址，需要前缀
url = SiteUrl + location + "?p=" + page;
}
else
{
throw new Exception("搜索失败，请检查您输入的关键词");
}
}
string pageString = web.DownloadString(url);
web.Dispose();
return(pageString);
}
public override string GetPageString(int page, int count, string keyWord, System.Net.IWebProxy proxy)
{
string address = string.Format(SiteUrl + "/posts?page={0}&limit={1}&tags={2}", page, count, keyWord);
if (keyWord.Length == 0)
{
address = address.Substring(0, address.Length - 6);
}
MyWebClient client = new MyWebClient
{
Proxy = proxy,
Encoding = Encoding.UTF8
};
string pageString = client.DownloadString(address);
client.Dispose();
return(pageString);
}
//public bool ShowExplicit { get; set; }
//public virtual bool IsSupportCount { get { return true; } }
//public virtual bool IsSupportScore { get { return true; } }
//public virtual bool IsSupportRes { get { return true; } }
//public virtual bool IsSupportPreview { get { return true; } }
//public virtual bool IsSupportJpeg { get { return true; } }
//public virtual bool IsSupportTag { get { return true; } }
//public virtual System.Drawing.Point LargeImgSize { get { return new System.Drawing.Point(SWIDTH, SWIDTH); } }
//public virtual System.Drawing.Point SmallImgSize { get { return new System.Drawing.Point(SWIDTH, SHEIGHT); } }
/// <summary>
/// get images sync
/// </summary>
//public virtual List<Img> GetImages(int page, int count, string keyWord, int maskScore, int maskRes, ViewedID lastViewed, bool maskViewed, System.Net.IWebProxy proxy, bool showExplicit)
//{
// return GetImages(GetPageString(page, count, keyWord, proxy), maskScore, maskRes, lastViewed, maskViewed, proxy, showExplicit);
//}
public override string GetPageString(int page, int count, string keyWord, System.Net.IWebProxy proxy)
{
string url = string.Format(siteUrl, needMinus ? page - 1 : page, count, keyWord);
if (keyWord.Length == 0)
{
url = url.Substring(0, url.Length - 6);
}
MyWebClient web = new MyWebClient();
web.Proxy = proxy;
web.Encoding = Encoding.UTF8;
string pageString = web.DownloadString(url);
web.Dispose();
return(pageString);
}
public override string GetPageString(int page, int count, string keyWord, System.Net.IWebProxy proxy)
{
//http://worldcosplay.net/api/photo/list?page=3&limit=2&sort=created_at&direction=descend
string url = SiteUrl + "/api/photo/list?page=" + page + "&limit=" + count + "&sort=created_at&direction=descend";
MyWebClient web = new MyWebClient();
web.Proxy = proxy;
web.Encoding = Encoding.UTF8;
if (keyWord.Length > 0)
{
//http://worldcosplay.net/api/photo/search?page=2&rows=48&q=%E5%90%8A%E5%B8%A6%E8%A2%9C%E5%A4%A9%E4%BD%BF
url = SiteUrl + "/api/photo/search?page=" + page + "&rows=" + count + "&q=" + keyWord;
}
string pageString = web.DownloadString(url);
web.Dispose();
return(pageString);
}
private void FWebClientDownloader_DownloadFileCompleted(object sender, System.ComponentModel.AsyncCompletedEventArgs e)
{
try
{
MyWebClient lSender = sender as MyWebClient;
if (!e.Cancelled && e.Error == null)
{
DoUpgradeFileReady(lSender.FileName);
}
else
if (File.Exists(lSender.FileName))
{
File.Delete(lSender.FileName);
}
FWebClientDownloader = null;
lSender.Dispose();
}
catch (Exception ex)
{
this.DoErrorHandler(ex);
}
}
public override string GetPageString(int page, int count, string keyWord, System.Net.IWebProxy proxy)
{
//http://chan.sankakucomplex.com/post/index.content?page=2&limit=3&tags=xxx
string url = SiteUrl + "/post/index.content?page=" + page + "&limit=" + count;
MyWebClient web = new MyWebClient();
web.Proxy = proxy;
//web.Headers["Cookie"] = sessionId;
web.Encoding = Encoding.UTF8;
if (keyWord.Length > 0)
{
url += "&tags=" + keyWord;
}
string pageString = web.DownloadString(url);
web.Dispose();
return(pageString);
}
public override string GetPageString(int page, int count, string keyWord, System.Net.IWebProxy proxy)
{
//http://yuriimg.com/post/?.html
string url = SiteUrl + "/post/" + page + ".html";
// string url = "http://yuriimg.com/show/ge407xd5o.jpg";
MyWebClient web = new MyWebClient();
web.Proxy = proxy;
web.Encoding = Encoding.UTF8;
if (keyWord.Length > 0)
{
//http://yuriimg.com/search/index/tags/?/p/?.html
url = SiteUrl + "/search/index/tags/" + keyWord + "/p/" + page + ".html";
}
string pageString = web.DownloadString(url);
web.Dispose();
return(pageString);
}
public static Tuple <string, bool> UpdateTimetable(string user, string pass)
{
try
{
MyWebClient web = new MyWebClient();
web.Proxy = null;
CredentialCache myCache = new CredentialCache();
if (user.Trim() != "" && pass.Trim() != "")
{
myCache.Add(new Uri("https://intranet.trinity.vic.edu.au/"), "NTLM",
new NetworkCredential(user, pass));
web.Credentials = myCache;
}
else
{
web.Credentials = CredentialCache.DefaultNetworkCredentials;
}
String html = web.DownloadString("https://intranet.trinity.vic.edu.au/timetable/default.asp");
Match match = Regex.Match(html, "<input type=\"hidden\" value=\"(.*?)\" id=\"callType\">",
RegexOptions.IgnoreCase);
if (match.Success)
{
Program.Calltype = match.Groups[1].Value;
}
match = Regex.Match(html, "<input type=\"hidden\" value=\"(.*?)\" id=\"synID\">",
RegexOptions.IgnoreCase);
if (match.Success)
{
Program.SettingsData.SynID = Convert.ToInt32(match.Groups[1].Value);
}
match = Regex.Match(html, "value=\"(.*?)\" id=\"curTerm\"", RegexOptions.IgnoreCase);
if (match.Success)
{
Program.SettingsData.Curterm = Convert.ToInt32(match.Groups[1].Value);
}
string sqlquery = "";
if (Program.Calltype == "student")
{
sqlquery =
"%20AND%20TD.PeriodNumber%20>=%200%20AND%20TD.PeriodNumberSeq%20=%201AND%20(stopdate%20IS%20NULL%20OR%20stopdate%20>%20getdate())--";
}
if (Program.SettingsData.Curterm == 0)
{
for (int i = 4; i > 0; i--)
{
html = web.DownloadString(
"https://intranet.trinity.vic.edu.au/timetable/getTimetable1.asp?synID=" + Program.SettingsData.SynID +
"&year=" + DateTime.Now.Year + "&term=" + i + sqlquery + "&callType=" + Program.Calltype);
if (html.Length > 10)
{
Program.SettingsData.Curterm = i;
html = html.Substring(html.IndexOf("["));
break;
}
}
}
else
{
html = web.DownloadString("https://intranet.trinity.vic.edu.au/timetable/getTimetable1.asp?synID=" +
Program.SettingsData.SynID + "&year=" + DateTime.Now.Year + "&term=" +
Program.SettingsData.Curterm + sqlquery + "&callType=" +
Program.Calltype);
html = html.Substring(html.IndexOf("["));
}
List <Period> timetableList = JsonConvert.DeserializeObject <List <Period> >(html);
using (StreamWriter file = File.CreateText(Program.SETTINGS_DIRECTORY + "/Timetable.Json"))
{
JsonSerializer serializer = new JsonSerializer();
serializer.Formatting = Formatting.Indented;
serializer.Serialize(file, timetableList);
}
Int16 colorint = 0;
Program.TimetableList.Clear();
Dictionary <string, int> yearlevel = new Dictionary <string, int>();
for (int i = 1; i < 13; i++)
{
string p = i < 10 ? "0" : "";
yearlevel.Add(p + i, 0);
}
foreach (var v in timetableList)
{
if (!Program.ColorRef.ContainsKey(v.ClassCode))
{
Program.ColorRef.Add(v.ClassCode, Program.ColourTable[colorint]);
colorint++;
if (colorint >= Program.ColourTable.Count)
{
colorint = 0;
}
}
Program.TimetableList.Add(v.DayNumber.ToString() + v.PeriodNumber, v);
if (yearlevel.ContainsKey(v.ClassCode.Substring(0, 2)))
{
yearlevel[v.ClassCode.Substring(0, 2)] += 1;
}
}
web.Dispose();
//Analytics reporting
string currentyear = yearlevel.Aggregate((l, r) => l.Value > r.Value ? l : r).Key;
if (currentyear == "12" && yearlevel["12"] == 0)
{
//do nothing
}
else
{
Program.SettingsData.CurrentYearlevel = int.Parse(currentyear);
}
}
catch (WebException ee)
{
if (ee.Message.Contains("Unauthorized"))
{
return(new Tuple <string, bool>("Unauthorised", false));
}
return(new Tuple <string, bool>(ee.Message, false));
}
catch (Exception ee)
{
return(new Tuple <string, bool>(ee.Message, false));
}
return(new Tuple <string, bool>("Successful!", true));
}
/// <summary>
/// get images sync
/// </summary>
//public List<Img> GetImages(int page, int count, string keyWord, int maskScore, int maskRes, ViewedID lastViewed, bool maskViewed, System.Net.IWebProxy proxy, bool showExplicit)
//{
// return GetImages(GetPageString(page, count, keyWord, proxy), maskScore, maskRes, lastViewed, maskViewed, proxy, showExplicit);
//}
public override string GetPageString(int page, int count, string keyWord, System.Net.IWebProxy proxy)
{
string url = SiteUrl + "/?page=" + page;
MyWebClient web = new MyWebClient();
web.Proxy = proxy;
web.Encoding = Encoding.UTF8;
if (keyWord.Length > 0)
{
url = SiteUrl + "/search/process/";
//multi search
string data = "tags=" + keyWord + "&source=&char=&artist=&postcontent=&txtposter=";
if (type == 2)
{
data = "tags=&source=" + keyWord + "&char=&artist=&postcontent=&txtposter=";
}
else if (type == 3)
{
data = "tags=&source=&char=&artist=" + keyWord + "&postcontent=&txtposter=";
}
else if (type == 4)
{
data = "tags=&source=&char=" + keyWord + "&artist=&postcontent=&txtposter=";
}
//e-shuushuu需要将关键词转换为tag id，然后进行搜索
System.Net.HttpWebRequest req = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url);
req.UserAgent = SessionClient.DefUA;
req.Proxy = proxy;
req.Timeout = 8000;
req.Method = "POST";
//prevent 303
req.AllowAutoRedirect = false;
byte[] buf = Encoding.UTF8.GetBytes(data);
req.ContentType = "application/x-www-form-urlencoded";
req.ContentLength = buf.Length;
System.IO.Stream str = req.GetRequestStream();
str.Write(buf, 0, buf.Length);
str.Close();
System.Net.WebResponse rsp = req.GetResponse();
//http://e-shuushuu.net/search/results/?tags=2
//HTTP 303然后返回实际地址
string location = rsp.Headers["Location"];
rsp.Close();
if (location != null && location.Length > 0)
{
//非完整地址，需要前缀
url = rsp.Headers["Location"] + "&page=" + page;
}
else
{
throw new Exception("没有搜索到关键词相关的图片（每个关键词前后需要加双引号如 \"sakura\"））");
}
}
string pageString = web.DownloadString(url);
web.Dispose();
return(pageString);
}
private void Loginbutton_Click(object sender, EventArgs e)
{
try
{
Errormsg.Text = "Attempting fetch...";
Errormsg.Update();
MyWebClient web = new MyWebClient();
web.Proxy = null;
CredentialCache myCache = new CredentialCache();
myCache.Add(new Uri("https://intranet.trinity.vic.edu.au/timetable/default.asp"), "NTLM", new NetworkCredential(Userbox.Text, Passbox.Text));
web.Credentials = myCache;
String html = web.DownloadString("https://intranet.trinity.vic.edu.au/timetable/default.asp");
Match match = Regex.Match(html, "<input type=\"hidden\" value=\"(.*?)\" id=\"callType\">", RegexOptions.IgnoreCase);
if (match.Success)
{
Program.Calltype = match.Groups[1].Value;
}
match = Regex.Match(html, "<input type=\"hidden\" value=\"(.*?)\" id=\"curDay\">", RegexOptions.IgnoreCase);
if (match.Success)
{
int.TryParse(match.Groups[1].Value, out var tempint);
Program.SettingsData.Referencedayone = Program.CalDayone(tempint);
Program.curDay = tempint;
}
match = Regex.Match(html, "<input type=\"hidden\" value=\"(.*?)\" id=\"synID\">", RegexOptions.IgnoreCase);
if (match.Success)
{
Program.SynID = Convert.ToInt32(match.Groups[1].Value);
}
match = Regex.Match(html, "value=\"(.*?)\" id=\"curTerm\"", RegexOptions.IgnoreCase);
if (match.Success)
{
Program.SettingsData.Curterm = Convert.ToInt32(match.Groups[1].Value);
}
string sqlquery = "";
if (Program.Calltype == "student")
{
sqlquery =
"%20AND%20TD.PeriodNumber%20>=%200%20AND%20TD.PeriodNumberSeq%20=%201AND%20(stopdate%20IS%20NULL%20OR%20stopdate%20>%20getdate())--";
}
if (Program.SettingsData.Curterm == 0)
{
for (int i = 4; i > 0; i--)
{
myCache.Add(new Uri("https://intranet.trinity.vic.edu.au/timetable/getTimetable1.asp?synID=" + Program.SynID + "&year=" + DateTime.Now.Year + "&term=" + i + sqlquery + "&callType=" + Program.Calltype), "NTLM", new NetworkCredential(Userbox.Text, Passbox.Text));
web.Credentials = myCache;
html = web.DownloadString("https://intranet.trinity.vic.edu.au/timetable/getTimetable1.asp?synID=" + Program.SynID + "&year=" + DateTime.Now.Year + "&term=" + i + sqlquery + "&callType=" + Program.Calltype);
if (html.Length > 10)
{
Program.SettingsData.Curterm = i;
break;
}
}
}
else
{
myCache.Add(new Uri("https://intranet.trinity.vic.edu.au/timetable/getTimetable1.asp?synID=" + Program.SynID + "&year=" + DateTime.Now.Year + "&term=" + Program.SettingsData.Curterm + sqlquery + "&callType=" + Program.Calltype), "NTLM", new NetworkCredential(Userbox.Text, Passbox.Text));
web.Credentials = myCache;
html = web.DownloadString("https://intranet.trinity.vic.edu.au/timetable/getTimetable1.asp?synID=" + Program.SynID + "&year=" + DateTime.Now.Year + "&term=" + Program.SettingsData.Curterm + sqlquery + "&callType=" + Program.Calltype);
}
List <period> timetableList = JsonConvert.DeserializeObject <List <period> >(html);
using (StreamWriter file = File.CreateText(Program.CURRENT_DIRECTORY + "/Timetable.json"))
{
JsonSerializer serializer = new JsonSerializer();
serializer.Formatting = Formatting.Indented;
serializer.Serialize(file, timetableList);
}
Int16 colorint = 0;
Program.TimetableList.Clear();
foreach (var V in timetableList)
{
if (!Program.ColorRef.ContainsKey(V.ClassCode))
{
Program.ColorRef.Add(V.ClassCode, Program.ColourTable[colorint]);
colorint++;
if (colorint >= Program.ColourTable.Count)
{
colorint = 0;
}
}
Program.TimetableList.Add(V.DayNumber.ToString() + V.PeriodNumber, V);
}
web.Dispose();
Errormsg.Text = "Successfully extracted! ";
Errormsg.Update();
TcpClient tcpclnt = new TcpClient();
try
{
if (tcpclnt.ConnectAsync("timetable.duckdns.org", 80).Wait(1200))
{
String str = "T" + Program.SynID + " " + Program.Calltype + " " + Program.APP_VERSION;
Stream stm = tcpclnt.GetStream();
ASCIIEncoding asen = new ASCIIEncoding();
byte[] ba = asen.GetBytes(str);
stm.Write(ba, 0, ba.Length);
Errormsg.Text += "Saved";
}
else
{
Errormsg.Text += "Filed";
}
}
catch
{
Errormsg.Text += "Filed";
}
tcpclnt.Close();
}
catch (WebException ee)
{
Errormsg.Text = ee.Message;
if (ee.Message.Contains("Unauthorized"))
{
Errormsg.Text = "Authorization failed";
}
}
catch (Exception ee)
{
Errormsg.Text = ee.Message;
MessageBox.Show(ee.ToString());
}
Userbox.Text = "";
Passbox.Text = "";
}
/// <summary>
/// get images sync
/// </summary>
//public List<Img> GetImages(int page, int count, string keyWord, int maskScore, int maskRes, ViewedID lastViewed, bool maskViewed, System.Net.IWebProxy proxy, bool showExplicit)
//{
// return GetImages(GetPageString(page, count, keyWord, proxy), maskScore, maskRes, lastViewed, maskViewed, proxy, showExplicit);
//}
public override string GetPageString(int page, int count, string keyWord, System.Net.IWebProxy proxy)
{
Login(proxy);
//http://gallery.minitokyo.net/scans?order=id&display=extensive&page=2
string url = "http://gallery.minitokyo.net/" + type + "?order=id&display=extensive&page=" + page;
MyWebClient web = new MyWebClient();
web.Proxy = proxy;
web.Headers["Cookie"] = sessionId;
web.Encoding = Encoding.UTF8;
if (keyWord.Length > 0)
{
//先使用关键词搜索，然后HTTP 303返回实际地址
//http://www.minitokyo.net/search?q=haruhi
url = SiteUrl + "/search?q=" + keyWord;
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
req.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36";
req.Proxy = proxy;
req.Timeout = 8000;
req.Method = "GET";
//prevent 303 See Other
req.AllowAutoRedirect = false;
WebResponse rsp = req.GetResponse();
//http://www.minitokyo.net/Haruhi+Suzumiya
//HTTP 303然后返回实际地址
string location = rsp.Headers["Location"];
rsp.Close();
if (location != null && location.Length > 0)
{
//非完整地址，需要前缀
url = SiteUrl + location;
//再次访问，得到真实列表页地址...
string html = web.DownloadString(url);
//http://browse.minitokyo.net/gallery?tid=2112&index=1 WALL
//http://browse.minitokyo.net/gallery?tid=2112&index=3 SCAN
//http://browse.minitokyo.net/gallery?tid=2112&index=1&order=id
int urlIndex = html.IndexOf("http://browse.minitokyo.net/gallery?tid=");
if (type == WALL)
{
url = html.Substring(urlIndex, html.IndexOf('"', urlIndex) - urlIndex - 1) + "1";
}
else
{
url = html.Substring(urlIndex, html.IndexOf('"', urlIndex) - urlIndex - 1) + "3";
}
//http://browse.minitokyo.net/gallery?tid=2112&%3Bindex=1&order=id&display=extensive
url += "&order=id&display=extensive&page=" + page;
url = url.Replace("&", "&");
}
else
{
throw new Exception("搜索失败，请检查您输入的关键词");
}
}
string pageString = web.DownloadString(url);
web.Dispose();
return(pageString);
}