public DownloadThread(string URL_in, ref CommFetchOptions opts_in)
{
this.URL = URL_in;
this.opts = opts_in;
try
{
ThreadStart tsGenericMethod = new ThreadStart(this.Go);
Thread trdGenericThread = new Thread(tsGenericMethod);
trdGenericThread.IsBackground = true;
trdGenericThread.Start();
DateTime dtStartTime = DateTime.Now;
for (; ;)
{
if (trdGenericThread.ThreadState == System.Threading.ThreadState.Stopped)
{
break;
}
if (trdGenericThread.ThreadState == System.Threading.ThreadState.StopRequested)
{
break;
}
if (trdGenericThread.ThreadState == System.Threading.ThreadState.Aborted)
{
break;
}
if (trdGenericThread.ThreadState == System.Threading.ThreadState.AbortRequested)
{
break;
}
Thread.Sleep(15);
frmMain.DoEvents();
}
}
catch { }
opts_in = this.opts;
}
public static byte[] Download(string sURL, ref CommFetchOptions options)
{
return Download_WC(sURL, ref options);
}
private static byte[] Download_WC_real(string sURL, ref CommFetchOptions options)
{
WebRequest oReq;
HttpWebRequest ohReq;
Uri url = new Uri(sURL);
byte[] bContent = null;
bool bReceivedUnicodeEncoding = false;
if (options.DataType_JSON)
Comm.SetAllowUnsafeHeaderParsing20(true);
System.Net.ServicePointManager.Expect100Continue = false;
try
{
oReq = WebRequest.Create(sURL);
ohReq = (HttpWebRequest)oReq;
if (string.IsNullOrEmpty(options.Accept))
ohReq.Accept = "*/*";
else
ohReq.Accept = options.Accept;
if (options.Method == CommFetchOptions.Methods.POST)
oReq.Method = "POST";
else
oReq.Method = "GET";
oReq.Headers.Add("Accept-Language: en-US,en;q=0.8");
oReq.Headers.Add("Accept-Encoding: gzip,deflate");
oReq.Headers.Add("Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3");
if (options.HTTP_Version == CommFetchOptions.HTTP_Versions.Version_1p0)
ohReq.ProtocolVersion = System.Net.HttpVersion.Version10;
else
ohReq.ProtocolVersion = System.Net.HttpVersion.Version11;
if (options.UserAgent == CommFetchOptions.UserAgentModes.IE9)
ohReq.UserAgent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)";
else if (options.UserAgent == CommFetchOptions.UserAgentModes.AIR)
ohReq.UserAgent = "Mozilla/5.0 (Android; U; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) AdobeAIR/4.0";
else if (options.UserAgent == CommFetchOptions.UserAgentModes.Dalvik)
ohReq.UserAgent = Comm.RandomDalvikString;
else
ohReq.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36";
for (int i = 0; i < options.Cookies.Count; i++)
Utils.Logger("Cookie sent: " + options.Cookies[i].Name + "=" + options.Cookies[i].Value.ToString());
for (int i = 0; i < options.Cookies.Count; i++)
{
if (options.Cookies[i].Name.ToLower() == "csrftoken")
{
Utils.Logger("Request header added: X-Requested-With: XMLHttpRequest");
oReq.Headers.Add("X-Requested-With: XMLHttpRequest");
Utils.Logger("Request header added: X-CSRFToken=" + options.Cookies[i].Value.ToString());
ohReq.Headers.Add("X-CSRFToken", options.Cookies[i].Value.ToString());
break;
}
}
if (options.DataType_JSON)
{
ohReq.Accept = "application/json, text/javascript, */*; q=0.01";
Utils.Logger("Request data type switched to JSON");
}
if (options.XMLHttpRequest)
oReq.Headers.Add("X-Requested-With: XMLHttpRequest");
if (options.UserAgent == CommFetchOptions.UserAgentModes.Dalvik)
oReq.Headers.Add("X-Unity-Version: 4.5.4f1");
ohReq.KeepAlive = true;
oReq.Proxy = System.Net.WebRequest.DefaultWebProxy;
if (!string.IsNullOrEmpty(options.Auth_Username))
oReq.Credentials = new NetworkCredential(options.Auth_Username, options.Auth_Password);
if (options.Credentials != null)
oReq.Credentials = options.Credentials;
if (options.Referer == CommFetchOptions.Referers.Full_URL)
ohReq.Referer = sURL;
else if (options.Referer == CommFetchOptions.Referers.Host_Only)
ohReq.Referer = "http://" + url.Host + "/";
else if (options.Referer == CommFetchOptions.Referers.Custom)
ohReq.Referer = options.CustomRefererURL;
//else if (options.Referer == CommFetchOptions.Referers.None)
// ; // no referer
if (options.Method == CommFetchOptions.Methods.POST)
{
ohReq.ContentLength = (long)(Utils.CByteUTF8(options.POST_Data).Length);
if (options.DataType_JSON)
{
ohReq.ContentType = "application/json";
//ohReq.ContentType = "application/json, text/javascript, */*; q=0.01";
}
else
ohReq.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
}
else
{
oReq.Headers.Add("Pragma: no-cache");
oReq.Headers.Add("Cache-Control: max-age=0");
}
ohReq.CookieContainer = options.CookieContainerGet();
oReq.Timeout = (options.TimeOut * 1000);
try
{
ohReq.Host = url.Host;
}
catch { }
ohReq.MaximumAutomaticRedirections = 3;
ohReq.AutomaticDecompression = (options.AutomaticDecompress) ? DecompressionMethods.GZip | DecompressionMethods.Deflate : DecompressionMethods.None;
if (options.Method == CommFetchOptions.Methods.POST)
{
Stream oDataStream = oReq.GetRequestStream();
oDataStream.Write(Utils.CByteUTF8(options.POST_Data), 0, Utils.CByteUTF8(options.POST_Data).Length);
oDataStream.Close();
if (Comm.WantToSeePageFetches)
if (options.DataType_JSON)
Utils.Logger("<fs-><color=#565656>[" + DateTime.Now.Hour.ToString("00") + ":" + DateTime.Now.Minute.ToString("00") + ":" + DateTime.Now.Second.ToString("00") + "." + DateTime.Now.Millisecond.ToString("000") + "]</color><fx>" +
" <fs-><color=#5656aa><b>Posted data:</b><fs+> " + options.POST_Data);
}
WebResponse oResp = oReq.GetResponse();
HttpWebResponse ohResp = (HttpWebResponse)oResp;
foreach (Cookie cookie in ohResp.Cookies)
options.CookieContainerAdd(cookie);
for (int i = 0; i < oResp.Headers.Count; i++)
{
string header = oResp.Headers.Keys[i];
string value = oResp.Headers.Get(i);
if (header == "Content-Type")
{
value = value.ToLower();
if (value.StartsWith("text/html"))
{
// assume unicode
bReceivedUnicodeEncoding = true;
}
else
{
if (value.ToLower().Contains("charset=utf-8"))
bReceivedUnicodeEncoding = true;
}
}
}
if (bReceivedUnicodeEncoding)
{
string sContent = "";
using (StreamReader oSRead = new StreamReader(oResp.GetResponseStream(), Encoding.UTF8, true))
{
sContent = oSRead.ReadToEnd();
oSRead.Close();
}
bContent = Utils.CByteUTF8(sContent);
}
else
{
BinaryReader oSRead = new BinaryReader(oResp.GetResponseStream());
List<byte> data = new List<byte>();
for (; ; ) { try { data.Add(oSRead.ReadByte()); } catch { break; } }
oSRead.Close();
bContent = data.ToArray();
data.Clear();
}
oResp.Close();
}
catch (TimeoutException ex)
{
options.LastException = ex;
if (options.WantErrors)
return Utils.CByteUTF8("[timeout]");
return null;
}
catch (Exception ex)
{
options.LastException = ex;
if (options.WantErrors)
return Utils.CByteUTF8("[exception: " + ex.Message + "]");
return null;
}
if (!options.AutomaticDecompress)
{
return bContent;
}
else
{
byte[] b = Comm.DecompressBytes(bContent);
if (Comm.WantToSeePageFetches)
if (options.DataType_JSON)
Utils.Logger("<fs-><color=#565656>[" + DateTime.Now.Hour.ToString("00") + ":" + DateTime.Now.Minute.ToString("00") + ":" + DateTime.Now.Second.ToString("00") + "." + DateTime.Now.Millisecond.ToString("000") + "]</color><fx>" +
" <fs-><color=#5656aa><b>Received data:</b><fs+> " + Utils.CStr(b));
//Utils.Logger("Notice: server returned UTF-8 (we requested UTF-8)");
return b;
}
}
public DownloadThread(string URL_in, ref CommFetchOptions opts_in)
{
this.URL = URL_in;
this.opts = opts_in;
try
{
ThreadStart tsGenericMethod = new ThreadStart(this.Go);
Thread trdGenericThread = new Thread(tsGenericMethod);
trdGenericThread.IsBackground = true;
trdGenericThread.Start();
DateTime dtStartTime = DateTime.Now;
for (; ; )
{
if (trdGenericThread.ThreadState == System.Threading.ThreadState.Stopped) break;
if (trdGenericThread.ThreadState == System.Threading.ThreadState.StopRequested) break;
if (trdGenericThread.ThreadState == System.Threading.ThreadState.Aborted) break;
if (trdGenericThread.ThreadState == System.Threading.ThreadState.AbortRequested) break;
Thread.Sleep(15);
frmMain.DoEvents();
}
}
catch { }
opts_in = this.opts;
}
// WebClient download method
private static byte[] Download_WC(string sURL, ref CommFetchOptions options)
{
return (new DownloadThread(sURL, ref options)).data;
}
public static byte[] Download(string sURL, ref CommFetchOptions options)
{
return(Download_WC(sURL, ref options));
}
private static byte[] Download_WC_real(string sURL, ref CommFetchOptions options)
{
WebRequest oReq;
HttpWebRequest ohReq;
Uri url = new Uri(sURL);
byte[] bContent = null;
bool bReceivedUnicodeEncoding = false;
if (options.DataType_JSON)
{
Comm.SetAllowUnsafeHeaderParsing20(true);
}
System.Net.ServicePointManager.Expect100Continue = false;
try
{
oReq = WebRequest.Create(sURL);
ohReq = (HttpWebRequest)oReq;
if (string.IsNullOrEmpty(options.Accept))
{
ohReq.Accept = "*/*";
}
else
{
ohReq.Accept = options.Accept;
}
if (options.Method == CommFetchOptions.Methods.POST)
{
oReq.Method = "POST";
}
else
{
oReq.Method = "GET";
}
oReq.Headers.Add("Accept-Language: en-US,en;q=0.8");
oReq.Headers.Add("Accept-Encoding: gzip,deflate");
oReq.Headers.Add("Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3");
if (options.HTTP_Version == CommFetchOptions.HTTP_Versions.Version_1p0)
{
ohReq.ProtocolVersion = System.Net.HttpVersion.Version10;
}
else
{
ohReq.ProtocolVersion = System.Net.HttpVersion.Version11;
}
if (options.UserAgent == CommFetchOptions.UserAgentModes.IE9)
{
ohReq.UserAgent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)";
}
else if (options.UserAgent == CommFetchOptions.UserAgentModes.AIR)
{
ohReq.UserAgent = "Mozilla/5.0 (Android; U; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) AdobeAIR/4.0";
}
else if (options.UserAgent == CommFetchOptions.UserAgentModes.Dalvik)
{
ohReq.UserAgent = Comm.RandomDalvikString;
}
else
{
ohReq.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36";
}
for (int i = 0; i < options.Cookies.Count; i++)
{
Utils.Logger("Cookie sent: " + options.Cookies[i].Name + "=" + options.Cookies[i].Value.ToString());
}
for (int i = 0; i < options.Cookies.Count; i++)
{
if (options.Cookies[i].Name.ToLower() == "csrftoken")
{
Utils.Logger("Request header added: X-Requested-With: XMLHttpRequest");
oReq.Headers.Add("X-Requested-With: XMLHttpRequest");
Utils.Logger("Request header added: X-CSRFToken=" + options.Cookies[i].Value.ToString());
ohReq.Headers.Add("X-CSRFToken", options.Cookies[i].Value.ToString());
break;
}
}
if (options.DataType_JSON)
{
ohReq.Accept = "application/json, text/javascript, */*; q=0.01";
Utils.Logger("Request data type switched to JSON");
}
if (options.XMLHttpRequest)
{
oReq.Headers.Add("X-Requested-With: XMLHttpRequest");
}
if (options.UserAgent == CommFetchOptions.UserAgentModes.Dalvik)
{
oReq.Headers.Add("X-Unity-Version: 4.5.4f1");
}
ohReq.KeepAlive = true;
oReq.Proxy = System.Net.WebRequest.DefaultWebProxy;
if (!string.IsNullOrEmpty(options.Auth_Username))
{
oReq.Credentials = new NetworkCredential(options.Auth_Username, options.Auth_Password);
}
if (options.Credentials != null)
{
oReq.Credentials = options.Credentials;
}
if (options.Referer == CommFetchOptions.Referers.Full_URL)
{
ohReq.Referer = sURL;
}
else if (options.Referer == CommFetchOptions.Referers.Host_Only)
{
ohReq.Referer = "http://" + url.Host + "/";
}
else if (options.Referer == CommFetchOptions.Referers.Custom)
{
ohReq.Referer = options.CustomRefererURL;
}
//else if (options.Referer == CommFetchOptions.Referers.None)
// ; // no referer
if (options.Method == CommFetchOptions.Methods.POST)
{
ohReq.ContentLength = (long)(Utils.CByteUTF8(options.POST_Data).Length);
if (options.DataType_JSON)
{
ohReq.ContentType = "application/json";
//ohReq.ContentType = "application/json, text/javascript, */*; q=0.01";
}
else
{
ohReq.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
}
}
else
{
oReq.Headers.Add("Pragma: no-cache");
oReq.Headers.Add("Cache-Control: max-age=0");
}
ohReq.CookieContainer = options.CookieContainerGet();
oReq.Timeout = (options.TimeOut * 1000);
try
{
ohReq.Host = url.Host;
}
catch { }
ohReq.MaximumAutomaticRedirections = 3;
ohReq.AutomaticDecompression = (options.AutomaticDecompress) ? DecompressionMethods.GZip | DecompressionMethods.Deflate : DecompressionMethods.None;
if (options.Method == CommFetchOptions.Methods.POST)
{
Stream oDataStream = oReq.GetRequestStream();
oDataStream.Write(Utils.CByteUTF8(options.POST_Data), 0, Utils.CByteUTF8(options.POST_Data).Length);
oDataStream.Close();
if (Comm.WantToSeePageFetches)
{
if (options.DataType_JSON)
{
Utils.Logger("<fs-><color=#565656>[" + DateTime.Now.Hour.ToString("00") + ":" + DateTime.Now.Minute.ToString("00") + ":" + DateTime.Now.Second.ToString("00") + "." + DateTime.Now.Millisecond.ToString("000") + "]</color><fx>" +
" <fs-><color=#5656aa><b>Posted data:</b><fs+> " + options.POST_Data);
}
}
}
WebResponse oResp = oReq.GetResponse();
HttpWebResponse ohResp = (HttpWebResponse)oResp;
foreach (Cookie cookie in ohResp.Cookies)
{
options.CookieContainerAdd(cookie);
}
for (int i = 0; i < oResp.Headers.Count; i++)
{
string header = oResp.Headers.Keys[i];
string value = oResp.Headers.Get(i);
if (header == "Content-Type")
{
value = value.ToLower();
if (value.StartsWith("text/html"))
{
// assume unicode
bReceivedUnicodeEncoding = true;
}
else
{
if (value.ToLower().Contains("charset=utf-8"))
{
bReceivedUnicodeEncoding = true;
}
}
}
}
if (bReceivedUnicodeEncoding)
{
string sContent = "";
using (StreamReader oSRead = new StreamReader(oResp.GetResponseStream(), Encoding.UTF8, true))
{
sContent = oSRead.ReadToEnd();
oSRead.Close();
}
bContent = Utils.CByteUTF8(sContent);
}
else
{
BinaryReader oSRead = new BinaryReader(oResp.GetResponseStream());
List <byte> data = new List <byte>();
for (; ;)
{
try { data.Add(oSRead.ReadByte()); } catch { break; }
}
oSRead.Close();
bContent = data.ToArray();
data.Clear();
}
oResp.Close();
}
catch (TimeoutException ex)
{
options.LastException = ex;
if (options.WantErrors)
{
return(Utils.CByteUTF8("[timeout]"));
}
return(null);
}
catch (Exception ex)
{
options.LastException = ex;
if (options.WantErrors)
{
return(Utils.CByteUTF8("[exception: " + ex.Message + "]"));
}
return(null);
}
if (!options.AutomaticDecompress)
{
return(bContent);
}
else
{
byte[] b = Comm.DecompressBytes(bContent);
if (Comm.WantToSeePageFetches)
{
if (options.DataType_JSON)
{
Utils.Logger("<fs-><color=#565656>[" + DateTime.Now.Hour.ToString("00") + ":" + DateTime.Now.Minute.ToString("00") + ":" + DateTime.Now.Second.ToString("00") + "." + DateTime.Now.Millisecond.ToString("000") + "]</color><fx>" +
" <fs-><color=#5656aa><b>Received data:</b><fs+> " + Utils.CStr(b));
}
}
//Utils.Logger("Notice: server returned UTF-8 (we requested UTF-8)");
return(b);
}
}
// WebClient download method
private static byte[] Download_WC(string sURL, ref CommFetchOptions options)
{
return((new DownloadThread(sURL, ref options)).data);
}
