public string HttpGetRequest(string url, bool followRedirect = true)
{
Is403Error = false;
RequestStatusCode = 0;
CurrentRetryCount = 0;
WebClientRequest = (HttpWebRequest)WebRequest.Create(url);
WebClientRequest.CookieContainer = _cJar;
WebClientRequest.UserAgent = WebUserAgentString;
WebClientRequest.Accept = "*/*";
WebClientRequest.KeepAlive = false;
WebClientRequest.Method = "GET";
if (followRedirect)
{
WebClientRequest.AllowAutoRedirect = false;
}
while (!CheckWebResponse())
{
Thread.Sleep(5000);
CurrentRetryCount++;
Log.Info($"HTTP Get Retry: {CurrentRetryCount} of {MaxWebRetries}");
if (CurrentRetryCount >= MaxWebRetries)
{
return(string.Empty);
}
}
if (followRedirect && (RequestStatusCode == (int)HttpStatusCode.Moved ||
RequestStatusCode == (int)HttpStatusCode.Found))
{
while (WebClientResponse.StatusCode == HttpStatusCode.Found ||
WebClientResponse.StatusCode == HttpStatusCode.Moved)
{
WebClientResponse.Close();
WebClientRequest = (HttpWebRequest)WebRequest.Create(WebClientResponse.Headers["Location"]);
WebClientRequest.AllowAutoRedirect = false;
WebClientRequest.CookieContainer = _cJar;
WebClientResponse = (HttpWebResponse)WebClientRequest.GetResponse();
}
}
StreamReader = new StreamReader(WebClientResponse.GetResponseStream()
?? throw new InvalidOperationException());
RequestStatusCode = (int)WebClientResponse.StatusCode;
var responseData = StreamReader.ReadToEnd();
return(responseData);
}
public string HttpPostRequest(string url, string post, bool followRedirect = true, string refer = "")
{
WebClientRequest = (HttpWebRequest)WebRequest.Create(url);
WebClientRequest.CookieContainer = _cJar;
WebClientRequest.UserAgent = WebUserAgentString;
WebClientRequest.KeepAlive = false;
WebClientRequest.Method = "POST";
WebClientRequest.Referer = refer;
if (followRedirect)
{
WebClientRequest.AllowAutoRedirect = false;
}
var postBytes = Encoding.ASCII.GetBytes(post);
WebClientRequest.ContentType = "text/x-www-form-urlencoded";
WebClientRequest.ContentLength = postBytes.Length;
var requestStream = WebClientRequest.GetRequestStream();
requestStream.Write(postBytes, 0, postBytes.Length);
requestStream.Close();
WebClientResponse = (HttpWebResponse)WebClientRequest.GetResponse();
if (followRedirect && (WebClientResponse.StatusCode == HttpStatusCode.Moved ||
WebClientResponse.StatusCode == HttpStatusCode.Found))
{
while (WebClientResponse.StatusCode == HttpStatusCode.Found ||
WebClientResponse.StatusCode == HttpStatusCode.Moved)
{
WebClientResponse.Close();
WebClientRequest = (HttpWebRequest)WebRequest.Create(WebClientResponse.Headers["Location"]);
WebClientRequest.AllowAutoRedirect = false;
WebClientRequest.CookieContainer = _cJar;
WebClientResponse = (HttpWebResponse)WebClientRequest.GetResponse();
}
}
StreamReader = new StreamReader(WebClientResponse.GetResponseStream()
?? throw new InvalidOperationException());
return(StreamReader.ReadToEnd());
}
bool ShouldDownloadCheck(bool canReadWrite, string lastModifiedFile)
{
var f = new FileInfo(lastModifiedFile);
bool shouldDownload = !File.Exists(lastModifiedFile);
if (canReadWrite && File.Exists(lastModifiedFile))
{
var lastModified = File.ReadAllText(lastModifiedFile);
DateTime lastModifiedDate;
if (!DateTime.TryParse(lastModified, out lastModifiedDate))
{
shouldDownload = true;
}
else
{
var request = new WebClientRequest();
request.Headers = new WebHeaderCollection();
var client = new ParallelWebClient(request);
client.OpenReadTask(ALL_COUNTRIES_URL, "HEAD").ContinueWith(t =>
{
var headers = client.ResponseHeaders;
foreach (string h in headers.Keys)
{
if (h.ToLowerInvariant() == "last-modified")
{
var header = headers[h];
DateTime headerDate = DateTime.Now;
DateTime.TryParse(header.ToString(), out headerDate);
if (lastModifiedDate < headerDate)
{
shouldDownload = true;
}
break;
}
}
}).Wait();
}
}
_logger.DebugFormat("ShouldDownloadCheck({0} , {1}) => {2}", canReadWrite, f.Name, shouldDownload);
return(shouldDownload);
}
private bool CheckWebResponse()
{
try
{
WebClientResponse = (HttpWebResponse)WebClientRequest.GetResponse();
RequestStatusCode = (int)WebClientResponse.StatusCode;
if (RequestStatusCode != (int)HttpStatusCode.OK)
{
SuccessfulWebRequest = false;
throw new Exception($"Api Error: Request status code does not match 200OK: {RequestStatusCode}");
}
SuccessfulWebRequest = true;
}
catch (WebException cwrException)
{
RequestStatusCode = (int)((HttpWebResponse)cwrException.Response).StatusCode;
Is403Error = RequestStatusCode == (int)HttpStatusCode.Forbidden;
Log.Error($"Http Get Exception: {cwrException.Message}");
if (cwrException.InnerException != null)
{
Log.Error($"Inner Exception: {cwrException.InnerException.Message}");
}
SuccessfulWebRequest = false;
}
catch (Exception unhandledException)
{
Log.Error($"Http Get Unhandled Exception: {unhandledException.Message}");
if (unhandledException.InnerException != null)
{
Log.Error($"Inner Exception: {unhandledException.InnerException.Message}");
}
SuccessfulWebRequest = false;
}
return(SuccessfulWebRequest);
}
private async Task LoginAsync(CancellationToken cancellationToken)
{
Console.WriteLine("Logging into Crunchyroll.");
WebClientRequest request = new WebClientRequest(LoginUrl);
request.UserAgent = "animerecs.com stream update tool";
request.PostParameters.Add(new KeyValuePair <string, string>("formname", "RpcApiUser_Login"));
request.PostParameters.Add(new KeyValuePair <string, string>("fail_url", "http://www.crunchyroll.com/login"));
request.PostParameters.Add(new KeyValuePair <string, string>("name", _crunchyrollUsername));
request.PostParameters.Add(new KeyValuePair <string, string>("password", _crunchyrollPassword));
using (var response = await _webClient.PostAsync(request, cancellationToken).ConfigureAwait(continueOnCapturedContext: false))
{
// don't care about response text, just the cookie
}
CookieCollection cookies = _webClient.Cookies.GetCookies(new Uri("http://www.crunchyroll.com"));
if (cookies["sess_id"] == null)
{
throw new Exception("Crunchyroll sess_id cookie was not set after logging in, maybe username/password was wrong.");
}
}
public static Task Update()
{
return(Task.Factory.StartNew(() =>
{
lock (_lock)
{
string lastModifiedFile = System.IO.Path.Combine(Root, LastModified);
string tmpFile = System.IO.Path.Combine(Root, Guid.NewGuid().ToString());
countriesRawFile = System.IO.Path.Combine(Root, countriesRawFile);
string allCountriesFile = System.IO.Path.Combine(Root, dataFile + ".zip");
string countryInfoFile = System.IO.Path.Combine(Root, "countryInfo.txt");
string featureCodes_enFile = System.IO.Path.Combine(Root, "featureCodes_en.txt");
string timeZonesFile = System.IO.Path.Combine(Root, "timeZones.txt");
//check to see if we have read/write persmission to disk
bool canReadWrite = false;
try
{
System.IO.File.WriteAllText(tmpFile, "testing write access");
string contents = System.IO.File.ReadAllText(tmpFile);
canReadWrite = !string.IsNullOrEmpty(contents);
System.IO.File.Delete(tmpFile);
}
catch (Exception)
{
}
//load file: GeoData-LastModified.txt
//if available then
//try to http head the data source url: http://download.geonames.org/export/dump/allCountries.zip
//pull off last-modified header
//if newer than the file last-modified, do a GET to download
//else
// do a get to download
bool shouldDownload = !System.IO.File.Exists(lastModifiedFile);
if (canReadWrite && System.IO.File.Exists(lastModifiedFile))
{
var lastModified = System.IO.File.ReadAllText(lastModifiedFile);
DateTime lastModifiedDate;
if (!DateTime.TryParse(lastModified, out lastModifiedDate))
{
shouldDownload = true;
}
else
{
var request = new WebClientRequest();
request.Headers = new WebHeaderCollection();
var client = new ParallelWebClient(request);
client.OpenReadTask(allCountriesUrl, "HEAD").ContinueWith(t =>
{
var headers = client.ResponseHeaders;
foreach (string h in headers.Keys)
{
if (h.ToLowerInvariant() == "last-modified")
{
var header = headers[h];
DateTime headerDate = DateTime.Now;
DateTime.TryParse(header.ToString(), out headerDate);
if (lastModifiedDate < headerDate)
{
shouldDownload = true;
}
break;
}
}
}).Wait();
}
}
if (canReadWrite && shouldDownload)
{
var downloadTasks = new List <Task>();
var request = new WebClientRequest();
request.Headers = new WebHeaderCollection();
var client = new ParallelWebClient(request);
downloadTasks.Add(
client.DownloadData(allCountriesUrl).ContinueWith(t =>
{
if (System.IO.File.Exists(allCountriesFile))
{
System.IO.File.Delete(allCountriesFile);
}
System.IO.File.WriteAllBytes(allCountriesFile, t.Result);
var headers = client.ResponseHeaders;
foreach (string h in headers.Keys)
{
if (h.ToLowerInvariant() == "last-modified")
{
var header = headers[h];
DateTime headerDate = DateTime.Now;
DateTime.TryParse(header.ToString(), out headerDate);
System.IO.File.WriteAllText(lastModifiedFile, headerDate.ToString());
break;
}
}
})
);
downloadTasks.Add(client.DownloadData(countryInfoUrl).ContinueWith(t =>
{
if (System.IO.File.Exists(countryInfoFile))
{
System.IO.File.Delete(countryInfoFile);
}
System.IO.File.WriteAllBytes(countryInfoFile, t.Result);
}));
downloadTasks.Add(client.DownloadData(featureCodes_enUrl).ContinueWith(t =>
{
if (System.IO.File.Exists(featureCodes_enFile))
{
System.IO.File.Delete(featureCodes_enFile);
}
System.IO.File.WriteAllBytes(featureCodes_enFile, t.Result);
}));
downloadTasks.Add(client.DownloadData(timeZonesUrl).ContinueWith(t =>
{
if (System.IO.File.Exists(timeZonesFile))
{
System.IO.File.Delete(timeZonesFile);
}
System.IO.File.WriteAllBytes(timeZonesFile, t.Result);
}));
Task.WaitAll(downloadTasks.ToArray());
//downloaded, now convert zip into serialized dat file
if (System.IO.File.Exists(allCountriesFile))
{
if (System.IO.File.Exists(countriesRawFile))
{
System.IO.File.Delete(countriesRawFile);
}
ICSharpCode.SharpZipLib.Zip.FastZip fz = new FastZip();
fz.ExtractZip(allCountriesFile, Root, FastZip.Overwrite.Always, null, null, null, true);
if (System.IO.File.Exists(countriesRawFile))
{
GeoData gd = GeoNameParser.ParseFile(countriesRawFile, timeZonesFile,
featureCodes_enFile, countryInfoFile);
Serialize.SerializeBinaryToDisk(gd, DataFile);
}
if (System.IO.File.Exists(allCountriesFile))
{
System.IO.File.Delete(allCountriesFile);
}
if (System.IO.File.Exists(countriesRawFile))
{
System.IO.File.Delete(countriesRawFile);
}
if (System.IO.File.Exists(countryInfoFile))
{
System.IO.File.Delete(countryInfoFile);
}
if (System.IO.File.Exists(featureCodes_enFile))
{
System.IO.File.Delete(featureCodes_enFile);
}
if (System.IO.File.Exists(timeZonesFile))
{
System.IO.File.Delete(timeZonesFile);
}
}
}
}
}));
}
