private async Task <string> GetUrlRedirection(string url)
{
HttpWebRequest request = webRequestFactory.CreateGetReqeust(url);
cookieService.GetUriCookie(request.CookieContainer, new Uri("https://www.tumblr.com/"));
string location;
using (var response = await request.GetResponseAsync().TimeoutAfter(shellService.Settings.TimeOut) as HttpWebResponse)
{
location = response.ResponseUri.ToString();
}
return(location);
}
protected HttpWebRequest CreateStubReqeust(string url)
{
var request = (HttpWebRequest)WebRequest.Create(url);
request.ProtocolVersion = HttpVersion.Version11;
request.UserAgent =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36";
request.AllowAutoRedirect = true;
request.KeepAlive = true;
request.Pipelined = true;
request.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
// Timeouts don't work with GetResponseAsync() as it internally uses BeginGetResponse.
// See docs: https://msdn.microsoft.com/en-us/library/system.net.httpwebrequest.timeout(v=vs.110).aspx
// Quote: The Timeout property has no effect on asynchronous requests made with the BeginGetResponse or BeginGetRequestStream method.
// TODO: Use HttpClient instead?
request.ReadWriteTimeout = shellService.Settings.TimeOut * 1000;
request.Timeout = shellService.Settings.TimeOut * 1000;
request.CookieContainer = new CookieContainer();
cookieService.GetUriCookie(request.CookieContainer, new Uri("https://www.tumblr.com/"));
cookieService.GetUriCookie(request.CookieContainer, new Uri("https://" + blog.Name.Replace("+", "-") + ".tumblr.com"));
ServicePointManager.DefaultConnectionLimit = 400;
request = SetWebRequestProxy(request, shellService.Settings);
return(request);
}
public void PerformTumblrLogout()
{
var request = webRequestFactory.CreateGetReqeust("https://www.tumblr.com/");
cookieService.GetUriCookie(request.CookieContainer, new Uri("https://www.tumblr.com/"));
cookieService.RemoveUriCookie(new Uri("https://www.tumblr.com"));
var tosCookie =
request.CookieContainer.GetCookies(
new Uri("https://www.tumblr.com/"))["pfg"]; // pfg cookie contains ToS/GDPR agreement
var tosCookieCollection = new CookieCollection
{
tosCookie
};
cookieService.SetUriCookie(tosCookieCollection);
}
protected virtual async Task <string> RequestDataAsync(string url, params string[] cookieHosts)
{
var requestRegistration = new CancellationTokenRegistration();
try
{
HttpWebRequest request = webRequestFactory.CreateGetReqeust(url);
foreach (string cookieHost in cookieHosts)
{
cookieService.GetUriCookie(request.CookieContainer, new Uri(cookieHost));
}
requestRegistration = ct.Register(() => request.Abort());
return(await webRequestFactory.ReadReqestToEnd(request).TimeoutAfter(shellService.Settings.TimeOut));
}
finally
{
requestRegistration.Dispose();
}
}
protected async Task <string> RequestDataAsync(string url, Dictionary <string, string> headers = null, IEnumerable <string> cookieHosts = null)
{
var requestRegistration = new CancellationTokenRegistration();
try
{
HttpWebRequest request = webRequestFactory.CreateGetReqeust(url, "", headers);
cookieHosts = cookieHosts ?? new List <String>();
foreach (string cookieHost in cookieHosts)
{
cookieService.GetUriCookie(request.CookieContainer, new Uri(cookieHost));
}
requestRegistration = ct.Register(() => request.Abort());
return(await webRequestFactory.ReadReqestToEnd(request));
}
finally
{
requestRegistration.Dispose();
}
}
protected virtual async Task <string> RequestDataAsync(string url, params string[] cookieHosts)
{
var requestRegistration = new CancellationTokenRegistration();
try
{
HttpWebRequest request = webRequestFactory.CreateGetReqeust(url);
foreach (string cookieHost in cookieHosts)
{
cookieService.GetUriCookie(request.CookieContainer, new Uri(cookieHost));
}
string username = blog.Name + ".tumblr.com";
string password = blog.Password;
string encoded = System.Convert.ToBase64String(System.Text.Encoding.GetEncoding("ISO-8859-1").GetBytes(username + ":" + password));
request.Headers.Add("Authorization", "Basic " + encoded);
//request.Credentials = new NetworkCredential(blog.Name + ".tumblr.com", blog.Password);
requestRegistration = ct.Register(() => request.Abort());
return(await webRequestFactory.ReadReqestToEnd(request).TimeoutAfter(shellService.Settings.TimeOut));
}
finally
{
requestRegistration.Dispose();
}
}
private async Task <string> RequestTumblrKey()
{
const string url = "https://www.tumblr.com/login";
var request = webRequestFactory.CreateGetRequest(url);
cookieService.GetUriCookie(request.CookieContainer, new Uri("https://www.tumblr.com/"));
using (var response = await request.GetResponseAsync().TimeoutAfter(shellService.Settings.TimeOut).ConfigureAwait(false) as HttpWebResponse)
{
cookieService.SetUriCookie(response.Cookies);
using (var stream = webRequestFactory.GetStreamForApiRequest(response.GetResponseStream()))
{
using (var buffer = new BufferedStream(stream))
{
using (var reader = new StreamReader(buffer))
{
return(reader.ReadToEnd());
}
}
}
}
}
// TODO: Needs a complete rewrite. Also a append/cache function for resuming incomplete files on the disk.
// Should be in separated class with support for events for downloadspeed, is resumable file?, etc.
// Should check if file is complete, else it will trigger an WebException -- 416 requested range not satisfiable at every request
public async Task <bool> DownloadFileWithResumeAsync(string url, string destinationPath)
{
long totalBytesReceived = 0;
var attemptCount = 0;
int bufferSize = settings.BufferSize * 4096;
if (File.Exists(destinationPath))
{
var fileInfo = new FileInfo(destinationPath);
totalBytesReceived = fileInfo.Length;
if (totalBytesReceived >= await CheckDownloadSizeAsync(url).TimeoutAfter(settings.TimeOut))
{
return(true);
}
}
if (ct.IsCancellationRequested)
{
return(false);
}
FileMode fileMode = totalBytesReceived > 0 ? FileMode.Append : FileMode.Create;
using (var fileStream = new FileStream(destinationPath, fileMode, FileAccess.Write, FileShare.Read, bufferSize, true))
{
while (true)
{
attemptCount += 1;
if (attemptCount > settings.MaxNumberOfRetries)
{
return(false);
}
var requestRegistration = new CancellationTokenRegistration();
try
{
HttpWebRequest request = webRequestFactory.CreateGetReqeust(url);
cookieService.GetUriCookie(request.CookieContainer, new Uri("https://www.tumblr.com/"));
requestRegistration = ct.Register(() => request.Abort());
request.AddRange(totalBytesReceived);
long totalBytesToReceive = 0;
using (WebResponse response = await request.GetResponseAsync().TimeoutAfter(settings.TimeOut))
{
totalBytesToReceive = totalBytesReceived + response.ContentLength;
using (Stream responseStream = response.GetResponseStream())
{
using (Stream throttledStream = GetStreamForDownload(responseStream))
{
var buffer = new byte[4096];
var bytesRead = 0;
//Stopwatch sw = Stopwatch.StartNew();
while ((bytesRead = await throttledStream.ReadAsync(buffer, 0, buffer.Length, ct).TimeoutAfter(settings.TimeOut)) > 0)
{
await fileStream.WriteAsync(buffer, 0, bytesRead);
totalBytesReceived += bytesRead;
//float currentSpeed = totalBytesReceived / (float)sw.Elapsed.TotalSeconds;
//OnProgressChanged(new DownloadProgressChangedEventArgs(totalBytesReceived,
// totalBytesToReceive, (long)currentSpeed));
}
}
}
}
if (totalBytesReceived >= totalBytesToReceive)
{
break;
}
}
catch (IOException ioException)
{
// file in use
long win32ErrorCode = ioException.HResult & 0xFFFF;
if (win32ErrorCode == 0x21 || win32ErrorCode == 0x20)
{
return(false);
}
// retry (IOException: Received an unexpected EOF or 0 bytes from the transport stream)
}
catch (WebException webException)
{
if (webException.Status == WebExceptionStatus.ConnectionClosed)
{
// retry
}
else
{
throw;
}
}
finally
{
requestRegistration.Dispose();
}
}
return(true);
}
}