/// <summary>
/// Append task to download queue.
/// </summary>
/// <param name="task"></param>
public Task AppendTask(NetTask task)
{
return(Task.Run(async() =>
{
await semaphore.WaitAsync().ConfigureAwait(false);
_ = Task.Run(() =>
{
NetField.Do(task);
semaphore.Release();
}).ConfigureAwait(false);
}));
}
public abstract void RunOnPass(ref NetTask content);
public static void Do(NetTask content)
{
var retry_count = 0;
RETRY_PROCEDURE:
if (content.Cancel != null && content.Cancel.IsCancellationRequested)
{
content.CancleCallback();
return;
}
NetTaskPass.RunOnField(ref content);
REDIRECTION:
if (content.Cancel != null && content.Cancel.IsCancellationRequested)
{
content.CancleCallback();
return;
}
content.StartCallback?.Invoke();
try
{
//
// Initialize http-web-request
//
var request = (HttpWebRequest)WebRequest.Create(content.Url);
content.Request = request;
request.Accept = content.Accept;
request.UserAgent = content.UserAgent;
if (content.Referer != null)
{
request.Referer = content.Referer;
}
else
{
request.Referer = (content.Url.StartsWith("https://") ? "https://" : (content.Url.Split(':')[0] + "//")) + request.RequestUri.Host;
}
if (content.Cookie != null)
{
request.Headers.Add(HttpRequestHeader.Cookie, content.Cookie);
}
if (content.Headers != null)
{
content.Headers.ToList().ForEach(p => request.Headers.Add(p.Key, p.Value));
}
if (content.Proxy != null)
{
request.Proxy = content.Proxy;
}
if (content.TimeoutInfinite)
{
request.Timeout = Timeout.Infinite;
}
else
{
request.Timeout = content.TimeoutMillisecond;
}
request.AllowAutoRedirect = content.AutoRedirection;
//
// POST Data
//
if (content.Query != null)
{
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
var request_stream = new StreamWriter(request.GetRequestStream());
var query = string.Join("&", content.Query.ToList().Select(x => $"{x.Key}={x.Value}"));
request_stream.Write(query);
request_stream.Close();
if (content.Cancel != null && content.Cancel.IsCancellationRequested)
{
content.CancleCallback();
return;
}
}
else if (content.RequestBody != null)
{
var request_stream = new StreamWriter(request.GetRequestStream());
request_stream.Write(content.RequestBody);
request_stream.Close();
if (content.Cancel != null && content.Cancel.IsCancellationRequested)
{
content.CancleCallback();
return;
}
}
//
// Wait request
//
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
if (response.StatusCode == HttpStatusCode.NotFound ||
response.StatusCode == HttpStatusCode.Forbidden ||
response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.BadRequest ||
response.StatusCode == HttpStatusCode.InternalServerError)
{
//
// Cannot continue
//
content.ErrorCallback?.Invoke(NetTask.NetError.CannotContinueByCriticalError, response.StatusCode);
return;
}
else if (response.StatusCode == HttpStatusCode.Moved ||
response.StatusCode == HttpStatusCode.Redirect)
{
if (content.AutoRedirection)
{
var old = content.Url;
content.Url = response.Headers.Get("Location");
goto REDIRECTION;
}
}
else if (response.StatusCode == HttpStatusCode.OK)
{
if (content.Cancel != null && content.Cancel.IsCancellationRequested)
{
content.CancleCallback();
return;
}
content.HeaderReceive?.Invoke(response.Headers.ToString());
content.CookieReceive?.Invoke(response.Cookies);
Stream istream = response.GetResponseStream();
Stream ostream = null;
if (content.DownloadString || content.MemoryCache)
{
ostream = new MemoryStream();
}
else if (content.DriveCache)
{
// TODO:
}
else
{
ostream = File.OpenWrite(content.Filename);
}
content.SizeCallback?.Invoke(response.ContentLength);
if (content.NotifyOnlySize)
{
ostream.Close();
istream.Close();
return;
}
if (content.Cancel != null && content.Cancel.IsCancellationRequested)
{
content.CancleCallback();
return;
}
byte[] buffer = new byte[content.DownloadBufferSize];
long byte_read = 0;
//
// Download loop
//
do
{
if (content.Cancel != null && content.Cancel.IsCancellationRequested)
{
content.CancleCallback();
return;
}
byte_read = istream.Read(buffer, 0, buffer.Length);
ostream.Write(buffer, 0, (int)byte_read);
if (content.Cancel != null && content.Cancel.IsCancellationRequested)
{
content.CancleCallback();
return;
}
content.DownloadCallback?.Invoke(byte_read);
} while (byte_read != 0);
//
// Notify Complete
//
if (content.DownloadString)
{
if (content.Encoding == null)
{
content.CompleteCallbackString(Encoding.UTF8.GetString(((MemoryStream)ostream).ToArray()));
}
else
{
content.CompleteCallbackString(content.Encoding.GetString(((MemoryStream)ostream).ToArray()));
}
}
else if (content.MemoryCache)
{
content.CompleteCallbackBytes(((MemoryStream)ostream).ToArray());
}
else
{
content.CompleteCallback?.Invoke();
}
ostream.Close();
istream.Close();
return;
}
}
}
catch (WebException e)
{
var response = (HttpWebResponse)e.Response;
if (response != null && response.StatusCode == HttpStatusCode.Moved)
{
if (content.AutoRedirection)
{
var old = content.Url;
content.Url = response.Headers.Get("Location");
goto REDIRECTION;
}
}
if (content.FailUrls != null && retry_count < content.FailUrls.Count)
{
content.Url = content.FailUrls[retry_count++];
content.RetryCallback?.Invoke(retry_count);
goto RETRY_PROCEDURE;
}
if ((response != null && (
response.StatusCode == HttpStatusCode.NotFound ||
response.StatusCode == HttpStatusCode.Forbidden ||
response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.BadRequest ||
response.StatusCode == HttpStatusCode.InternalServerError)) ||
e.Status == WebExceptionStatus.NameResolutionFailure ||
e.Status == WebExceptionStatus.UnknownError)
{
if (response != null && response.StatusCode == HttpStatusCode.Forbidden && response.Cookies != null)
{
content.CookieReceive?.Invoke(response.Cookies);
return;
}
//
// Cannot continue
//
if (e.Status == WebExceptionStatus.UnknownError)
{
content.ErrorCallback?.Invoke(NetTask.NetError.UnknowError, e);
}
else
{
content.ErrorCallback?.Invoke(NetTask.NetError.CannotContinueByCriticalError, e);
}
return;
}
}
catch (UriFormatException e)
{
//
// Cannot continue
//
content.ErrorCallback?.Invoke(NetTask.NetError.UriFormatError, e);
return;
}
catch (Exception e)
{
content.ErrorCallback?.Invoke(NetTask.NetError.UnknowError, e);
}
//
// Request Aborted
//
if (content.Aborted)
{
content.ErrorCallback?.Invoke(NetTask.NetError.Aborted, null);
return;
}
//
// Retry
//
if (content.FailUrls != null && retry_count < content.FailUrls.Count)
{
content.Url = content.FailUrls[retry_count++];
content.RetryCallback?.Invoke(retry_count);
goto RETRY_PROCEDURE;
}
if (content.RetryWhenFail)
{
if (content.RetryCount > retry_count)
{
retry_count += 1;
content.RetryCallback?.Invoke(retry_count);
goto RETRY_PROCEDURE;
}
//
// Many retry
//
content.ErrorCallback?.Invoke(NetTask.NetError.ManyRetry, null);
}
content.ErrorCallback?.Invoke(NetTask.NetError.Unhandled, null);
}
/// <summary>
/// Append task to download queue.
/// </summary>
/// <param name="task"></param>
public void AppendTask(NetTask task)
{
Scheduler.Add(task);
}
/// <summary>
/// Download string from download task.
/// </summary>
/// <param name="task"></param>
/// <returns></returns>
public string DownloadString(NetTask task)
{
return(DownloadStringAsync(task).Result);
}
