public Task Add(NetTask task)
{
return(Task.Run(async() =>
{
await semaphore.WaitAsync().ConfigureAwait(false);
_ = Task.Run(() =>
{
NetField.Do(task);
semaphore.Release();
}).ConfigureAwait(false);
}));
}
public static async Task <List <string> > DownloadStrings(List <string> urls, string cookie = "", Action complete = null, Action error = null)
{
var interrupt = new ManualResetEvent(false);
var result = new string[urls.Count];
var count = urls.Count;
int iter = 0;
foreach (var url in urls)
{
var itertmp = iter;
var task = NetTask.MakeDefault(url);
task.DownloadString = true;
var headers = new Dictionary <string, string>();
// headers["accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
// headers["user-agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.3";
// headers["referer"] = "https://hitomi.la/reader/1234.html";
task.Headers = headers;
task.CompleteCallbackString = (str) =>
{
result[itertmp] = str;
if (Interlocked.Decrement(ref count) == 0)
{
interrupt.Set();
}
complete?.Invoke();
};
task.ErrorCallback = (code) =>
{
if (Interlocked.Decrement(ref count) == 0)
{
interrupt.Set();
}
error?.Invoke();
};
task.Cookie = cookie;
await AppProvider.DownloadQueue.Add(task).ConfigureAwait(false);
iter++;
}
interrupt.WaitOne();
return(result.ToList());
}
public static async Task <List <string> > DownloadStrings(List <string> urls, string cookie = "", Action complete = null, Action error = null)
{
var interrupt = new ManualResetEvent(false);
var result = new string[urls.Count];
var count = urls.Count;
int iter = 0;
foreach (var url in urls)
{
var itertmp = iter;
var task = NetTask.MakeDefault(url);
task.DownloadString = true;
task.CompleteCallbackString = (str) =>
{
result[itertmp] = str;
if (Interlocked.Decrement(ref count) == 0)
{
interrupt.Set();
}
complete?.Invoke();
};
task.ErrorCallback = (code) =>
{
if (Interlocked.Decrement(ref count) == 0)
{
interrupt.Set();
}
error?.Invoke();
};
task.Cookie = cookie;
await Program.DownloadQueue.Add(task).ConfigureAwait(false);
iter++;
}
interrupt.WaitOne();
return(result.ToList());
}
public abstract void RunOnPass(ref NetTask content);
public static void Do(NetTask content)
{
var retry_count = 0;
RETRY_PROCEDURE:
if (content.Cancel != null && content.Cancel.IsCancellationRequested)
{
content.CancleCallback();
return;
}
NetTaskPass.RunOnField(ref content);
REDIRECTION:
if (content.Cancel != null && content.Cancel.IsCancellationRequested)
{
content.CancleCallback();
return;
}
content.StartCallback?.Invoke();
try
{
//
// Initialize http-web-request
//
var request = (HttpWebRequest)WebRequest.Create(content.Url);
content.Request = request;
request.Accept = content.Accept;
request.UserAgent = content.UserAgent;
if (content.Referer != null)
{
request.Referer = content.Referer;
}
else
{
request.Referer = (content.Url.StartsWith("https://") ? "https://" : (content.Url.Split(':')[0] + "//")) + request.RequestUri.Host;
}
if (content.Cookie != null)
{
request.Headers.Add(HttpRequestHeader.Cookie, content.Cookie);
}
if (content.Headers != null)
{
content.Headers.ToList().ForEach(p => request.Headers.Add(p.Key, p.Value));
}
if (content.Proxy != null)
{
request.Proxy = content.Proxy;
}
if (content.TimeoutInfinite)
{
request.Timeout = Timeout.Infinite;
}
else
{
request.Timeout = content.TimeoutMillisecond;
}
request.AllowAutoRedirect = content.AutoRedirection;
//
// POST Data
//
if (content.Query != null)
{
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
var request_stream = new StreamWriter(request.GetRequestStream());
var query = string.Join("&", content.Query.ToList().Select(x => $"{x.Key}={x.Value}"));
request_stream.Write(query);
request_stream.Close();
if (content.Cancel != null && content.Cancel.IsCancellationRequested)
{
content.CancleCallback();
return;
}
}
//
// Wait request
//
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
if (response.StatusCode == HttpStatusCode.NotFound ||
response.StatusCode == HttpStatusCode.Forbidden ||
response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.BadRequest ||
response.StatusCode == HttpStatusCode.InternalServerError)
{
//
// Cannot continue
//
content.ErrorCallback?.Invoke(NetTask.NetError.CannotContinueByCriticalError);
return;
}
else if (response.StatusCode == HttpStatusCode.Moved ||
response.StatusCode == HttpStatusCode.Redirect)
{
if (content.AutoRedirection)
{
var old = content.Url;
content.Url = response.Headers.Get("Location");
goto REDIRECTION;
}
}
else if (response.StatusCode == HttpStatusCode.OK)
{
if (content.Cancel != null && content.Cancel.IsCancellationRequested)
{
content.CancleCallback();
return;
}
content.HeaderReceive?.Invoke(response.Headers.ToString());
content.CookieReceive?.Invoke(response.Cookies);
Stream istream = response.GetResponseStream();
Stream ostream = null;
if (content.DownloadString || content.MemoryCache)
{
ostream = new MemoryStream();
}
else if (content.DriveCache)
{
// TODO:
}
else
{
ostream = File.OpenWrite(content.Filename);
}
content.SizeCallback?.Invoke(response.ContentLength);
if (content.NotifyOnlySize)
{
ostream.Close();
istream.Close();
return;
}
if (content.Cancel != null && content.Cancel.IsCancellationRequested)
{
content.CancleCallback();
return;
}
byte[] buffer = new byte[content.DownloadBufferSize];
long byte_read = 0;
//
// Download loop
//
do
{
if (content.Cancel != null && content.Cancel.IsCancellationRequested)
{
content.CancleCallback();
return;
}
byte_read = istream.Read(buffer, 0, buffer.Length);
ostream.Write(buffer, 0, (int)byte_read);
if (content.Cancel != null && content.Cancel.IsCancellationRequested)
{
content.CancleCallback();
return;
}
content.DownloadCallback?.Invoke(byte_read);
} while (byte_read != 0);
//
// Notify Complete
//
if (content.DownloadString)
{
if (content.Encoding == null)
{
content.CompleteCallbackString(Encoding.UTF8.GetString(((MemoryStream)ostream).ToArray()));
}
else
{
content.CompleteCallbackString(content.Encoding.GetString(((MemoryStream)ostream).ToArray()));
}
}
else if (content.MemoryCache)
{
content.CompleteCallbackBytes(((MemoryStream)ostream).ToArray());
}
else
{
content.CompleteCallback?.Invoke();
}
ostream.Close();
istream.Close();
return;
}
}
}
catch (WebException e)
{
var response = (HttpWebResponse)e.Response;
if (response != null && response.StatusCode == HttpStatusCode.Moved)
{
if (content.AutoRedirection)
{
var old = content.Url;
content.Url = response.Headers.Get("Location");
goto REDIRECTION;
}
}
//lock (Log.Logs.Instance)
//{
// Log.Logs.Instance.PushError("[NetField] Web Excpetion - " + e.Message + "\r\n" + e.StackTrace);
// Log.Logs.Instance.PushError(content);
//}
if (content.FailUrls != null && retry_count < content.FailUrls.Count)
{
content.Url = content.FailUrls[retry_count++];
content.RetryCallback?.Invoke(retry_count);
goto RETRY_PROCEDURE;
}
if ((response != null && (
response.StatusCode == HttpStatusCode.NotFound ||
response.StatusCode == HttpStatusCode.Forbidden ||
response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.BadRequest ||
response.StatusCode == HttpStatusCode.InternalServerError)) ||
e.Status == WebExceptionStatus.NameResolutionFailure ||
e.Status == WebExceptionStatus.UnknownError)
{
if (response != null && response.StatusCode == HttpStatusCode.Forbidden && response.Cookies != null)
{
content.CookieReceive?.Invoke(response.Cookies);
return;
}
//
// Cannot continue
//
if (e.Status == WebExceptionStatus.UnknownError)
{
content.ErrorCallback?.Invoke(NetTask.NetError.UnknowError);
}
else
{
content.ErrorCallback?.Invoke(NetTask.NetError.CannotContinueByCriticalError);
}
return;
}
}
catch (UriFormatException e)
{
//
// Cannot continue
//
content.ErrorCallback?.Invoke(NetTask.NetError.UriFormatError);
return;
}
catch (Exception e)
{
}
//
// Request Aborted
//
if (content.Aborted)
{
content.ErrorCallback?.Invoke(NetTask.NetError.Aborted);
return;
}
//
// Retry
//
if (content.FailUrls != null && retry_count < content.FailUrls.Count)
{
content.Url = content.FailUrls[retry_count++];
content.RetryCallback?.Invoke(retry_count);
goto RETRY_PROCEDURE;
}
if (content.RetryWhenFail)
{
if (content.RetryCount > retry_count)
{
retry_count += 1;
content.RetryCallback?.Invoke(retry_count);
goto RETRY_PROCEDURE;
}
//
// Many retry
//
content.ErrorCallback?.Invoke(NetTask.NetError.ManyRetry);
}
content.ErrorCallback?.Invoke(NetTask.NetError.Unhandled);
}
public static string DownloadString(NetTask task)
{
return(DownloadStringAsync(task).Result);
}
public static string DownloadString(string url)
{
return(DownloadStringAsync(NetTask.MakeDefault(url)).Result);
}