/// <summary> /// Append task to download queue. /// </summary> /// <param name="task"></param> public Task AppendTask(NetTask task) { return(Task.Run(async() => { await semaphore.WaitAsync().ConfigureAwait(false); _ = Task.Run(() => { NetField.Do(task); semaphore.Release(); }).ConfigureAwait(false); })); }
public abstract void RunOnPass(ref NetTask content);
public static void Do(NetTask content) { var retry_count = 0; RETRY_PROCEDURE: if (content.Cancel != null && content.Cancel.IsCancellationRequested) { content.CancleCallback(); return; } NetTaskPass.RunOnField(ref content); REDIRECTION: if (content.Cancel != null && content.Cancel.IsCancellationRequested) { content.CancleCallback(); return; } content.StartCallback?.Invoke(); try { // // Initialize http-web-request // var request = (HttpWebRequest)WebRequest.Create(content.Url); content.Request = request; request.Accept = content.Accept; request.UserAgent = content.UserAgent; if (content.Referer != null) { request.Referer = content.Referer; } else { request.Referer = (content.Url.StartsWith("https://") ? "https://" : (content.Url.Split(':')[0] + "//")) + request.RequestUri.Host; } if (content.Cookie != null) { request.Headers.Add(HttpRequestHeader.Cookie, content.Cookie); } if (content.Headers != null) { content.Headers.ToList().ForEach(p => request.Headers.Add(p.Key, p.Value)); } if (content.Proxy != null) { request.Proxy = content.Proxy; } if (content.TimeoutInfinite) { request.Timeout = Timeout.Infinite; } else { request.Timeout = content.TimeoutMillisecond; } request.AllowAutoRedirect = content.AutoRedirection; // // POST Data // if (content.Query != null) { request.Method = "POST"; request.ContentType = "application/x-www-form-urlencoded"; var request_stream = new StreamWriter(request.GetRequestStream()); var query = string.Join("&", content.Query.ToList().Select(x => $"{x.Key}={x.Value}")); request_stream.Write(query); request_stream.Close(); if (content.Cancel != null && content.Cancel.IsCancellationRequested) { content.CancleCallback(); return; } } else if (content.RequestBody != null) { var request_stream = new StreamWriter(request.GetRequestStream()); request_stream.Write(content.RequestBody); request_stream.Close(); if (content.Cancel != null && content.Cancel.IsCancellationRequested) { content.CancleCallback(); return; } } // // Wait request // using (HttpWebResponse response = (HttpWebResponse)request.GetResponse()) { if (response.StatusCode == HttpStatusCode.NotFound || response.StatusCode == HttpStatusCode.Forbidden || response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.InternalServerError) { // // Cannot continue // content.ErrorCallback?.Invoke(NetTask.NetError.CannotContinueByCriticalError, response.StatusCode); return; } else if (response.StatusCode == HttpStatusCode.Moved || response.StatusCode == HttpStatusCode.Redirect) { if (content.AutoRedirection) { var old = content.Url; content.Url = response.Headers.Get("Location"); goto REDIRECTION; } } else if (response.StatusCode == HttpStatusCode.OK) { if (content.Cancel != null && content.Cancel.IsCancellationRequested) { content.CancleCallback(); return; } content.HeaderReceive?.Invoke(response.Headers.ToString()); content.CookieReceive?.Invoke(response.Cookies); Stream istream = response.GetResponseStream(); Stream ostream = null; if (content.DownloadString || content.MemoryCache) { ostream = new MemoryStream(); } else if (content.DriveCache) { // TODO: } else { ostream = File.OpenWrite(content.Filename); } content.SizeCallback?.Invoke(response.ContentLength); if (content.NotifyOnlySize) { ostream.Close(); istream.Close(); return; } if (content.Cancel != null && content.Cancel.IsCancellationRequested) { content.CancleCallback(); return; } byte[] buffer = new byte[content.DownloadBufferSize]; long byte_read = 0; // // Download loop // do { if (content.Cancel != null && content.Cancel.IsCancellationRequested) { content.CancleCallback(); return; } byte_read = istream.Read(buffer, 0, buffer.Length); ostream.Write(buffer, 0, (int)byte_read); if (content.Cancel != null && content.Cancel.IsCancellationRequested) { content.CancleCallback(); return; } content.DownloadCallback?.Invoke(byte_read); } while (byte_read != 0); // // Notify Complete // if (content.DownloadString) { if (content.Encoding == null) { content.CompleteCallbackString(Encoding.UTF8.GetString(((MemoryStream)ostream).ToArray())); } else { content.CompleteCallbackString(content.Encoding.GetString(((MemoryStream)ostream).ToArray())); } } else if (content.MemoryCache) { content.CompleteCallbackBytes(((MemoryStream)ostream).ToArray()); } else { content.CompleteCallback?.Invoke(); } ostream.Close(); istream.Close(); return; } } } catch (WebException e) { var response = (HttpWebResponse)e.Response; if (response != null && response.StatusCode == HttpStatusCode.Moved) { if (content.AutoRedirection) { var old = content.Url; content.Url = response.Headers.Get("Location"); goto REDIRECTION; } } if (content.FailUrls != null && retry_count < content.FailUrls.Count) { content.Url = content.FailUrls[retry_count++]; content.RetryCallback?.Invoke(retry_count); goto RETRY_PROCEDURE; } if ((response != null && ( response.StatusCode == HttpStatusCode.NotFound || response.StatusCode == HttpStatusCode.Forbidden || response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.InternalServerError)) || e.Status == WebExceptionStatus.NameResolutionFailure || e.Status == WebExceptionStatus.UnknownError) { if (response != null && response.StatusCode == HttpStatusCode.Forbidden && response.Cookies != null) { content.CookieReceive?.Invoke(response.Cookies); return; } // // Cannot continue // if (e.Status == WebExceptionStatus.UnknownError) { content.ErrorCallback?.Invoke(NetTask.NetError.UnknowError, e); } else { content.ErrorCallback?.Invoke(NetTask.NetError.CannotContinueByCriticalError, e); } return; } } catch (UriFormatException e) { // // Cannot continue // content.ErrorCallback?.Invoke(NetTask.NetError.UriFormatError, e); return; } catch (Exception e) { content.ErrorCallback?.Invoke(NetTask.NetError.UnknowError, e); } // // Request Aborted // if (content.Aborted) { content.ErrorCallback?.Invoke(NetTask.NetError.Aborted, null); return; } // // Retry // if (content.FailUrls != null && retry_count < content.FailUrls.Count) { content.Url = content.FailUrls[retry_count++]; content.RetryCallback?.Invoke(retry_count); goto RETRY_PROCEDURE; } if (content.RetryWhenFail) { if (content.RetryCount > retry_count) { retry_count += 1; content.RetryCallback?.Invoke(retry_count); goto RETRY_PROCEDURE; } // // Many retry // content.ErrorCallback?.Invoke(NetTask.NetError.ManyRetry, null); } content.ErrorCallback?.Invoke(NetTask.NetError.Unhandled, null); }
/// <summary> /// Append task to download queue. /// </summary> /// <param name="task"></param> public void AppendTask(NetTask task) { Scheduler.Add(task); }
/// <summary> /// Download string from download task. /// </summary> /// <param name="task"></param> /// <returns></returns> public string DownloadString(NetTask task) { return(DownloadStringAsync(task).Result); }