예제 #1
0
 public Task Add(NetTask task)
 {
     return(Task.Run(async() =>
     {
         await semaphore.WaitAsync().ConfigureAwait(false);
         _ = Task.Run(() =>
         {
             NetField.Do(task);
             semaphore.Release();
         }).ConfigureAwait(false);
     }));
 }
예제 #2
0
        public static async Task <List <string> > DownloadStrings(List <string> urls, string cookie = "", Action complete = null, Action error = null)
        {
            var interrupt = new ManualResetEvent(false);
            var result    = new string[urls.Count];
            var count     = urls.Count;
            int iter      = 0;

            foreach (var url in urls)
            {
                var itertmp = iter;
                var task    = NetTask.MakeDefault(url);
                task.DownloadString = true;
                var headers = new Dictionary <string, string>();
                // headers["accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
                // headers["user-agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.3";
                // headers["referer"] = "https://hitomi.la/reader/1234.html";
                task.Headers = headers;
                task.CompleteCallbackString = (str) =>
                {
                    result[itertmp] = str;
                    if (Interlocked.Decrement(ref count) == 0)
                    {
                        interrupt.Set();
                    }
                    complete?.Invoke();
                };
                task.ErrorCallback = (code) =>
                {
                    if (Interlocked.Decrement(ref count) == 0)
                    {
                        interrupt.Set();
                    }
                    error?.Invoke();
                };
                task.Cookie = cookie;
                await AppProvider.DownloadQueue.Add(task).ConfigureAwait(false);

                iter++;
            }

            interrupt.WaitOne();

            return(result.ToList());
        }
예제 #3
0
        public static async Task <List <string> > DownloadStrings(List <string> urls, string cookie = "", Action complete = null, Action error = null)
        {
            var interrupt = new ManualResetEvent(false);
            var result    = new string[urls.Count];
            var count     = urls.Count;
            int iter      = 0;

            foreach (var url in urls)
            {
                var itertmp = iter;
                var task    = NetTask.MakeDefault(url);
                task.DownloadString         = true;
                task.CompleteCallbackString = (str) =>
                {
                    result[itertmp] = str;
                    if (Interlocked.Decrement(ref count) == 0)
                    {
                        interrupt.Set();
                    }
                    complete?.Invoke();
                };
                task.ErrorCallback = (code) =>
                {
                    if (Interlocked.Decrement(ref count) == 0)
                    {
                        interrupt.Set();
                    }
                    error?.Invoke();
                };
                task.Cookie = cookie;
                await Program.DownloadQueue.Add(task).ConfigureAwait(false);

                iter++;
            }

            interrupt.WaitOne();

            return(result.ToList());
        }
예제 #4
0
 public abstract void RunOnPass(ref NetTask content);
예제 #5
0
        public static void Do(NetTask content)
        {
            var retry_count = 0;

RETRY_PROCEDURE:

            if (content.Cancel != null && content.Cancel.IsCancellationRequested)
            {
                content.CancleCallback();
                return;
            }

            NetTaskPass.RunOnField(ref content);

REDIRECTION:

            if (content.Cancel != null && content.Cancel.IsCancellationRequested)
            {
                content.CancleCallback();
                return;
            }

            content.StartCallback?.Invoke();

            try
            {
                //
                //  Initialize http-web-request
                //

                var request = (HttpWebRequest)WebRequest.Create(content.Url);
                content.Request = request;

                request.Accept    = content.Accept;
                request.UserAgent = content.UserAgent;

                if (content.Referer != null)
                {
                    request.Referer = content.Referer;
                }
                else
                {
                    request.Referer = (content.Url.StartsWith("https://") ? "https://" : (content.Url.Split(':')[0] + "//")) + request.RequestUri.Host;
                }

                if (content.Cookie != null)
                {
                    request.Headers.Add(HttpRequestHeader.Cookie, content.Cookie);
                }

                if (content.Headers != null)
                {
                    content.Headers.ToList().ForEach(p => request.Headers.Add(p.Key, p.Value));
                }

                if (content.Proxy != null)
                {
                    request.Proxy = content.Proxy;
                }

                if (content.TimeoutInfinite)
                {
                    request.Timeout = Timeout.Infinite;
                }
                else
                {
                    request.Timeout = content.TimeoutMillisecond;
                }

                request.AllowAutoRedirect = content.AutoRedirection;

                //
                //  POST Data
                //

                if (content.Query != null)
                {
                    request.Method      = "POST";
                    request.ContentType = "application/x-www-form-urlencoded";

                    var request_stream = new StreamWriter(request.GetRequestStream());
                    var query          = string.Join("&", content.Query.ToList().Select(x => $"{x.Key}={x.Value}"));
                    request_stream.Write(query);
                    request_stream.Close();

                    if (content.Cancel != null && content.Cancel.IsCancellationRequested)
                    {
                        content.CancleCallback();
                        return;
                    }
                }

                //
                //  Wait request
                //

                using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
                {
                    if (response.StatusCode == HttpStatusCode.NotFound ||
                        response.StatusCode == HttpStatusCode.Forbidden ||
                        response.StatusCode == HttpStatusCode.Unauthorized ||
                        response.StatusCode == HttpStatusCode.BadRequest ||
                        response.StatusCode == HttpStatusCode.InternalServerError)
                    {
                        //
                        //  Cannot continue
                        //

                        content.ErrorCallback?.Invoke(NetTask.NetError.CannotContinueByCriticalError);
                        return;
                    }
                    else if (response.StatusCode == HttpStatusCode.Moved ||
                             response.StatusCode == HttpStatusCode.Redirect)
                    {
                        if (content.AutoRedirection)
                        {
                            var old = content.Url;
                            content.Url = response.Headers.Get("Location");
                            goto REDIRECTION;
                        }
                    }
                    else if (response.StatusCode == HttpStatusCode.OK)
                    {
                        if (content.Cancel != null && content.Cancel.IsCancellationRequested)
                        {
                            content.CancleCallback();
                            return;
                        }

                        content.HeaderReceive?.Invoke(response.Headers.ToString());
                        content.CookieReceive?.Invoke(response.Cookies);

                        Stream istream = response.GetResponseStream();
                        Stream ostream = null;

                        if (content.DownloadString || content.MemoryCache)
                        {
                            ostream = new MemoryStream();
                        }
                        else if (content.DriveCache)
                        {
                            // TODO:
                        }
                        else
                        {
                            ostream = File.OpenWrite(content.Filename);
                        }

                        content.SizeCallback?.Invoke(response.ContentLength);

                        if (content.NotifyOnlySize)
                        {
                            ostream.Close();
                            istream.Close();
                            return;
                        }

                        if (content.Cancel != null && content.Cancel.IsCancellationRequested)
                        {
                            content.CancleCallback();
                            return;
                        }

                        byte[] buffer    = new byte[content.DownloadBufferSize];
                        long   byte_read = 0;

                        //
                        //  Download loop
                        //

                        do
                        {
                            if (content.Cancel != null && content.Cancel.IsCancellationRequested)
                            {
                                content.CancleCallback();
                                return;
                            }

                            byte_read = istream.Read(buffer, 0, buffer.Length);
                            ostream.Write(buffer, 0, (int)byte_read);

                            if (content.Cancel != null && content.Cancel.IsCancellationRequested)
                            {
                                content.CancleCallback();
                                return;
                            }

                            content.DownloadCallback?.Invoke(byte_read);
                        } while (byte_read != 0);

                        //
                        //  Notify Complete
                        //

                        if (content.DownloadString)
                        {
                            if (content.Encoding == null)
                            {
                                content.CompleteCallbackString(Encoding.UTF8.GetString(((MemoryStream)ostream).ToArray()));
                            }
                            else
                            {
                                content.CompleteCallbackString(content.Encoding.GetString(((MemoryStream)ostream).ToArray()));
                            }
                        }
                        else if (content.MemoryCache)
                        {
                            content.CompleteCallbackBytes(((MemoryStream)ostream).ToArray());
                        }
                        else
                        {
                            content.CompleteCallback?.Invoke();
                        }

                        ostream.Close();
                        istream.Close();

                        return;
                    }
                }
            }
            catch (WebException e)
            {
                var response = (HttpWebResponse)e.Response;

                if (response != null && response.StatusCode == HttpStatusCode.Moved)
                {
                    if (content.AutoRedirection)
                    {
                        var old = content.Url;
                        content.Url = response.Headers.Get("Location");
                        goto REDIRECTION;
                    }
                }

                //lock (Log.Logs.Instance)
                //{
                //    Log.Logs.Instance.PushError("[NetField] Web Excpetion - " + e.Message + "\r\n" + e.StackTrace);
                //    Log.Logs.Instance.PushError(content);
                //}

                if (content.FailUrls != null && retry_count < content.FailUrls.Count)
                {
                    content.Url = content.FailUrls[retry_count++];
                    content.RetryCallback?.Invoke(retry_count);

                    goto RETRY_PROCEDURE;
                }

                if ((response != null && (
                         response.StatusCode == HttpStatusCode.NotFound ||
                         response.StatusCode == HttpStatusCode.Forbidden ||
                         response.StatusCode == HttpStatusCode.Unauthorized ||
                         response.StatusCode == HttpStatusCode.BadRequest ||
                         response.StatusCode == HttpStatusCode.InternalServerError)) ||
                    e.Status == WebExceptionStatus.NameResolutionFailure ||
                    e.Status == WebExceptionStatus.UnknownError)
                {
                    if (response != null && response.StatusCode == HttpStatusCode.Forbidden && response.Cookies != null)
                    {
                        content.CookieReceive?.Invoke(response.Cookies);
                        return;
                    }

                    //
                    //  Cannot continue
                    //

                    if (e.Status == WebExceptionStatus.UnknownError)
                    {
                        content.ErrorCallback?.Invoke(NetTask.NetError.UnknowError);
                    }
                    else
                    {
                        content.ErrorCallback?.Invoke(NetTask.NetError.CannotContinueByCriticalError);
                    }

                    return;
                }
            }
            catch (UriFormatException e)
            {
                //
                //  Cannot continue
                //

                content.ErrorCallback?.Invoke(NetTask.NetError.UriFormatError);
                return;
            }
            catch (Exception e)
            {
            }

            //
            //  Request Aborted
            //

            if (content.Aborted)
            {
                content.ErrorCallback?.Invoke(NetTask.NetError.Aborted);
                return;
            }

            //
            //  Retry
            //

            if (content.FailUrls != null && retry_count < content.FailUrls.Count)
            {
                content.Url = content.FailUrls[retry_count++];
                content.RetryCallback?.Invoke(retry_count);

                goto RETRY_PROCEDURE;
            }

            if (content.RetryWhenFail)
            {
                if (content.RetryCount > retry_count)
                {
                    retry_count += 1;

                    content.RetryCallback?.Invoke(retry_count);

                    goto RETRY_PROCEDURE;
                }

                //
                //  Many retry
                //

                content.ErrorCallback?.Invoke(NetTask.NetError.ManyRetry);
            }

            content.ErrorCallback?.Invoke(NetTask.NetError.Unhandled);
        }
예제 #6
0
 public static string DownloadString(NetTask task)
 {
     return(DownloadStringAsync(task).Result);
 }
예제 #7
0
 public static string DownloadString(string url)
 {
     return(DownloadStringAsync(NetTask.MakeDefault(url)).Result);
 }