コード例 #1
0
        /// <summary>
        /// Initializes a new instance of the <see cref="HttpClientHelper"/> class.
        /// </summary>
        /// <param name="jobSettings">Job settings</param>
        /// <param name="retryPolicy">Retry policy</param>
        public HttpClientHelper(Settings jobSettings, Polly.Retry.AsyncRetryPolicy retryPolicy)
        {
            _settings    = jobSettings;
            _retryPolicy = retryPolicy;

            //Use Tls1.2 as default transport layer
            ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;

            var httpClientHandler = new HttpClientHandler {
                AllowAutoRedirect      = false,
                UseCookies             = false,
                AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
            };

            _httpClient = new HttpClient(httpClientHandler)
            {
                Timeout = TimeSpan.FromMinutes(60) //Timeout for large uploads or downloads
            };

            _authenticationHelper = new AuthenticationHelper(_settings, _retryPolicy);
        }
コード例 #2
0
    private static async Task <WebDirectory> ScanIndexAsync(HttpClient httpClient, WebDirectory webDirectory)
    {
        webDirectory.Parser = Parser;

        try
        {
            Polly.Retry.AsyncRetryPolicy asyncRetryPolicy = Library.GetAsyncRetryPolicy((ex, waitTimeSpan, retry, pollyContext) =>
            {
                Logger.Warn($"Error retrieving directory listing for {webDirectory.Uri}, waiting {waitTimeSpan.TotalSeconds} seconds.. Error: {ex.Message}");
                RateLimiter.AddDelay(waitTimeSpan);
            }, 8);

            if (!webDirectory.Url.EndsWith("/"))
            {
                webDirectory.Url += "/";
            }

            long   pageIndex     = 0;
            string nextPageToken = null;

            do
            {
                await asyncRetryPolicy.ExecuteAndCaptureAsync(async() =>
                {
                    await RateLimiter.RateLimit();

                    Logger.Warn($"Retrieving listings for {webDirectory.Uri.PathAndQuery}, page {pageIndex + 1}{(!string.IsNullOrWhiteSpace(OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password]) ? $" with password: {OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password]}" : string.Empty)}");

                    HttpResponseMessage httpResponseMessage = await httpClient.PostAsync(webDirectory.Uri, new StringContent(JsonConvert.SerializeObject(new Dictionary <string, object>
                    {
                        { "page_index", pageIndex },
                        { "page_token", nextPageToken },
                        { "password", OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password] },
                        { "q", "" }
                    })));

                    webDirectory.ParsedSuccessfully = httpResponseMessage.IsSuccessStatusCode;
                    httpResponseMessage.EnsureSuccessStatusCode();

                    string responseJson = await httpResponseMessage.Content.ReadAsStringAsync();

                    Go2IndexResponse indexResponse = Go2IndexResponse.FromJson(responseJson);

                    webDirectory.ParsedSuccessfully = indexResponse.Error == null;

                    if (indexResponse.Error != null)
                    {
                        throw new Exception($"{indexResponse.Error.Code} | {indexResponse.Error.Message}");
                    }

                    if (indexResponse.Data?.Error != null)
                    {
                        if (indexResponse.Data.Error?.Message == "Rate Limit Exceeded")
                        {
                            throw new Exception("Rate limit exceeded");
                        }
                        else
                        {
                            throw new Exception($"Error in response: {indexResponse.Data.Error?.Code} | {indexResponse.Data.Error?.Message}");
                        }
                    }

                    nextPageToken = indexResponse.NextPageToken;
                    pageIndex     = indexResponse.CurPageIndex + 1;

                    foreach (File file in indexResponse.Data.Files)
                    {
                        if (file.MimeType == FolderMimeType)
                        {
                            webDirectory.Subdirectories.Add(new WebDirectory(webDirectory)
                            {
                                Parser = Parser,
                                // Yes, string concatenation, do not use new Uri(webDirectory.Uri, file.Name), because things could end with a space...
                                Url  = $"{webDirectory.Uri}{Uri.EscapeDataString(file.Name)}/",
                                Name = file.Name
                            });
                        }
                        else
                        {
                            webDirectory.Files.Add(new WebFile
                            {
                                Url      = new Uri(webDirectory.Uri, Uri.EscapeDataString(file.Name)).ToString(),
                                FileName = file.Name,
                                FileSize = file.Size
                            });
                        }
                    }
                });
            } while (!string.IsNullOrWhiteSpace(nextPageToken));
        }
        catch (Exception ex)
        {
            Logger.Error(ex, $"Error retrieving directory listing for {webDirectory.Url}");
            webDirectory.Error = true;

            OpenDirectoryIndexer.Session.Errors++;

            if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url))
            {
                OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url);
            }

            //throw;
        }

        return(webDirectory);
    }
コード例 #3
0
 /// <summary>
 /// Initializes a new instance of the <see cref="AuthenticationHelper"/> class.
 /// </summary>
 /// <param name="jobSettings">Job settings</param>
 /// <param name="retryPolicy">Retry policy</param>
 public AuthenticationHelper(Settings jobSettings, Polly.Retry.AsyncRetryPolicy retryPolicy)
 {
     _settings    = jobSettings;
     _retryPolicy = retryPolicy;
 }
コード例 #4
0
    private static async Task <WebDirectory> ScanAsync(IHtmlDocument htmlDocument, HttpClient httpClient, WebDirectory webDirectory)
    {
        webDirectory.Parser = Parser;

        try
        {
            Polly.Retry.AsyncRetryPolicy asyncRetryPolicy = Library.GetAsyncRetryPolicy((ex, waitTimeSpan, retry, pollyContext) =>
            {
                Logger.Warn($"Error retrieving directory listing for {webDirectory.Uri}, waiting {waitTimeSpan.TotalSeconds} seconds.. Error: {ex.Message}");
                RateLimiter.AddDelay(waitTimeSpan);
            }, 8);

            if (!webDirectory.Url.EndsWith("/"))
            {
                webDirectory.Url += "/";
            }

            long   pageIndex     = 0;
            string nextPageToken = string.Empty;

            do
            {
                await asyncRetryPolicy.ExecuteAndCaptureAsync(async() =>
                {
                    await RateLimiter.RateLimit();

                    Logger.Warn($"Retrieving listings for {webDirectory.Uri.PathAndQuery}, page {pageIndex + 1}{(!string.IsNullOrWhiteSpace(OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password]) ? $" with password: {OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password]}" : string.Empty)}");

                    Dictionary <string, string> postValues = new Dictionary <string, string>
                    {
                        { "password", OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password] },
                        { "page_token", nextPageToken },
                        { "page_index", pageIndex.ToString() }
                    };

                    HttpRequestMessage httpRequestMessage = new HttpRequestMessage(HttpMethod.Post, webDirectory.Uri)
                    {
                        Content = new FormUrlEncodedContent(postValues)
                    };
                    HttpResponseMessage httpResponseMessage = await httpClient.SendAsync(httpRequestMessage);

                    webDirectory.ParsedSuccessfully = httpResponseMessage.IsSuccessStatusCode;
                    httpResponseMessage.EnsureSuccessStatusCode();

                    try
                    {
                        string responseJson = await DecodeResponse(htmlDocument, httpClient, httpResponseMessage);

                        BhadooIndexResponse indexResponse = BhadooIndexResponse.FromJson(responseJson);

                        webDirectory.ParsedSuccessfully = indexResponse.Data.Error == null;

                        if (indexResponse.Data.Error?.Message == "Rate Limit Exceeded")
                        {
                            throw new Exception("Rate limit exceeded");
                        }
                        else
                        {
                            if (indexResponse.Data.Files == null)
                            {
                                throw new Exception("Directory listing retrieval error (Files null)");
                            }
                            else
                            {
                                nextPageToken = indexResponse.NextPageToken;
                                pageIndex     = indexResponse.CurPageIndex + 1;

                                foreach (File file in indexResponse.Data.Files)
                                {
                                    if (file.MimeType == FolderMimeType)
                                    {
                                        webDirectory.Subdirectories.Add(new WebDirectory(webDirectory)
                                        {
                                            Parser = Parser,
                                            // Yes, string concatenation, do not use new Uri(webDirectory.Uri, file.Name), because things could end with a space...
                                            Url  = $"{webDirectory.Uri}{GetSafeName(file.Name)}/",
                                            Name = file.Name
                                        });
                                    }
                                    else
                                    {
                                        webDirectory.Files.Add(new WebFile
                                        {
                                            Url      = new Uri(webDirectory.Uri, GetSafeName(file.Name)).ToString(),
                                            FileName = file.Name,
                                            FileSize = file.Size
                                        });
                                    }
                                }
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        throw new Exception($"Retrieving listings for {webDirectory.Uri.PathAndQuery}, page {pageIndex + 1}. Error: {ex.Message}");
                    }
                });
            } while (!string.IsNullOrWhiteSpace(nextPageToken));
        }
        catch (Exception ex)
        {
            Logger.Error(ex, $"Error retrieving directory listing for {webDirectory.Url}");
            webDirectory.Error = true;

            OpenDirectoryIndexer.Session.Errors++;

            if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url))
            {
                OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url);
            }

            //throw;
        }

        return(webDirectory);
    }
コード例 #5
0
    private static async Task <WebDirectory> ScanIndexAsync(HttpClient httpClient, WebDirectory webDirectory)
    {
        webDirectory.Parser = Parser;

        try
        {
            Polly.Retry.AsyncRetryPolicy asyncRetryPolicy = Library.GetAsyncRetryPolicy((ex, waitTimeSpan, retry, pollyContext) =>
            {
                Logger.Warn($"Error retrieving directory listing for {webDirectory.Uri}, waiting {waitTimeSpan.TotalSeconds} seconds.. Error: {ex.Message}");
                RateLimiter.AddDelay(waitTimeSpan);
            }, 8);

            await asyncRetryPolicy.ExecuteAndCaptureAsync(async() =>
            {
                await RateLimiter.RateLimit();

                if (!webDirectory.Url.EndsWith("/"))
                {
                    webDirectory.Url += "/";
                }

                Logger.Warn($"Retrieving listings for {webDirectory.Uri}");

                HttpResponseMessage httpResponseMessage = await httpClient.PostAsync($"{OpenDirectoryIndexer.Session.Root.Url}{Uri.EscapeDataString(webDirectory.Url.Replace(OpenDirectoryIndexer.Session.Root.Url, string.Empty).TrimEnd('/'))}/?rootId={OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_GdIndex_RootId]}", null);

                webDirectory.ParsedSuccessfully = httpResponseMessage.IsSuccessStatusCode;
                httpResponseMessage.EnsureSuccessStatusCode();

                string responseJson = await httpResponseMessage.Content.ReadAsStringAsync();

                GdIndexResponse indexResponse = GdIndexResponse.FromJson(responseJson);

                webDirectory.ParsedSuccessfully = indexResponse != null;

                foreach (File file in indexResponse.Files)
                {
                    if (file.MimeType == FolderMimeType)
                    {
                        webDirectory.Subdirectories.Add(new WebDirectory(webDirectory)
                        {
                            Parser = Parser,
                            // Yes, string concatenation, do not use new Uri(webDirectory.Uri, file.Name), because things could end with a space...
                            Url  = $"{webDirectory.Uri}{file.Name}/",
                            Name = file.Name
                        });
                    }
                    else
                    {
                        webDirectory.Files.Add(new WebFile
                        {
                            Url      = new Uri(webDirectory.Uri, file.Name).ToString(),
                            FileName = file.Name,
                            FileSize = file.Size
                        });
                    }
                }
            });
        }
        catch (Exception ex)
        {
            Logger.Error(ex, $"Error retrieving directory listing for {webDirectory.Url}");
            webDirectory.Error = true;

            OpenDirectoryIndexer.Session.Errors++;

            if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url))
            {
                OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url);
            }

            //throw;
        }

        return(webDirectory);
    }