Пример #1
0
        private async Task ProcessWebDirectoryAsync(string name, WebDirectory webDirectory, CancellationToken cancellationToken)
        {
            if (Session.Parameters.ContainsKey(Constants.Parameters_GdIndex_RootId))
            {
                await Site.GoIndex.GdIndex.GdIndexParser.ParseIndex(HttpClient, webDirectory, string.Empty);

                return;
            }

            if (!string.IsNullOrWhiteSpace(OpenDirectoryIndexerSettings.CommandLineOptions.UserAgent))
            {
                HttpClient.DefaultRequestHeaders.UserAgent.Clear();
                HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd(OpenDirectoryIndexerSettings.CommandLineOptions.UserAgent);
            }

            HttpResponseMessage httpResponseMessage = await HttpClient.GetAsync(webDirectory.Url, cancellationToken);

            string html = null;

            if (httpResponseMessage.IsSuccessStatusCode)
            {
                SetRootUrl(httpResponseMessage);

                html = await GetHtml(httpResponseMessage);
            }

            if (FirstRequest && !httpResponseMessage.IsSuccessStatusCode || httpResponseMessage.IsSuccessStatusCode && string.IsNullOrWhiteSpace(html) || html?.Contains("HTTP_USER_AGENT") == true)
            {
                Logger.Warn("First request fails, using Curl fallback User-Agent");
                HttpClient.DefaultRequestHeaders.UserAgent.Clear();
                HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd(Constants.UserAgent.Curl);
                httpResponseMessage = await HttpClient.GetAsync(webDirectory.Url, cancellationToken);

                if (httpResponseMessage.IsSuccessStatusCode)
                {
                    SetRootUrl(httpResponseMessage);

                    html = await GetHtml(httpResponseMessage);

                    Logger.Warn("Yes, Curl User-Agent did the trick!");
                }
            }

            if (FirstRequest && !httpResponseMessage.IsSuccessStatusCode || httpResponseMessage.IsSuccessStatusCode && string.IsNullOrWhiteSpace(html))
            {
                Logger.Warn("First request fails, using Chrome fallback User-Agent");
                HttpClient.DefaultRequestHeaders.UserAgent.Clear();
                HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd(Constants.UserAgent.Chrome);
                httpResponseMessage = await HttpClient.GetAsync(webDirectory.Url, cancellationToken);

                if (httpResponseMessage.IsSuccessStatusCode)
                {
                    SetRootUrl(httpResponseMessage);

                    html = await GetHtml(httpResponseMessage);

                    Logger.Warn("Yes, Chrome User-Agent did the trick!");
                }
            }

            if (!HttpClient.DefaultRequestHeaders.Contains("Referer"))
            {
                HttpClient.DefaultRequestHeaders.Add("Referer", webDirectory.Url);
            }

            bool   calibreDetected      = false;
            string calibreVersionString = string.Empty;

            if (httpResponseMessage.IsSuccessStatusCode)
            {
                FirstRequest = false;

                List <string> serverHeaders = new List <string>();

                if (httpResponseMessage.Headers.Contains("Server"))
                {
                    serverHeaders = httpResponseMessage.Headers.GetValues("Server").ToList();

                    calibreDetected = serverHeaders.Any(h => h.Contains("calibre"));
                }

                if (calibreDetected)
                {
                    string serverHeader = string.Join("/", serverHeaders);
                    calibreVersionString = serverHeader;
                }
                else
                {
                    if (html == null)
                    {
                        html = await GetHtml(httpResponseMessage);
                    }

                    // UNTESTED (cannot find or down Calibre with this issue)
                    const string calibreVersionIdentifier = "CALIBRE_VERSION = \"";
                    calibreDetected = html?.Contains(calibreVersionIdentifier) == true;

                    if (calibreDetected)
                    {
                        int calibreVersionIdentifierStart = html.IndexOf(calibreVersionIdentifier);
                        calibreVersionString = html.Substring(calibreVersionIdentifierStart, html.IndexOf("\"", ++calibreVersionIdentifierStart));
                    }
                }
            }

            if (calibreDetected)
            {
                Version calibreVersion = CalibreParser.ParseVersion(calibreVersionString);

                Console.WriteLine($"Calibre {calibreVersion} detected! I will index it at max 100 books per 30 seconds, else it will break Calibre...");
                Logger.Info($"Calibre {calibreVersion} detected! I will index it at max 100 books per 30 seconds, else it will break Calibre...");

                await CalibreParser.ParseCalibre(HttpClient, httpResponseMessage.RequestMessage.RequestUri, webDirectory, calibreVersion, cancellationToken);

                return;
            }

            if (httpResponseMessage.IsSuccessStatusCode && webDirectory.Url != httpResponseMessage.RequestMessage.RequestUri.ToString())
            {
                webDirectory.Url = httpResponseMessage.RequestMessage.RequestUri.ToString();
            }

            Uri originalUri = new Uri(webDirectory.Url);

            Logger.Debug($"[{name}] Finish download '{webDirectory.Url}'");

            // Process only same site
            if (httpResponseMessage.RequestMessage.RequestUri.Host == Session.Root.Uri.Host)
            {
                int httpStatusCode = (int)httpResponseMessage.StatusCode;

                if (!Session.HttpStatusCodes.ContainsKey(httpStatusCode))
                {
                    Session.HttpStatusCodes[httpStatusCode] = 0;
                }

                Session.HttpStatusCodes[httpStatusCode]++;

                if (httpResponseMessage.IsSuccessStatusCode)
                {
                    if (html == null)
                    {
                        html = await GetHtml(httpResponseMessage);
                    }

                    Session.TotalHttpTraffic += html.Length;

                    WebDirectory parsedWebDirectory = await DirectoryParser.ParseHtml(webDirectory, html, HttpClient);

                    bool processSubdirectories = parsedWebDirectory.Parser != "DirectoryListingModel01";
                    AddProcessedWebDirectory(webDirectory, parsedWebDirectory, processSubdirectories);
                }
                else
                {
                    httpResponseMessage.EnsureSuccessStatusCode();
                }
            }
            else
            {
                Logger.Warn($"[{name}] Skipped result of '{webDirectory.Url}' which points to '{httpResponseMessage.RequestMessage.RequestUri}'");
                Session.Skipped++;
            }
        }
        private async Task WebDirectoryProcessor(ConcurrentQueue <WebDirectory> queue, string name, CancellationToken token)
        {
            Logger.Debug($"Start [{name}]");

            do
            {
                Interlocked.Increment(ref RunningWebDirectoryThreads);

                if (queue.TryDequeue(out WebDirectory webDirectory))
                {
                    try
                    {
                        lock (WebDirectoryProcessorInfoLock)
                        {
                            WebDirectoryProcessorInfo[name] = webDirectory;
                        }

                        if (!Session.ProcessedUrls.Contains(webDirectory.Url))
                        {
                            Session.ProcessedUrls.Add(webDirectory.Url);
                            Logger.Info($"[{name}] Begin processing {webDirectory.Url}");

                            if (Session.Root.Uri.Scheme == "ftp")
                            {
                                WebDirectory parsedWebDirectory = await FtpParser.ParseFtpAsync(name, webDirectory);

                                AddProcessedWebDirectory(webDirectory, parsedWebDirectory);
                            }
                            else
                            if (Session.Root.Uri.Host == "drive.google.com")
                            {
                                string baseUrl = webDirectory.Url;

                                WebDirectory parsedWebDirectory = await GoogleDriveIndexer.IndexAsync(webDirectory);

                                parsedWebDirectory.Url = baseUrl;

                                AddProcessedWebDirectory(webDirectory, parsedWebDirectory);
                            }
                            else
                            {
                                if (webDirectory.Uri.Host == Session.Root.Uri.Host && webDirectory.Uri.LocalPath.StartsWith(Session.Root.Uri.LocalPath))
                                {
                                    Logger.Debug($"[{name}] Start download '{webDirectory.Url}'");
                                    Session.TotalHttpRequests++;

                                    await RetryPolicy.ExecuteAsync(async() =>
                                    {
                                        webDirectory.StartTime = DateTimeOffset.UtcNow;

                                        HttpResponseMessage httpResponseMessage = await HttpClient.GetAsync(webDirectory.Url);
                                        string html = null;

                                        if (httpResponseMessage.IsSuccessStatusCode)
                                        {
                                            html = await GetHtml(httpResponseMessage);
                                        }

                                        if (FirstRequest && !httpResponseMessage.IsSuccessStatusCode || httpResponseMessage.IsSuccessStatusCode && string.IsNullOrWhiteSpace(html))
                                        {
                                            Logger.Warn("First request fails, using Curl fallback User-Agent");
                                            HttpClient.DefaultRequestHeaders.UserAgent.Clear();
                                            HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent_Curl);
                                            httpResponseMessage = await HttpClient.GetAsync(webDirectory.Url);

                                            if (httpResponseMessage.IsSuccessStatusCode)
                                            {
                                                html = await GetHtml(httpResponseMessage);
                                                Logger.Warn("Yes, this Curl User-Agent did the trick!");
                                            }
                                        }

                                        if (FirstRequest && !httpResponseMessage.IsSuccessStatusCode || httpResponseMessage.IsSuccessStatusCode && string.IsNullOrWhiteSpace(html))
                                        {
                                            Logger.Warn("First request fails, using Chrome fallback User-Agent");
                                            HttpClient.DefaultRequestHeaders.UserAgent.Clear();
                                            HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent_Chrome);
                                            httpResponseMessage = await HttpClient.GetAsync(webDirectory.Url);

                                            if (httpResponseMessage.IsSuccessStatusCode)
                                            {
                                                html = await GetHtml(httpResponseMessage);
                                                Logger.Warn("Yes, the Chrome User-Agent did the trick!");
                                            }
                                        }

                                        bool calibreDetected        = false;
                                        string calibreVersionString = string.Empty;

                                        if (httpResponseMessage.IsSuccessStatusCode)
                                        {
                                            FirstRequest = false;

                                            List <string> serverHeaders = new List <string>();

                                            if (httpResponseMessage.Headers.Contains("Server"))
                                            {
                                                serverHeaders = httpResponseMessage.Headers.GetValues("Server").ToList();

                                                calibreDetected = serverHeaders.Any(h => h.Contains("calibre"));
                                            }

                                            if (calibreDetected)
                                            {
                                                string serverHeader  = string.Join("/", serverHeaders);
                                                calibreVersionString = serverHeader;
                                            }
                                            else
                                            {
                                                if (html == null)
                                                {
                                                    html = await GetHtml(httpResponseMessage);
                                                }

                                                // UNTESTED (cannot find or down Calibre with this issue)
                                                const string calibreVersionIdentifier = "CALIBRE_VERSION = \"";
                                                calibreDetected = html?.Contains(calibreVersionIdentifier) == true;

                                                if (calibreDetected)
                                                {
                                                    int calibreVersionIdentifierStart = html.IndexOf(calibreVersionIdentifier);
                                                    calibreVersionString = html.Substring(calibreVersionIdentifierStart, html.IndexOf("\"", ++calibreVersionIdentifierStart));
                                                }
                                            }
                                        }

                                        if (calibreDetected)
                                        {
                                            Version calibreVersion = CalibreParser.ParseVersion(calibreVersionString);

                                            Console.WriteLine($"Calibre {calibreVersion} detected! I will index it at max 100 books per 30 seconds, else it will break Calibre...");
                                            Logger.Info($"Calibre {calibreVersion} detected! I will index it at max 100 books per 30 seconds, else it will break Calibre...");

                                            await CalibreParser.ParseCalibre(HttpClient, httpResponseMessage.RequestMessage.RequestUri, webDirectory, calibreVersion);

                                            return;
                                        }

                                        Uri originalUri = new Uri(webDirectory.Url);
                                        Logger.Debug($"[{name}] Finish download '{webDirectory.Url}'");

                                        // Process only same site
                                        if (httpResponseMessage.RequestMessage.RequestUri.Host == Session.Root.Uri.Host)
                                        {
                                            int httpStatusCode = (int)httpResponseMessage.StatusCode;

                                            if (!Session.HttpStatusCodes.ContainsKey(httpStatusCode))
                                            {
                                                Session.HttpStatusCodes[httpStatusCode] = 0;
                                            }

                                            Session.HttpStatusCodes[httpStatusCode]++;

                                            if (httpResponseMessage.IsSuccessStatusCode)
                                            {
                                                if (html == null)
                                                {
                                                    html = await GetHtml(httpResponseMessage);
                                                }

                                                Session.TotalHttpTraffic += html.Length;

                                                WebDirectory parsedWebDirectory = await DirectoryParser.ParseHtml(webDirectory, html, HttpClient);
                                                AddProcessedWebDirectory(webDirectory, parsedWebDirectory);
                                            }
                                            else
                                            {
                                                Session.Errors++;
                                                webDirectory.Error = true;

                                                if (!Session.UrlsWithErrors.Contains(webDirectory.Url))
                                                {
                                                    Session.UrlsWithErrors.Add(webDirectory.Url);
                                                }

                                                httpResponseMessage.EnsureSuccessStatusCode();
                                            }
                                        }
                                        else
                                        {
                                            Logger.Warn($"[{name}] Skipped result of '{webDirectory.Url}' which points to '{httpResponseMessage.RequestMessage.RequestUri}'");
                                            Session.Skipped++;
                                        }
                                    });
                                }
                                else
                                {
                                    Logger.Warn($"[{name}] Skipped result of '{webDirectory.Url}' because it is not the same host or path");

                                    Session.Skipped++;
                                }
                            }

                            Logger.Info($"[{name}] Finished processing {webDirectory.Url}");
                        }
                        else
                        {
                            Logger.Warn($"[{name}] Skip, already processed: {webDirectory.Uri}");
                        }
                    }
                    catch (Exception ex)
                    {
                        Logger.Error(ex, $"Error processing Url: '{webDirectory.Url}' from parent '{webDirectory.ParentDirectory.Url}'");

                        Session.Errors++;

                        if (!Session.UrlsWithErrors.Contains(webDirectory.Url))
                        {
                            Session.UrlsWithErrors.Add(webDirectory.Url);
                        }
                    }
                    finally
                    {
                        lock (WebDirectoryProcessorInfoLock)
                        {
                            WebDirectoryProcessorInfo.Remove(name);
                        }
                    }
                }

                Interlocked.Decrement(ref RunningWebDirectoryThreads);

                // Needed!
                await Task.Delay(TimeSpan.FromMilliseconds(10));
            }while (!token.IsCancellationRequested && (!queue.IsEmpty || RunningWebDirectoryThreads > 0));

            Logger.Debug($"Finished [{name}]");
        }