private static async Task <WebDirectory> ScanIndexAsync(HttpClient httpClient, WebDirectory webDirectory)
        {
            Logger.Debug($"Retrieving listings for {webDirectory.Uri} with password: {OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password]}");

            webDirectory.Parser = Parser;

            try
            {
                await RateLimiter.RateLimit();

                if (!webDirectory.Url.EndsWith("/"))
                {
                    webDirectory.Url += "/";
                }

                long   pageIndex     = 0;
                string nextPageToken = null;

                do
                {
                    Logger.Warn($"Retrieving listings for {webDirectory.Uri} with password: {OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password]}, page {pageIndex + 1}");

                    HttpResponseMessage httpResponseMessage = await httpClient.PostAsync(webDirectory.Uri, new StringContent(JsonConvert.SerializeObject(new Dictionary <string, object>
                    {
                        { "page_index", pageIndex },
                        { "page_token", nextPageToken },
                        { "password", OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password] },
                        { "q", "" }
                    })));

                    webDirectory.ParsedSuccesfully = httpResponseMessage.IsSuccessStatusCode;
                    httpResponseMessage.EnsureSuccessStatusCode();

                    string responseJson = await httpResponseMessage.Content.ReadAsStringAsync();

                    Go2IndexResponse indexResponse = Go2IndexResponse.FromJson(responseJson);

                    webDirectory.ParsedSuccesfully = indexResponse.Error == null;

                    if (indexResponse.Error != null)
                    {
                        throw new Exception($"{indexResponse.Error.Code} | {indexResponse.Error.Message}");
                    }

                    nextPageToken = indexResponse.NextPageToken;
                    pageIndex     = indexResponse.CurPageIndex + 1;

                    foreach (File file in indexResponse.Data.Files)
                    {
                        if (file.MimeType == FolderMimeType)
                        {
                            webDirectory.Subdirectories.Add(new WebDirectory(webDirectory)
                            {
                                Parser = Parser,
                                // Yes, string concatenation, do not use new Uri(webDirectory.Uri, file.Name), because things could end with a space...
                                Url  = $"{webDirectory.Uri}{file.Name}/",
                                Name = file.Name
                            });
                        }
                        else
                        {
                            webDirectory.Files.Add(new WebFile
                            {
                                Url      = new Uri(webDirectory.Uri, file.Name).ToString(),
                                FileName = file.Name,
                                FileSize = file.Size
                            });
                        }
                    }
                } while (!string.IsNullOrWhiteSpace(nextPageToken));
            }
            catch (Exception ex)
            {
                Logger.Error(ex, $"Error processing {Parser} for URL: {webDirectory.Url}");
                webDirectory.Error = true;

                OpenDirectoryIndexer.Session.Errors++;

                if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url))
                {
                    OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url);
                }

                //throw;
            }

            return(webDirectory);
        }
        public static async Task <WebDirectory> ParseIndex(HttpClient httpClient, WebDirectory webDirectory)
        {
            try
            {
                if (!OpenDirectoryIndexer.Session.Parameters.ContainsKey(Constants.Parameters_Password))
                {
                    Console.WriteLine($"{Parser} will always be indexed at a maximum rate of 1 per second, else you will run into problems and errors.");
                    Logger.Info($"{Parser} will always be indexed at a maximum rate of 1 per second, else you will run into problems and errors.");

                    Console.WriteLine("Check if password is needed...");
                    Logger.Info("Check if password is needed...");
                    OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password] = null;

                    HttpResponseMessage httpResponseMessage = await httpClient.PostAsync(webDirectory.Uri, new StringContent(JsonConvert.SerializeObject(new Dictionary <string, object>
                    {
                        { "page_index", 0 },
                        { "page_token", null },
                        { "password", OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password] },
                        { "q", "" }
                    })));

                    Go2IndexResponse indexResponse = null;

                    if (httpResponseMessage.IsSuccessStatusCode)
                    {
                        string responseJson = await httpResponseMessage.Content.ReadAsStringAsync();

                        indexResponse = Go2IndexResponse.FromJson(responseJson);

                        if (indexResponse.Error?.Code == (int)HttpStatusCode.Unauthorized)
                        {
                            Console.WriteLine("Directory is password protected, please enter password:"******"Directory is password protected, please enter password.");

                            OpenDirectoryIndexer.Session.Parameters["GoIndex_Password"] = Console.ReadLine();

                            Console.WriteLine($"Using password: {OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password]}");
                            Logger.Info($"Using password: {OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password]}");

                            httpResponseMessage = await httpClient.PostAsync(webDirectory.Uri, new StringContent(JsonConvert.SerializeObject(new Dictionary <string, object>
                            {
                                { "page_index", 0 },
                                { "page_token", null },
                                { "password", OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password] },
                                { "q", "" }
                            })));

                            if (httpResponseMessage.IsSuccessStatusCode)
                            {
                                responseJson = await httpResponseMessage.Content.ReadAsStringAsync();

                                indexResponse = Go2IndexResponse.FromJson(responseJson);
                            }
                        }
                    }

                    if (indexResponse.Error == null)
                    {
                        Console.WriteLine("Password OK!");
                        Logger.Info("Password OK!");

                        webDirectory = await ScanIndexAsync(httpClient, webDirectory);
                    }
                    else
                    {
                        OpenDirectoryIndexer.Session.Parameters.Remove(Constants.Parameters_Password);
                        Console.WriteLine($"Error. Code: {indexResponse.Error.Code}, Message: {indexResponse.Error.Message}. Stopping.");
                        Logger.Error($"Error. Code: {indexResponse.Error.Code}, Message: {indexResponse.Error.Message}. Stopping.");
                    }
                }
                else
                {
                    webDirectory = await ScanIndexAsync(httpClient, webDirectory);
                }
            }
            catch (Exception ex)
            {
                Logger.Error(ex, $"Error parsing {Parser} for URL: {webDirectory.Url}");
                webDirectory.Error = true;

                OpenDirectoryIndexer.Session.Errors++;

                if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url))
                {
                    OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url);
                }

                throw;
            }

            return(webDirectory);
        }
 public static string ToJson(this Go2IndexResponse self) => JsonConvert.SerializeObject(self, Converter.Settings);