public static async Task <WebDirectory> ParseIndex(HttpClient httpClient, WebDirectory webDirectory) { try { string driveHash = GetDriveHash(webDirectory); if (!OpenDirectoryIndexer.Session.Parameters.ContainsKey(Constants.Parameters_Password)) { Console.WriteLine($"{Parser} will always be indexed at a maximum rate of 1 per second, else you will run into problems and errors."); Logger.Info($"{Parser} will always be indexed at a maximum rate of 1 per second, else you will run into problems and errors."); Console.WriteLine("Check if password is needed (unsupported currently)..."); Logger.Info("Check if password is needed (unsupported currently)..."); OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password] = string.Empty; HttpResponseMessage httpResponseMessage = await httpClient.GetAsync(GetFolderUrl(driveHash, string.Empty, 0)); if (httpResponseMessage.IsSuccessStatusCode) { string responseJson = await httpResponseMessage.Content.ReadAsStringAsync(); BlitzfilesTechResponse response = BlitzfilesTechResponse.FromJson(responseJson); webDirectory = await ScanAsync(httpClient, webDirectory); } } else { webDirectory = await ScanAsync(httpClient, webDirectory); } } catch (Exception ex) { Logger.Error(ex, $"Error parsing {Parser} for URL: {webDirectory.Url}"); webDirectory.Error = true; OpenDirectoryIndexer.Session.Errors++; if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url)) { OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url); } throw; } return(webDirectory); }
private static async Task <WebDirectory> ScanAsync(HttpClient httpClient, WebDirectory webDirectory) { Logger.Debug($"Retrieving listings for {webDirectory.Uri} with password: {OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password]}"); webDirectory.Parser = Parser; try { await RateLimiter.RateLimit(); string driveHash = GetDriveHash(webDirectory); string entryHash = string.Empty; long pageIndex = 0; long totalPages = 0; do { Logger.Warn($"Retrieving listings for {webDirectory.Uri} with password: {OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password]}, page {pageIndex + 1}"); HttpResponseMessage httpResponseMessage = await httpClient.GetAsync(GetFolderUrl(driveHash, entryHash, pageIndex)); webDirectory.ParsedSuccessfully = httpResponseMessage.IsSuccessStatusCode; httpResponseMessage.EnsureSuccessStatusCode(); string responseJson = await httpResponseMessage.Content.ReadAsStringAsync(); BlitzfilesTechResponse indexResponse = BlitzfilesTechResponse.FromJson(responseJson); entryHash = indexResponse.Link.Entry.Hash; pageIndex = indexResponse.FolderChildren.CurrentPage; totalPages = indexResponse.FolderChildren.LastPage; foreach (Entry entry in indexResponse.FolderChildren.Data) { if (entry.Type == "folder") { webDirectory.Subdirectories.Add(new WebDirectory(webDirectory) { Parser = Parser, Url = $"https://blitzfiles.tech/files/drive/s/{driveHash}:{entry.Hash}", Name = entry.Name }); } else { webDirectory.Files.Add(new WebFile { Url = $"https://blitzfiles.tech/files/secure/uploads/download?hashes={entry.Hash}&shareable_link={indexResponse.Link.Id}", FileName = entry.Name, FileSize = entry.FileSize }); } } } while (pageIndex < totalPages); } catch (Exception ex) { Logger.Error(ex, $"Error processing {Parser} for URL: {webDirectory.Url}"); webDirectory.Error = true; OpenDirectoryIndexer.Session.Errors++; if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url)) { OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url); } //throw; } return(webDirectory); }