public static async Task <WebDirectory> ParseIndex(HttpClient httpClient, WebDirectory webDirectory)
{
if (OpenDirectoryIndexer.Session.MaxThreads > 1)
{
throw new FriendlyException($"{Parser} can only scan at maximum of 1 thread, please call with -t 1 or --threads 1");
}
try
{
string driveHash = GetDriveHash(webDirectory);
if (!OpenDirectoryIndexer.Session.Parameters.ContainsKey(Constants.Parameters_Password))
{
Console.WriteLine($"{Parser} will always be indexed with only 1 thread, else you will run into problems and errors.");
Logger.Info($"{Parser} will always be indexed with only 1 thread, else you will run into problems and errors.");
//OpenDirectoryIndexer.Session.MaxThreads = 1;
Console.WriteLine("Check if password is needed (unsupported currently)...");
Logger.Info("Check if password is needed (unsupported currently)...");
OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password] = string.Empty;
HttpResponseMessage httpResponseMessage = await httpClient.GetAsync(GetFolderUrl(driveHash, string.Empty, 0));
if (httpResponseMessage.IsSuccessStatusCode)
{
string responseJson = await httpResponseMessage.Content.ReadAsStringAsync();
BlitzfilesTechResponse response = BlitzfilesTechResponse.FromJson(responseJson);
webDirectory = await ScanAsync(httpClient, webDirectory);
}
}
else
{
webDirectory = await ScanAsync(httpClient, webDirectory);
}
}
catch (Exception ex)
{
Logger.Error(ex, $"Error parsing {Parser} for URL: {webDirectory.Url}");
webDirectory.Error = true;
OpenDirectoryIndexer.Session.Errors++;
if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url))
{
OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url);
}
throw;
}
return(webDirectory);
}
private static async Task <WebDirectory> ScanAsync(HttpClient httpClient, WebDirectory webDirectory)
{
Logger.Debug($"Retrieving listings for {webDirectory.Uri} with password: {OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password]}");
webDirectory.Parser = Parser;
try
{
await RateLimiter.RateLimit();
string driveHash = GetDriveHash(webDirectory);
string entryHash = string.Empty;
long pageIndex = 0;
long totalPages = 0;
do
{
Logger.Warn($"Retrieving listings for {webDirectory.Uri} with password: {OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_Password]}, page {pageIndex + 1}");
HttpResponseMessage httpResponseMessage = await httpClient.GetAsync(GetFolderUrl(driveHash, entryHash, pageIndex));
webDirectory.ParsedSuccesfully = httpResponseMessage.IsSuccessStatusCode;
httpResponseMessage.EnsureSuccessStatusCode();
string responseJson = await httpResponseMessage.Content.ReadAsStringAsync();
BlitzfilesTechResponse indexResponse = BlitzfilesTechResponse.FromJson(responseJson);
entryHash = indexResponse.Link.Entry.Hash;
pageIndex = indexResponse.FolderChildren.CurrentPage;
totalPages = indexResponse.FolderChildren.LastPage;
foreach (Entry entry in indexResponse.FolderChildren.Data)
{
if (entry.Type == "folder")
{
webDirectory.Subdirectories.Add(new WebDirectory(webDirectory)
{
Parser = Parser,
Url = $"https://blitzfiles.tech/files/drive/s/{driveHash}:{entry.Hash}",
Name = entry.Name
});
}
else
{
webDirectory.Files.Add(new WebFile
{
Url = $"https://blitzfiles.tech/files/secure/uploads/download?hashes={entry.Hash}&shareable_link={indexResponse.Link.Id}",
FileName = entry.Name,
FileSize = entry.FileSize
});
}
}
} while (pageIndex < totalPages);
}
catch (Exception ex)
{
Logger.Error(ex, $"Error processing {Parser} for URL: {webDirectory.Url}");
webDirectory.Error = true;
OpenDirectoryIndexer.Session.Errors++;
if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url))
{
OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url);
}
//throw;
}
return(webDirectory);
}