public static async Task ParseCalibre(HttpClient httpClient, Uri calibreRootUri, WebDirectory webDirectory, Version version, CancellationToken cancellationToken)
{
try
{
Console.WriteLine("Retrieving libraries...");
Logger.Info("Retrieving libraries...");
if (version.Major < 3)
{
Console.WriteLine($"Calibre {version} is not supported..");
return;
}
HttpResponseMessage httpResponseMessage = await httpClient.GetAsync(new Uri(calibreRootUri, "./interface-data/update"), cancellationToken);
httpResponseMessage.EnsureSuccessStatusCode();
string updateResultJson = await httpResponseMessage.Content.ReadAsStringAsync(cancellationToken);
CalibreUpdate.CalibreUpdate calibreUpdate = CalibreUpdate.CalibreUpdate.FromJson(updateResultJson);
Console.WriteLine($"Retrieved {calibreUpdate.LibraryMap.Count} libraries");
Logger.Info($"Retrieved {calibreUpdate.LibraryMap.Count} libraries");
foreach (KeyValuePair <string, string> library in calibreUpdate.LibraryMap)
{
Console.WriteLine($"Retrieving metadata of books for library {library.Value}...");
Logger.Info($"Retrieving metadata of books for library {library.Value}...");
WebDirectory libraryWebDirectory = new WebDirectory(webDirectory)
{
Url = $"{calibreRootUri}/#library_id={library.Key}&panel=book_list",
Name = library.Value,
Parser = "Calibre"
};
webDirectory.Subdirectories.Add(libraryWebDirectory);
Uri libraryMetadataUri = new Uri(calibreRootUri, $"./interface-data/books-init?library_id={library.Key}&num=999999999");
httpResponseMessage = await httpClient.GetAsync(libraryMetadataUri, cancellationToken);
httpResponseMessage.EnsureSuccessStatusCode();
string libraryResultJson = await httpResponseMessage.Content.ReadAsStringAsync(cancellationToken);
libraryWebDirectory.Files.Add(new WebFile
{
FileName = "LibraryMetadata.json",
FileSize = libraryResultJson.Length,
Url = libraryMetadataUri.ToString()
});
CalibreResult.CalibreResult libraryResult = CalibreResult.CalibreResult.FromJson(libraryResultJson);
Console.WriteLine($"Retrieved metadata of {libraryResult.Metadata.Count} books for library {library.Value}");
Logger.Info($"Retrieved metadata of {libraryResult.Metadata.Count} books for library {library.Value}");
Console.WriteLine($"Parsing info of {libraryResult.Metadata.Count} books for library {library.Value}...");
Logger.Info($"Parsing info of {libraryResult.Metadata.Count} books for library {library.Value}...");
int booksToIndex = libraryResult.Metadata.Count;
int booksIndexed = 0;
Stopwatch stopwatch = Stopwatch.StartNew();
//RateLimiter rateLimiter = new RateLimiter(100, TimeSpan.FromSeconds(30));
Parallel.ForEach(libraryResult.Metadata.AsParallel().AsOrdered(), new ParallelOptions {
MaxDegreeOfParallelism = 100
}, (book) =>
{
// NOT async, else it will continue with the rest of the code..
// TODO: Need a nice fix which respects MaxDegreeOfParallelism
//rateLimiter.RateLimit().Wait();
GetBookInfo(httpClient, calibreRootUri, library, libraryWebDirectory, book);
int newBooksIndexed = Interlocked.Increment(ref booksIndexed);
if (newBooksIndexed % 100 == 0 && stopwatch.Elapsed > TimeSpan.FromSeconds(5))
{
Logger.Warn($"Parsing books at {100 * ((decimal)newBooksIndexed / booksToIndex):F1}% ({newBooksIndexed}/{booksToIndex})");
stopwatch.Restart();
}
});
Console.WriteLine($"Parsed info of {libraryResult.Metadata.Count} books for library {library.Value}");
Logger.Info($"Parsed info of {libraryResult.Metadata.Count} books for library {library.Value}");
}
}
catch (Exception ex)
{
Logger.Error(ex, "Error parsing Calibre");
webDirectory.Error = true;
}
}
public static string ToJson(this CalibreUpdate self) => JsonConvert.SerializeObject(self, Converter.Settings);