private void GenerateProcessedFilesTSV()
{
string bloomLibrarySite = "bloomlibrary.org";
if (_options.ParseDBEnvironment == EnvironmentSetting.Dev)
{
bloomLibrarySite = "dev." + bloomLibrarySite;
}
string[] fieldsToDereference = new string[] { "langPointers", "uploader" };
IEnumerable <BookModel> bookList = _parseClient.GetBooks(out bool didExitPrematurely, _options.QueryWhere, fieldsToDereference);
if (didExitPrematurely)
{
_logger.LogError("GetBooks() encountered an error and did not return all results. Aborting.");
return;
}
using (StreamWriter sw = new StreamWriter(this.OutputPath))
{
foreach (var book in bookList)
{
if (book.IsInCirculation == false)
{
continue;
}
Console.Out.WriteLine(book.ObjectId);
string folder = HarvesterS3Client.RemoveSiteAndBucketFromUrl(book.BaseUrl);
string langCode = "";
string langName = "";
if (book.Languages != null && book.Languages.Length > 0)
{
var language = book.Languages.First();
langCode = language.IsoCode;
langName = language.Name;
}
string bloomLibraryUrl = $"https://{bloomLibrarySite}/browse/detail/{book.ObjectId}";
string pdfUrl = _bloomS3Client.GetFileWithExtension(folder, "pdf", book.Title);
pdfUrl = $"{HarvesterS3Client.GetBloomS3UrlPrefix()}{_downloadBucketName}/{pdfUrl}";
string ePubUrl = _s3UploadClient.GetFileWithExtension(folder, "epub", book.Title);
if (!String.IsNullOrEmpty(ePubUrl))
{
ePubUrl = $"{HarvesterS3Client.GetBloomS3UrlPrefix()}{_uploadBucketName}/{ePubUrl}";
}
else
{
ePubUrl = "";
}
sw.WriteLine($"{book.Title}\t{langCode}\t{langName}\t{bloomLibraryUrl}\t{pdfUrl}\t{ePubUrl}");
}
}
}
public Harvester(HarvesterCommonOptions options)
{
// Note: If the same machine runs multiple BloomHarvester processes, then you need to add a suffix to this.
this.Identifier = Environment.MachineName;
if (options.SuppressLogs)
{
_logger = new ConsoleLogger();
}
else
{
EnvironmentSetting azureMonitorEnvironment = EnvironmentUtils.GetEnvOrFallback(options.LogEnvironment, options.Environment);
_logger = new AzureMonitorLogger(azureMonitorEnvironment, this.Identifier);
}
EnvironmentSetting parseDBEnvironment = EnvironmentUtils.GetEnvOrFallback(options.ParseDBEnvironment, options.Environment);
_parseClient = new ParseClient(parseDBEnvironment);
_parseClient.Logger = _logger;
string downloadBucketName;
string uploadBucketName;
switch (parseDBEnvironment)
{
case EnvironmentSetting.Prod:
downloadBucketName = BloomS3Client.ProductionBucketName;
uploadBucketName = HarvesterS3Client.HarvesterProductionBucketName;
break;
case EnvironmentSetting.Test:
downloadBucketName = BloomS3Client.UnitTestBucketName;
uploadBucketName = HarvesterS3Client.HarvesterUnitTestBucketName;
break;
case EnvironmentSetting.Dev:
case EnvironmentSetting.Local:
default:
downloadBucketName = BloomS3Client.SandboxBucketName;
uploadBucketName = HarvesterS3Client.HarvesterSandboxBucketName;
break;
}
_transfer = new BookTransfer(_parseClient,
bloomS3Client: new HarvesterS3Client(downloadBucketName),
htmlThumbnailer: null,
bookDownloadStartingEvent: new BookDownloadStartingEvent());
_s3UploadClient = new HarvesterS3Client(uploadBucketName);
}
