private void GenerateProcessedFilesTSV() { string bloomLibrarySite = "bloomlibrary.org"; if (_options.ParseDBEnvironment == EnvironmentSetting.Dev) { bloomLibrarySite = "dev." + bloomLibrarySite; } string[] fieldsToDereference = new string[] { "langPointers", "uploader" }; IEnumerable <BookModel> bookList = _parseClient.GetBooks(out bool didExitPrematurely, _options.QueryWhere, fieldsToDereference); if (didExitPrematurely) { _logger.LogError("GetBooks() encountered an error and did not return all results. Aborting."); return; } using (StreamWriter sw = new StreamWriter(this.OutputPath)) { foreach (var book in bookList) { if (book.IsInCirculation == false) { continue; } Console.Out.WriteLine(book.ObjectId); string folder = HarvesterS3Client.RemoveSiteAndBucketFromUrl(book.BaseUrl); string langCode = ""; string langName = ""; if (book.Languages != null && book.Languages.Length > 0) { var language = book.Languages.First(); langCode = language.IsoCode; langName = language.Name; } string bloomLibraryUrl = $"https://{bloomLibrarySite}/browse/detail/{book.ObjectId}"; string pdfUrl = _bloomS3Client.GetFileWithExtension(folder, "pdf", book.Title); pdfUrl = $"{HarvesterS3Client.GetBloomS3UrlPrefix()}{_downloadBucketName}/{pdfUrl}"; string ePubUrl = _s3UploadClient.GetFileWithExtension(folder, "epub", book.Title); if (!String.IsNullOrEmpty(ePubUrl)) { ePubUrl = $"{HarvesterS3Client.GetBloomS3UrlPrefix()}{_uploadBucketName}/{ePubUrl}"; } else { ePubUrl = ""; } sw.WriteLine($"{book.Title}\t{langCode}\t{langName}\t{bloomLibraryUrl}\t{pdfUrl}\t{ePubUrl}"); } } }
public Harvester(HarvesterCommonOptions options) { // Note: If the same machine runs multiple BloomHarvester processes, then you need to add a suffix to this. this.Identifier = Environment.MachineName; if (options.SuppressLogs) { _logger = new ConsoleLogger(); } else { EnvironmentSetting azureMonitorEnvironment = EnvironmentUtils.GetEnvOrFallback(options.LogEnvironment, options.Environment); _logger = new AzureMonitorLogger(azureMonitorEnvironment, this.Identifier); } EnvironmentSetting parseDBEnvironment = EnvironmentUtils.GetEnvOrFallback(options.ParseDBEnvironment, options.Environment); _parseClient = new ParseClient(parseDBEnvironment); _parseClient.Logger = _logger; string downloadBucketName; string uploadBucketName; switch (parseDBEnvironment) { case EnvironmentSetting.Prod: downloadBucketName = BloomS3Client.ProductionBucketName; uploadBucketName = HarvesterS3Client.HarvesterProductionBucketName; break; case EnvironmentSetting.Test: downloadBucketName = BloomS3Client.UnitTestBucketName; uploadBucketName = HarvesterS3Client.HarvesterUnitTestBucketName; break; case EnvironmentSetting.Dev: case EnvironmentSetting.Local: default: downloadBucketName = BloomS3Client.SandboxBucketName; uploadBucketName = HarvesterS3Client.HarvesterSandboxBucketName; break; } _transfer = new BookTransfer(_parseClient, bloomS3Client: new HarvesterS3Client(downloadBucketName), htmlThumbnailer: null, bookDownloadStartingEvent: new BookDownloadStartingEvent()); _s3UploadClient = new HarvesterS3Client(uploadBucketName); }