public static int HandleSilentDownload(DownloadBookOptions options)
{
// This task will be all the program does. We need to do enough setup so that
// the download code can work, then tear it down.
Program.SetUpErrorHandling();
try
{
using (var applicationContainer = new ApplicationContainer())
{
Program.SetUpLocalization(applicationContainer);
Browser.SetUpXulRunner();
Browser.XulRunnerShutdown += Program.OnXulRunnerShutdown;
LocalizationManager.SetUILanguage(Settings.Default.UserInterfaceLanguage, false);
var transfer = new BookTransfer(new BloomParseClient(), ProjectContext.CreateBloomS3Client(),
applicationContainer.BookThumbNailer, new BookDownloadStartingEvent()) /*not hooked to anything*/;
// Since Bloom is not a normal console app, when run from a command line, the new command prompt
// appears at once. The extra newlines here are attempting to separate this from our output.
Console.WriteLine("\nstarting download");
transfer.HandleDownloadWithoutProgress(options.Url, options.DestinationPath);
Console.WriteLine(("\ndownload complete\n"));
}
return(0);
}
catch (Exception ex)
{
Debug.WriteLine(ex.Message);
Console.WriteLine(ex.Message);
Console.WriteLine(ex.StackTrace);
return(1);
}
}
/// <summary>
/// Review: this is fragile and expensive. We're doing real internet traffic and creating real objects on S3 and parse.com
/// which (to a very small extent) costs us real money. This will be slow. Also, under S3 eventual consistency rules,
/// there is no guarantee that the data we just created will actually be retrievable immediately.
/// </summary>
/// <param name="bookName"></param>
/// <param name="id"></param>
/// <param name="uploader"></param>
/// <param name="data"></param>
/// <returns></returns>
public Tuple <string, string> UploadAndDownLoadNewBook(string bookName, string id, string uploader, string data, bool isTemplate = false)
{
// Create a book folder with meta.json that includes an uploader and id and some other files.
var originalBookFolder = MakeBook(bookName, id, uploader, data, true);
if (isTemplate)
{
var metadata = BookMetaData.FromFolder(originalBookFolder);
metadata.IsSuitableForMakingShells = true;
metadata.WriteToFolder(originalBookFolder);
}
// The files that actually get uploaded omit some of the ones in the folder.
// The only omitted one that messes up current unit tests is meta.bak
var filesToUpload = Directory.GetFiles(originalBookFolder).Where(p => !p.EndsWith(".bak") && !p.Contains(BookStorage.PrefixForCorruptHtmFiles));
int fileCount = filesToUpload.Count();
Login();
//HashSet<string> notifications = new HashSet<string>();
var progress = new SIL.Progress.StringBuilderProgress();
var s3Id = _transfer.UploadBook(originalBookFolder, progress);
var uploadMessages = progress.Text.Split(new string[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries);
var expectedFileCount = fileCount + 2; // should get one per file, plus one for metadata, plus one for book order
#if DEBUG
++expectedFileCount; // and if in debug mode, then plus one for S3 ID
#endif
Assert.That(uploadMessages.Length, Is.EqualTo(expectedFileCount));
Assert.That(progress.Text, Does.Contain(
LocalizationManager.GetString("PublishTab.Upload.UploadingBookMetadata", "Uploading book metadata",
"In this step, Bloom is uploading things like title, languages, & topic tags to the bloomlibrary.org database.")));
Assert.That(progress.Text, Does.Contain(Path.GetFileName(filesToUpload.First())));
_transfer.WaitUntilS3DataIsOnServer(BloomS3Client.UnitTestBucketName, originalBookFolder);
var dest = _workFolderPath.CombineForPath("output");
Directory.CreateDirectory(dest);
_downloadedBooks.Clear();
var url = BookTransfer.BloomS3UrlPrefix + BloomS3Client.UnitTestBucketName + "/" + s3Id;
var newBookFolder = _transfer.HandleDownloadWithoutProgress(url, dest);
Assert.That(Directory.GetFiles(newBookFolder).Length, Is.EqualTo(fileCount + 1)); // book order is added during upload
Assert.That(_downloadedBooks.Count, Is.EqualTo(1));
Assert.That(_downloadedBooks[0].FolderPath, Is.EqualTo(newBookFolder));
// Todo: verify that metadata was transferred to Parse.com
return(new Tuple <string, string>(originalBookFolder, newBookFolder));
}
private void ProcessOneBook(Book book)
{
try
{
_logger.TrackEvent("ProcessOneBook Start");
string message = $"Processing: {book.BaseUrl}";
Console.Out.WriteLine(message);
_logger.LogVerbose(message);
var initialUpdates = new UpdateOperation();
initialUpdates.UpdateField(Book.kHarvestStateField, Book.HarvestState.InProgress.ToString());
initialUpdates.UpdateField(Book.kHarvesterIdField, this.Identifier);
var startTime = new Parse.Model.Date(DateTime.UtcNow);
initialUpdates.UpdateField("harvestStartedAt", startTime.ToJson());
_parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, initialUpdates.ToJson());
// Download the book
_logger.TrackEvent("Download Book");
string decodedUrl = HttpUtility.UrlDecode(book.BaseUrl);
string urlWithoutTitle = RemoveBookTitleFromBaseUrl(decodedUrl);
string downloadRootDir = Path.Combine(Path.GetTempPath(), Path.Combine("BloomHarvester", this.Identifier));
_logger.LogVerbose("Download Dir: {0}", downloadRootDir);
string downloadBookDir = _transfer.HandleDownloadWithoutProgress(urlWithoutTitle, downloadRootDir);
// Process the book
var finalUpdates = new UpdateOperation();
var warnings = FindBookWarnings(book);
finalUpdates.UpdateField(Book.kWarningsField, Book.ToJson(warnings));
// ENHANCE: Do more processing here
_logger.TrackEvent("Upload Book");
UploadBook(decodedUrl, downloadBookDir);
// Write the updates
finalUpdates.UpdateField(Book.kHarvestStateField, Book.HarvestState.Done.ToString());
_parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, finalUpdates.ToJson());
_logger.TrackEvent("ProcessOneBook End - Success");
}
catch (Exception e)
{
YouTrackIssueConnector.SubmitToYouTrack(e, $"Unhandled exception thrown while processing book \"{book.BaseUrl}\"");
// Attempt to write to Parse that processing failed
if (!String.IsNullOrEmpty(book?.ObjectId))
{
try
{
var onErrorUpdates = new UpdateOperation();
onErrorUpdates.UpdateField(Book.kHarvestStateField, $"\"{Book.HarvestState.Failed.ToString()}\"");
onErrorUpdates.UpdateField(Book.kHarvesterIdField, this.Identifier);
_parseClient.UpdateObject(book.GetParseClassName(), book.ObjectId, onErrorUpdates.ToJson());
}
catch (Exception)
{
// If it fails, just let it be and throw the first exception rather than the nested exception.
}
}
throw;
}
}
