private void BulkUploadBooksOfOneCollection(IProgress progress, ApplicationContainer container,
BookUploadParameters uploadParams,
ref ProjectContext context)
{
foreach (var sub in Directory.GetDirectories(uploadParams.Folder))
{
var htmlFileCount = Directory.GetFiles(sub, "*.htm").Length;
if (htmlFileCount == 1)
{
// Our (perhaps insufficient) definition of a book folder is that it has exactly 1 htm file.
try
{
var bookParams = uploadParams;
bookParams.Folder = sub;
UploadBookInternal(progress, container, bookParams, ref context);
}
catch (Exception e)
{
var msg = String.Format("{0} was not uploaded due to error: {1}", sub, e.Message);
progress.WriteError(msg);
progress.WriteException(e);
++_booksWithErrors;
}
}
else
{
if (htmlFileCount > 1)
{
progress.WriteError($"{sub} has ${htmlFileCount} html files. One of them should be removed.");
++_booksWithErrors;
}
else
{
ReportSuspiciousFilesInFolderLackingHtml(progress, sub);
}
}
}
}
/// <summary>
/// Upload bloom books in the specified folder to the bloom library.
/// Folders that contain exactly one .htm file are interpreted as books and uploaded.
/// Other folders are searched recursively for children that appear to be bloom books.
/// The parent folder of a bloom book is searched for a .bloomCollection file and, if one is found,
/// the book is treated as part of that collection (e.g., for determining vernacular language).
/// If the .bloomCollection file is not found, the book is not uploaded.
/// N.B. The bulk upload process will go ahead and upload templates and books that are already on the server
/// (over-writing the existing book) without informing the user.
/// </summary>
/// <remarks>This method is triggered by starting Bloom with "upload" on the cmd line.</remarks>
public void BulkUpload(ApplicationContainer container, UploadParameters options)
{
BookUpload.Destination = options.Dest;
using (var progress = new MultiProgress())
{
var logFilePath = Path.Combine(options.Path, "BloomBulkUploadLog.txt");
progress.Add(new Bloom.Utils.ConsoleProgress());
progress.Add(new FileLogProgress(logFilePath));
if (!_singleBookUploader.IsThisVersionAllowedToUpload())
{
var oldVersionMsg = LocalizationManager.GetString("PublishTab.Upload.OldVersion",
"Sorry, this version of Bloom Desktop is not compatible with the current version of BloomLibrary.org. Please upgrade to a newer version.");
progress.WriteMessage(oldVersionMsg);
return;
}
Debug.Assert(!String.IsNullOrWhiteSpace(options.UploadUser));
if (!_singleBookUploader.ParseClient.AttemptSignInAgainForCommandLine(options.UploadUser, options.Dest, progress))
{
progress.WriteError("Problem logging in. See messages above.");
System.Environment.Exit(1);
}
progress.WriteMessage("Uploading books as user {0}", options.UploadUser);
var bookParams = new BookUploadParameters(options);
BulkRepairInstanceIds(options.Path);
ProjectContext
context = null; // Expensive to create; hold each one we make until we find a book that needs a different one.
try
{
_collectionFoldersUploaded = new HashSet <string>();
_newBooksUploaded = 0;
_booksUpdated = 0;
_booksSkipped = 0;
_booksWithErrors = 0;
progress.WriteMessageWithColor("green", $"\n\nStarting upload at {DateTime.Now.ToString()}\n");
progress.WriteMessageWithColor("Magenta", $"Looking in '{bookParams.Folder}'...");
UploadCollectionOrKeepLookingDeeper(progress, container, bookParams, ref context);
if (_collectionFoldersUploaded.Count > 0)
{
progress.WriteMessageWithColor("green", "\n\nAll finished!");
progress.WriteMessage("Processed {0} collection folders.", _collectionFoldersUploaded.Count);
}
else
{
progress.WriteError("Did not find any collections to upload.");
}
progress.WriteMessage("Uploaded {0} new books.", _newBooksUploaded);
progress.WriteMessage("Updated {0} books that had changed.", _booksUpdated);
progress.WriteMessage("Skipped {0} books that had not changed.", _booksSkipped);
if (_booksSkipped > 0)
{
progress.WriteMessage("(If you don't want Bloom to skip books it thinks have not changed, you can use the --force argument to force all books to re-upload, or just use the Bloom UI to force upload this one book).");
}
if (_booksWithErrors > 0)
{
progress.WriteError("Failed to upload {0} books. See \"{1}\" for details.", _booksWithErrors,
logFilePath);
}
}
finally
{
context?.Dispose();
}
}
}
private void UploadBookInternal(IProgress progress, ApplicationContainer container, BookUploadParameters uploadParams,
ref ProjectContext context)
{
progress.WriteMessageWithColor("Cyan", "Starting to upload " + uploadParams.Folder);
// Make sure the files we want to upload are up to date.
// Unfortunately this requires making a book object, which requires making a ProjectContext, which must be created with the
// proper parent book collection if possible.
var parent = Path.GetDirectoryName(uploadParams.Folder);
var collectionPath = Directory.GetFiles(parent, "*.bloomCollection").FirstOrDefault();
if (collectionPath == null || !RobustFile.Exists(collectionPath))
{
progress.WriteError("Skipping book because no collection file was found in its parent directory.");
return;
}
_collectionFoldersUploaded.Add(collectionPath);
// Compute the book hash file and compare it to the existing one for bulk upload.
var currentHashes = BookUpload.HashBookFolder(uploadParams.Folder);
progress.WriteMessage(currentHashes);
var pathToLocalHashInfoFromLastUpload = Path.Combine(uploadParams.Folder, HashInfoFromLastUpload);
if (!uploadParams.ForceUpload)
{
var canSkip = false;
if (Program.RunningUnitTests)
{
canSkip = _singleBookUploader.CheckAgainstLocalHashfile(currentHashes, pathToLocalHashInfoFromLastUpload);
}
else
{
canSkip = _singleBookUploader.CheckAgainstHashFileOnS3(currentHashes, uploadParams.Folder, progress);
RobustFile.WriteAllText(pathToLocalHashInfoFromLastUpload, currentHashes); // ensure local copy is saved
}
if (canSkip)
{
// local copy of hashes file is identical or has been saved
progress.WriteMessageWithColor("green", $"Skipping '{Path.GetFileName(uploadParams.Folder)}' because it has not changed since being uploaded.");
++_booksSkipped;
return; // skip this one; we already uploaded it earlier.
}
}
// save local copy of hashes file: it will be uploaded with the other book files
RobustFile.WriteAllText(pathToLocalHashInfoFromLastUpload, currentHashes);
if (context == null || context.SettingsPath != collectionPath)
{
context?.Dispose();
// optimise: creating a context seems to be quite expensive. Probably the only thing we need to change is
// the collection. If we could update that in place...despite autofac being told it has lifetime scope...we would save some time.
// Note however that it's not good enough to just store it in the project context. The one that is actually in
// the autofac object (_scope in the ProjectContext) is used by autofac to create various objects, in particular, books.
context = container.CreateProjectContext(collectionPath);
Program.SetProjectContext(context);
}
var server = context.BookServer;
var bookInfo = new BookInfo(uploadParams.Folder, true);
var book = server.GetBookFromBookInfo(bookInfo);
book.BringBookUpToDate(new NullProgress());
bookInfo.Bookshelf = book.CollectionSettings.DefaultBookshelf;
var bookshelfName = String.IsNullOrWhiteSpace(book.CollectionSettings.DefaultBookshelf) ? "(none)" : book.CollectionSettings.DefaultBookshelf;
progress.WriteMessage($"Bookshelf is '{bookshelfName}'");
// Assemble the various arguments needed to make the objects normally involved in an upload.
// We leave some constructor arguments not actually needed for this purpose null.
var bookSelection = new BookSelection();
bookSelection.SelectBook(book);
var currentEditableCollectionSelection = new CurrentEditableCollectionSelection();
var collection = new BookCollection(collectionPath, BookCollection.CollectionType.SourceCollection, bookSelection);
currentEditableCollectionSelection.SelectCollection(collection);
var publishModel = new PublishModel(bookSelection, new PdfMaker(), currentEditableCollectionSelection, context.Settings, server, _thumbnailer);
publishModel.PageLayout = book.GetLayout();
var view = new PublishView(publishModel, new SelectedTabChangedEvent(), new LocalizationChangedEvent(), _singleBookUploader, null, null, null, null);
var blPublishModel = new BloomLibraryPublishModel(_singleBookUploader, book, publishModel);
string dummy;
// Normally we let the user choose which languages to upload. Here, just the ones that have complete information.
var langDict = book.AllPublishableLanguages();
var languagesToUpload = langDict.Keys.Where(l => langDict[l]).ToList();
if (!string.IsNullOrEmpty(book.CollectionSettings.SignLanguageIso639Code) && BookUpload.GetVideoFilesToInclude(book).Any())
{
languagesToUpload.Insert(0, book.CollectionSettings.SignLanguageIso639Code);
}
if (blPublishModel.MetadataIsReadyToPublish && (languagesToUpload.Any() || blPublishModel.OkToUploadWithNoLanguages))
{
if (blPublishModel.BookIsAlreadyOnServer)
{
var msg = $"Overwriting the copy of {uploadParams.Folder} on the server...";
progress.WriteWarning(msg);
}
using (var tempFolder = new TemporaryFolder(Path.Combine("BloomUpload", Path.GetFileName(book.FolderPath))))
{
BookUpload.PrepareBookForUpload(ref book, server, tempFolder.FolderPath, progress);
uploadParams.LanguagesToUpload = languagesToUpload.ToArray();
_singleBookUploader.FullUpload(book, progress, view, uploadParams, out dummy);
}
progress.WriteMessageWithColor("Green", "{0} has been uploaded", uploadParams.Folder);
if (blPublishModel.BookIsAlreadyOnServer)
{
++_booksUpdated;
}
else
{
++_newBooksUploaded;
}
}
else
{
// report to the user why we are not uploading their book
var reason = blPublishModel.GetReasonForNotUploadingBook();
progress.WriteError("{0} was not uploaded. {1}", uploadParams.Folder, reason);
++_booksWithErrors;
}
}
/// <summary>
/// Handles the recursion through directories: if a folder looks like a Bloom book upload it; otherwise, try its children.
/// Invisible folders like .hg are ignored.
/// </summary>
private void UploadCollectionOrKeepLookingDeeper(IProgress progress, ApplicationContainer container, BookUploadParameters uploadParams,
ref ProjectContext context)
{
if (IsPrivateFolder(uploadParams.Folder))
{
return;
}
var collectionPath = Directory.GetFiles(uploadParams.Folder, "*.bloomCollection").FirstOrDefault();
if (collectionPath != null)
{
var settings = new CollectionSettings(collectionPath);
if (string.IsNullOrEmpty(settings.DefaultBookshelf))
{
// My thinking here is that if we are bothering to do a bulk upload, they should have set a
// default bookshelf. If this expectation proves false, then we can just add an argument
// to disable it. For Kyrgyzstan, missing bookshelves was a problem I needed to catch.
progress.WriteError($"Skipping {uploadParams.Folder} because there is no default bookshelf.");
return;
}
if (!settings.HaveEnterpriseFeatures)
{
progress.WriteError($"Skipping {uploadParams.Folder} because bulk upload is an Enterprise-only feature.");
return;
}
BulkUploadBooksOfOneCollection(progress, container, uploadParams, ref context);
return;
}
else // go looking deeper for collection folders
{
foreach (var sub in Directory.GetDirectories(uploadParams.Folder))
{
if (!IsPrivateFolder(uploadParams.Folder))
{
var childParams = uploadParams;
childParams.Folder = sub;
progress.WriteMessageWithColor("Magenta", $"\nLooking in '{sub}'...");
UploadCollectionOrKeepLookingDeeper(progress, container, childParams, ref context);
}
}
}
}
/// <summary>
/// Common routine used in normal upload and bulk upload.
/// </summary>
internal string FullUpload(Book.Book book, IProgress progress, PublishView publishView, BookUploadParameters bookParams, out string parseId)
{
book.Storage.CleanupUnusedSupportFiles(isForPublish: false); // we are publishing, but this is the real folder not a copy, so play safe.
var bookFolder = book.FolderPath;
parseId = ""; // in case of early return
// Set this in the metadata so it gets uploaded. Do this in the background task as it can take some time.
// These bits of data can't easily be set while saving the book because we save one page at a time
// and they apply to the book as a whole.
book.BookInfo.LanguageTableReferences = ParseClient.GetLanguagePointers(book.BookData.MakeLanguageUploadData(bookParams.LanguagesToUpload));
book.BookInfo.PageCount = book.GetPages().Count();
book.BookInfo.Save();
// If the caller wants to preserve existing thumbnails, recreate them only if one or more of them do not exist.
var thumbnailsExist = File.Exists(Path.Combine(bookFolder, "thumbnail-70.png")) &&
File.Exists(Path.Combine(bookFolder, "thumbnail-256.png")) &&
File.Exists(Path.Combine(bookFolder, "thumbnail.png"));
if (!bookParams.PreserveThumbnails || !thumbnailsExist)
{
var thumbnailMsg = LocalizationManager.GetString("PublishTab.Upload.MakingThumbnail", "Making thumbnail image...");
progress.WriteStatus(thumbnailMsg);
//the largest thumbnail I found on Amazon was 300px high. Prathambooks.org about the same.
_thumbnailer.MakeThumbnailOfCover(book, 70); // this is a sacrificial one to prime the pump, to fix BL-2673
_thumbnailer.MakeThumbnailOfCover(book, 70);
if (progress.CancelRequested)
{
return("");
}
_thumbnailer.MakeThumbnailOfCover(book, 256);
if (progress.CancelRequested)
{
return("");
}
// It is possible the user never went back to the Collection tab after creating/updating the book, in which case
// the 'normal' thumbnail never got created/updating. See http://issues.bloomlibrary.org/youtrack/issue/BL-3469.
_thumbnailer.MakeThumbnailOfCover(book);
if (progress.CancelRequested)
{
return("");
}
}
var uploadPdfPath = UploadPdfPath(bookFolder);
// If there is not already a locked preview in the book folder
// (which we take to mean the user has created a customized one that he prefers),
// make sure we have a current correct preview and then copy it to the book folder so it gets uploaded.
if (!FileHelper.IsLocked(uploadPdfPath))
{
var pdfMsg = LocalizationManager.GetString("PublishTab.Upload.MakingPdf", "Making PDF Preview...");
progress.WriteStatus(pdfMsg);
publishView.MakePDFForUpload(progress);
if (RobustFile.Exists(publishView.PdfPreviewPath))
{
RobustFile.Copy(publishView.PdfPreviewPath, uploadPdfPath, true);
}
else
{
return(""); // no PDF, no upload (See BL-6719)
}
}
if (progress.CancelRequested)
{
return("");
}
return(UploadBook(bookFolder, progress, out parseId, Path.GetFileName(uploadPdfPath),
GetAudioFilesToInclude(book, bookParams.ExcludeNarrationAudio, bookParams.ExcludeMusic), GetVideoFilesToInclude(book),
bookParams.LanguagesToUpload, book.CollectionSettings));
}
/// <summary>
/// Common routine used in normal upload and bulk upload.
/// </summary>
internal string FullUpload(Book.Book book, IProgress progress, PublishView publishView, BookUploadParameters bookParams, out string parseId)
{
// this (isForPublish:true) is dangerous and the product of much discussion.
// See "finally" block later to see that we put branding files back
book.Storage.CleanupUnusedSupportFiles(isForPublish: true);
try
{
var bookFolder = book.FolderPath;
parseId = ""; // in case of early return
var languagesToUpload = book.BookInfo.PublishSettings.BloomLibrary.TextLangs.IncludedLanguages().ToArray();
// When initializing, we may set the collection's sign language to IncludeByDefault so the checkbox on the publish screen
// gets set by default. Also, videos could have been removed since the user last visited the publish screen (e.g. bulk upload).
// So we need to make sure we have videos before including the sign language.
if (book.HasSignLanguageVideos())
{
languagesToUpload = languagesToUpload.Union(book.BookInfo.PublishSettings.BloomLibrary.SignLangs.IncludedLanguages()).ToArray();
}
// Set this in the metadata so it gets uploaded. Do this in the background task as it can take some time.
// These bits of data can't easily be set while saving the book because we save one page at a time
// and they apply to the book as a whole.
book.BookInfo.LanguageTableReferences =
ParseClient.GetLanguagePointers(book.BookData.MakeLanguageUploadData(languagesToUpload));
book.BookInfo.PageCount = book.GetPages().Count();
book.BookInfo.Save();
// If the caller wants to preserve existing thumbnails, recreate them only if one or more of them do not exist.
var thumbnailsExist = File.Exists(Path.Combine(bookFolder, "thumbnail-70.png")) &&
File.Exists(Path.Combine(bookFolder, "thumbnail-256.png")) &&
File.Exists(Path.Combine(bookFolder, "thumbnail.png"));
if (!bookParams.PreserveThumbnails || !thumbnailsExist)
{
var thumbnailMsg = LocalizationManager.GetString("PublishTab.Upload.MakingThumbnail",
"Making thumbnail image...");
progress.WriteStatus(thumbnailMsg);
//the largest thumbnail I found on Amazon was 300px high. Prathambooks.org about the same.
_thumbnailer.MakeThumbnailOfCover(book,
70); // this is a sacrificial one to prime the pump, to fix BL-2673
_thumbnailer.MakeThumbnailOfCover(book, 70);
if (progress.CancelRequested)
{
return("");
}
_thumbnailer.MakeThumbnailOfCover(book, 256);
if (progress.CancelRequested)
{
return("");
}
// It is possible the user never went back to the Collection tab after creating/updating the book, in which case
// the 'normal' thumbnail never got created/updating. See http://issues.bloomlibrary.org/youtrack/issue/BL-3469.
_thumbnailer.MakeThumbnailOfCover(book);
if (progress.CancelRequested)
{
return("");
}
}
var uploadPdfPath = UploadPdfPath(bookFolder);
var videoFiles = GetVideoFilesToInclude(book);
bool hasVideo = videoFiles.Any();
if (hasVideo)
{
var skipPdfMsg = LocalizationManager.GetString("PublishTab.Upload.SkipMakingPdf", "Skipping PDF because this book has video");
progress.WriteStatus(skipPdfMsg);
}
else
{
// If there is not already a locked preview in the book folder
// (which we take to mean the user has created a customized one that he prefers),
// make sure we have a current correct preview and then copy it to the book folder so it gets uploaded.
if (!FileHelper.IsLocked(uploadPdfPath))
{
var pdfMsg = LocalizationManager.GetString("PublishTab.Upload.MakingPdf", "Making PDF Preview...");
progress.WriteStatus(pdfMsg);
publishView.MakePDFForUpload(progress);
if (RobustFile.Exists(publishView.PdfPreviewPath))
{
RobustFile.Copy(publishView.PdfPreviewPath, uploadPdfPath, true);
}
else
{
return(""); // no PDF, no upload (See BL-6719)
}
}
}
if (progress.CancelRequested)
{
return("");
}
return(UploadBook(bookFolder,
progress,
out parseId,
hasVideo ? null : Path.GetFileName(uploadPdfPath),
GetAudioFilesToInclude(book, bookParams.ExcludeMusic),
videoFiles,
languagesToUpload,
book.CollectionSettings));
}
finally
{
// Put back all the branding files which we removed above in the call to CleanupUnusedSupportFiles()
book.UpdateSupportFiles();
// NB: alternatively, we considered refactoring CleanupUnusedSupportFiles() to give us a list of files
// to not upload.
}
}
