/// <summary>
/// Worker function for a background thread task. See first lines for required args passed to RunWorkerAsync, which triggers this.
/// </summary>
/// <param name="sender"></param>
/// <param name="doWorkEventArgs"></param>
private void BackgroundUpload(object sender, DoWorkEventArgs doWorkEventArgs)
{
var args = (object[])doWorkEventArgs.Argument;
var folder = (string)args[0];
var dlg = (BulkUploadProgressDlg)args[1];
var appContext = (ApplicationContainer)args[2];
var excludeAudio = (bool)args[3];
var alreadyUploaded = GetUploadLogIfPresent(folder);
ProjectContext context = null; // Expensive to create; hold each one we make until we find a book that needs a different one.
try
{
UploadInternal(folder, dlg, appContext, excludeAudio, alreadyUploaded, ref context);
// If we make it here, append a "finished" note to our log file
AppendBookToUploadLogFile("\n\nAll finished!\nIn order to repeat the uploading, this file will need to be deleted.");
}
finally
{
context?.Dispose();
}
}
/// <summary>
/// Handles the recursion through directories: if a folder looks like a Bloom book upload it; otherwise, try its children.
/// Invisible folders like .hg are ignored.
/// </summary>
/// <param name="folder"></param>
/// <param name="dlg"></param>
/// <param name="container"></param>
/// <param name="context"></param>
private void UploadInternal(string folder, BulkUploadProgressDlg dlg, ApplicationContainer container, ref ProjectContext context)
{
if (Path.GetFileName(folder).StartsWith("."))
return; // secret folder, probably .hg
if (Directory.GetFiles(folder, "*.htm").Count() == 1)
{
// Exactly one htm file, assume this is a bloom book folder.
dlg.Progress.WriteMessage("Starting to upload " + folder);
// Make sure the files we want to upload are up to date.
// Unfortunately this requires making a book object, which requires making a ProjectContext, which must be created with the
// proper parent book collection if possible.
var parent = Path.GetDirectoryName(folder);
var collectionPath = Directory.GetFiles(parent, "*.bloomCollection").FirstOrDefault();
if (collectionPath == null && context == null)
{
collectionPath = Settings.Default.MruProjects.Latest;
}
if (context == null || context.SettingsPath != collectionPath)
{
if (context != null)
context.Dispose();
// optimise: creating a context seems to be quite expensive. Probably the only thing we need to change is
// the collection. If we could update that in place...despite autofac being told it has lifetime scope...we would save some time.
// Note however that it's not good enough to just store it in the project context. The one that is actually in
// the autofac object (_scope in the ProjectContext) is used by autofac to create various objects, in particular, books.
context = container.CreateProjectContext(collectionPath);
}
var server = context.BookServer;
var book = server.GetBookFromBookInfo(new BookInfo(folder, true));
book.BringBookUpToDate(new NullProgress());
// Assemble the various arguments needed to make the objects normally involved in an upload.
// We leave some constructor arguments not actually needed for this purpose null.
var bookSelection = new BookSelection();
bookSelection.SelectBook(book);
var currentEditableCollectionSelection = new CurrentEditableCollectionSelection();
if (collectionPath != null)
{
var collection = new BookCollection(collectionPath, BookCollection.CollectionType.SourceCollection,
bookSelection);
currentEditableCollectionSelection.SelectCollection(collection);
}
var publishModel = new PublishModel(bookSelection, new PdfMaker(), currentEditableCollectionSelection, null, server, _thumbnailer, null);
publishModel.PageLayout = book.GetLayout();
var view = new PublishView(publishModel, new SelectedTabChangedEvent(), new LocalizationChangedEvent(), this, null, null);
string dummy;
// Normally we let the user choose which languages to upload. Here, just the ones that have complete information.
var langDict = book.AllLanguages;
var languagesToUpload = langDict.Keys.Where(l => langDict[l]).ToArray();
if (languagesToUpload.Any())
FullUpload(book, dlg.Progress, view, languagesToUpload, out dummy, dlg);
return;
}
foreach (var sub in Directory.GetDirectories(folder))
UploadInternal(sub, dlg, container, ref context);
}
/// <summary>
/// Upload bloom books in the specified folder to the bloom library.
/// Folders that contain exactly one .htm file are interpreted as books and uploaded.
/// Other folders are searched recursively for children that appear to be bloom books.
/// The parent folder of a bloom book is searched for a .bloomCollection file and, if one is found,
/// the book is treated as part of that collection (e.g., for determining vernacular language).
/// If the .bloomCollection file is not found, the book is not uploaded.
/// N.B. The bulk upload process will go ahead and upload templates and books that are already on the server
/// (over-writing the existing book) without informing the user.
/// </summary>
/// <remarks>This method is triggered by starting Bloom with "upload" on the cmd line.</remarks>
public void BulkUpload(ApplicationContainer container, UploadParameters options)
{
BookUpload.Destination = options.Dest;
using (var progress = new MultiProgress())
{
var logFilePath = Path.Combine(options.Path, "BloomBulkUploadLog.txt");
progress.Add(new Bloom.Utils.ConsoleProgress());
progress.Add(new FileLogProgress(logFilePath));
if (!_singleBookUploader.IsThisVersionAllowedToUpload())
{
var oldVersionMsg = LocalizationManager.GetString("PublishTab.Upload.OldVersion",
"Sorry, this version of Bloom Desktop is not compatible with the current version of BloomLibrary.org. Please upgrade to a newer version.");
progress.WriteMessage(oldVersionMsg);
return;
}
Debug.Assert(!String.IsNullOrWhiteSpace(options.UploadUser));
if (!_singleBookUploader.ParseClient.AttemptSignInAgainForCommandLine(options.UploadUser, options.Dest, progress))
{
progress.WriteError("Problem logging in. See messages above.");
System.Environment.Exit(1);
}
progress.WriteMessage("Uploading books as user {0}", options.UploadUser);
var bookParams = new BookUploadParameters(options);
BulkRepairInstanceIds(options.Path);
ProjectContext
context = null; // Expensive to create; hold each one we make until we find a book that needs a different one.
try
{
_collectionFoldersUploaded = new HashSet <string>();
_newBooksUploaded = 0;
_booksUpdated = 0;
_booksSkipped = 0;
_booksWithErrors = 0;
progress.WriteMessageWithColor("green", $"\n\nStarting upload at {DateTime.Now.ToString()}\n");
progress.WriteMessageWithColor("Magenta", $"Looking in '{bookParams.Folder}'...");
UploadCollectionOrKeepLookingDeeper(progress, container, bookParams, ref context);
if (_collectionFoldersUploaded.Count > 0)
{
progress.WriteMessageWithColor("green", "\n\nAll finished!");
progress.WriteMessage("Processed {0} collection folders.", _collectionFoldersUploaded.Count);
}
else
{
progress.WriteError("Did not find any collections to upload.");
}
progress.WriteMessage("Uploaded {0} new books.", _newBooksUploaded);
progress.WriteMessage("Updated {0} books that had changed.", _booksUpdated);
progress.WriteMessage("Skipped {0} books that had not changed.", _booksSkipped);
if (_booksSkipped > 0)
{
progress.WriteMessage("(If you don't want Bloom to skip books it thinks have not changed, you can use the --force argument to force all books to re-upload, or just use the Bloom UI to force upload this one book).");
}
if (_booksWithErrors > 0)
{
progress.WriteError("Failed to upload {0} books. See \"{1}\" for details.", _booksWithErrors,
logFilePath);
}
}
finally
{
context?.Dispose();
}
}
}
private void UploadBookInternal(IProgress progress, ApplicationContainer container, BookUploadParameters uploadParams,
ref ProjectContext context)
{
progress.WriteMessageWithColor("Cyan", "Starting to upload " + uploadParams.Folder);
// Make sure the files we want to upload are up to date.
// Unfortunately this requires making a book object, which requires making a ProjectContext, which must be created with the
// proper parent book collection if possible.
var parent = Path.GetDirectoryName(uploadParams.Folder);
var collectionPath = Directory.GetFiles(parent, "*.bloomCollection").FirstOrDefault();
if (collectionPath == null || !RobustFile.Exists(collectionPath))
{
progress.WriteError("Skipping book because no collection file was found in its parent directory.");
return;
}
_collectionFoldersUploaded.Add(collectionPath);
// Compute the book hash file and compare it to the existing one for bulk upload.
var currentHashes = BookUpload.HashBookFolder(uploadParams.Folder);
progress.WriteMessage(currentHashes);
var pathToLocalHashInfoFromLastUpload = Path.Combine(uploadParams.Folder, HashInfoFromLastUpload);
if (!uploadParams.ForceUpload)
{
var canSkip = false;
if (Program.RunningUnitTests)
{
canSkip = _singleBookUploader.CheckAgainstLocalHashfile(currentHashes, pathToLocalHashInfoFromLastUpload);
}
else
{
canSkip = _singleBookUploader.CheckAgainstHashFileOnS3(currentHashes, uploadParams.Folder, progress);
RobustFile.WriteAllText(pathToLocalHashInfoFromLastUpload, currentHashes); // ensure local copy is saved
}
if (canSkip)
{
// local copy of hashes file is identical or has been saved
progress.WriteMessageWithColor("green", $"Skipping '{Path.GetFileName(uploadParams.Folder)}' because it has not changed since being uploaded.");
++_booksSkipped;
return; // skip this one; we already uploaded it earlier.
}
}
// save local copy of hashes file: it will be uploaded with the other book files
RobustFile.WriteAllText(pathToLocalHashInfoFromLastUpload, currentHashes);
if (context == null || context.SettingsPath != collectionPath)
{
context?.Dispose();
// optimise: creating a context seems to be quite expensive. Probably the only thing we need to change is
// the collection. If we could update that in place...despite autofac being told it has lifetime scope...we would save some time.
// Note however that it's not good enough to just store it in the project context. The one that is actually in
// the autofac object (_scope in the ProjectContext) is used by autofac to create various objects, in particular, books.
context = container.CreateProjectContext(collectionPath);
Program.SetProjectContext(context);
}
var server = context.BookServer;
var bookInfo = new BookInfo(uploadParams.Folder, true);
var book = server.GetBookFromBookInfo(bookInfo);
book.BringBookUpToDate(new NullProgress());
bookInfo.Bookshelf = book.CollectionSettings.DefaultBookshelf;
var bookshelfName = String.IsNullOrWhiteSpace(book.CollectionSettings.DefaultBookshelf) ? "(none)" : book.CollectionSettings.DefaultBookshelf;
progress.WriteMessage($"Bookshelf is '{bookshelfName}'");
// Assemble the various arguments needed to make the objects normally involved in an upload.
// We leave some constructor arguments not actually needed for this purpose null.
var bookSelection = new BookSelection();
bookSelection.SelectBook(book);
var currentEditableCollectionSelection = new CurrentEditableCollectionSelection();
var collection = new BookCollection(collectionPath, BookCollection.CollectionType.SourceCollection, bookSelection);
currentEditableCollectionSelection.SelectCollection(collection);
var publishModel = new PublishModel(bookSelection, new PdfMaker(), currentEditableCollectionSelection, context.Settings, server, _thumbnailer);
publishModel.PageLayout = book.GetLayout();
var view = new PublishView(publishModel, new SelectedTabChangedEvent(), new LocalizationChangedEvent(), _singleBookUploader, null, null, null, null);
var blPublishModel = new BloomLibraryPublishModel(_singleBookUploader, book, publishModel);
string dummy;
// Normally we let the user choose which languages to upload. Here, just the ones that have complete information.
var langDict = book.AllPublishableLanguages();
var languagesToUpload = langDict.Keys.Where(l => langDict[l]).ToList();
if (!string.IsNullOrEmpty(book.CollectionSettings.SignLanguageIso639Code) && BookUpload.GetVideoFilesToInclude(book).Any())
{
languagesToUpload.Insert(0, book.CollectionSettings.SignLanguageIso639Code);
}
if (blPublishModel.MetadataIsReadyToPublish && (languagesToUpload.Any() || blPublishModel.OkToUploadWithNoLanguages))
{
if (blPublishModel.BookIsAlreadyOnServer)
{
var msg = $"Overwriting the copy of {uploadParams.Folder} on the server...";
progress.WriteWarning(msg);
}
using (var tempFolder = new TemporaryFolder(Path.Combine("BloomUpload", Path.GetFileName(book.FolderPath))))
{
BookUpload.PrepareBookForUpload(ref book, server, tempFolder.FolderPath, progress);
uploadParams.LanguagesToUpload = languagesToUpload.ToArray();
_singleBookUploader.FullUpload(book, progress, view, uploadParams, out dummy);
}
progress.WriteMessageWithColor("Green", "{0} has been uploaded", uploadParams.Folder);
if (blPublishModel.BookIsAlreadyOnServer)
{
++_booksUpdated;
}
else
{
++_newBooksUploaded;
}
}
else
{
// report to the user why we are not uploading their book
var reason = blPublishModel.GetReasonForNotUploadingBook();
progress.WriteError("{0} was not uploaded. {1}", uploadParams.Folder, reason);
++_booksWithErrors;
}
}
/// <summary>
/// Handles the recursion through directories: if a folder looks like a Bloom book upload it; otherwise, try its children.
/// Invisible folders like .hg are ignored.
/// </summary>
/// <param name="folder"></param>
/// <param name="dlg"></param>
/// <param name="container"></param>
/// <param name="excludeAudio"></param>
/// <param name="alreadyUploaded"></param>
/// <param name="context"></param>
private void UploadInternal(string folder, BulkUploadProgressDlg dlg, ApplicationContainer container, bool excludeAudio, string[] alreadyUploaded, ref ProjectContext context)
{
var lastFolderPart = Path.GetFileName(folder);
if (lastFolderPart != null && lastFolderPart.StartsWith("."))
{
return; // secret folder, probably .hg
}
if (Directory.GetFiles(folder, "*.htm").Length == 1)
{
if (alreadyUploaded.Contains(folder))
{
return; // skip this one; we already successfully uploaded it at some point
}
// Exactly one htm file, assume this is a bloom book folder.
dlg.Progress.WriteMessage("Starting to upload " + folder);
// Make sure the files we want to upload are up to date.
// Unfortunately this requires making a book object, which requires making a ProjectContext, which must be created with the
// proper parent book collection if possible.
var parent = Path.GetDirectoryName(folder);
var collectionPath = Directory.GetFiles(parent, "*.bloomCollection").FirstOrDefault();
if (collectionPath == null)
{
collectionPath = Settings.Default.MruProjects.Latest;
}
if (collectionPath == null)
{
throw new ApplicationException("Collection not found in this or parent directory.");
}
if (context == null || context.SettingsPath != collectionPath)
{
context?.Dispose();
// optimise: creating a context seems to be quite expensive. Probably the only thing we need to change is
// the collection. If we could update that in place...despite autofac being told it has lifetime scope...we would save some time.
// Note however that it's not good enough to just store it in the project context. The one that is actually in
// the autofac object (_scope in the ProjectContext) is used by autofac to create various objects, in particular, books.
context = container.CreateProjectContext(collectionPath);
Program.SetProjectContext(context);
}
var server = context.BookServer;
var bookInfo = new BookInfo(folder, true);
bookInfo.BookshelfList = GetBookshelfName(folder);
var book = server.GetBookFromBookInfo(bookInfo);
book.BringBookUpToDate(new NullProgress());
// Assemble the various arguments needed to make the objects normally involved in an upload.
// We leave some constructor arguments not actually needed for this purpose null.
var bookSelection = new BookSelection();
bookSelection.SelectBook(book);
var currentEditableCollectionSelection = new CurrentEditableCollectionSelection();
var collection = new BookCollection(collectionPath, BookCollection.CollectionType.SourceCollection, bookSelection);
currentEditableCollectionSelection.SelectCollection(collection);
var publishModel = new PublishModel(bookSelection, new PdfMaker(), currentEditableCollectionSelection, context.Settings, server, _thumbnailer, null);
publishModel.PageLayout = book.GetLayout();
var view = new PublishView(publishModel, new SelectedTabChangedEvent(), new LocalizationChangedEvent(), this, null, null, null);
var blPublishModel = new BloomLibraryPublishModel(this, book);
string dummy;
// Normally we let the user choose which languages to upload. Here, just the ones that have complete information.
var langDict = book.AllLanguages;
var languagesToUpload = langDict.Keys.Where(l => langDict[l]).ToArray();
if (blPublishModel.MetadataIsReadyToPublish && (languagesToUpload.Any() || blPublishModel.OkToUploadWithNoLanguages))
{
if (blPublishModel.BookIsAlreadyOnServer)
{
var msg = "Apparently this book is already on the server. Overwriting...";
ReportToLogBoxAndLogger(dlg.Progress, folder, msg);
}
FullUpload(book, dlg.Progress, view, languagesToUpload, out dummy, excludeAudio);
AppendBookToUploadLogFile(folder);
}
else
{
// report to the user why we are not uploading their book
ReportToLogBoxAndLogger(dlg.Progress, folder, blPublishModel.GetReasonForNotUploadingBook());
}
return;
}
foreach (var sub in Directory.GetDirectories(folder))
{
UploadInternal(sub, dlg, container, excludeAudio, alreadyUploaded, ref context);
}
}
public void Dispose()
{
_context.Dispose();
}
protected override void Dispose(bool disposing)
{
db.Dispose();
base.Dispose(disposing);
}
public void Dispose()
{
ctx.Dispose();
}
public void Dispose()
{
context.Dispose();
GC.SuppressFinalize(this);
service = null;
}
public void Dispose()
{
_projectContext.Dispose();
}
public void Dispose()
{
Db.Dispose();
GC.SuppressFinalize(this);
}
private void LoginForm_FormClosed(object sender, FormClosedEventArgs e)
{
_context.Dispose();
}
public void Dispose()
{
DbContext.Dispose();
}
public void Dispose()
{
context.Dispose();
//throw new NotImplementedException();
}
