// Return true if both files exist, are readable, and have the same content.
static bool SameFileContent(string path1, string path2)
{
if (!RobustFile.Exists(path1))
{
return(false);
}
if (!RobustFile.Exists(path2))
{
return(false);
}
try
{
var first = RobustFile.ReadAllBytes(path1);
var second = RobustFile.ReadAllBytes(path2);
if (first.Length != second.Length)
{
return(false);
}
for (int i = 0; i < first.Length; i++)
{
if (first[i] != second[i])
{
return(false);
}
}
return(true);
}
catch (IOException)
{
return(false); // can't even read
}
}
public void HandleModifiedFile_CheckedOutToMe_ContentChangedRemotely_RaisesCheckedOutByNoneAndErrorMessage()
{
// Setup //
// Simulate a book that was checked out and modified by me, but then we get a remote change
// notification.
const string bookFolderName = "My conflicting change book";
var bookBuilder = new BookFolderBuilder()
.WithRootFolder(_collectionFolder.FolderPath)
.WithTitle(bookFolderName)
.WithHtm("We will be simulating a remote change to this.")
.Build();
string bookFolderPath = bookBuilder.BuiltBookFolderPath;
string bookPath = bookBuilder.BuiltBookHtmPath;
_collection.PutBook(bookFolderPath);
var pathToBookFileInRepo = _collection.GetPathToBookFileInRepo(bookFolderName);
// Save the data we will eventually write back to the .bloom file to simulate the remote change.
var remoteContent = RobustFile.ReadAllBytes(pathToBookFileInRepo);
_collection.AttemptLock(bookFolderName);
RobustFile.WriteAllText(bookPath, "Pretend this was the state when we checked it out.");
_collection.PutBook(bookFolderPath);
RobustFile.WriteAllText(bookPath, "This is a further change locally, not checked in anywhere");
// But now it's been changed remotely to the other state. (Ignore the fact that it was a previous local state;
// that was just a trick to get a valid alternative state.)
RobustFile.WriteAllBytes(pathToBookFileInRepo, remoteContent);
var prevMessages = _tcLog.Messages.Count;
// System Under Test //
_collection.HandleModifiedFile(new BookRepoChangeEventArgs()
{
BookFileName = $"{bookFolderName}.bloom"
});
// Verification
var eventArgs = (BookStatusChangeEventArgs)_mockTcManager.Invocations[0].Arguments[0];
Assert.That(eventArgs.CheckedOutByWhom, Is.EqualTo(CheckedOutBy.None));
Assert.That(_tcLog.Messages[prevMessages].MessageType, Is.EqualTo(MessageAndMilestoneType.Error));
Assert.That(_tcLog.Messages[prevMessages].L10NId, Is.EqualTo("TeamCollection.EditedFileChangedRemotely"));
}
/// <summary>
/// For electronic books, we want to minimize the actual size of images since they'll
/// be displayed on small screens anyway. So before zipping up the file, we replace its
/// bytes with the bytes of a reduced copy of itself. If the original image is already
/// small enough, we return its bytes directly.
/// We also make png images have transparent backgrounds. This is currently only necessary
/// for cover pages, but it's an additional complication to detect which those are,
/// and doesn't seem likely to cost much extra to do.
/// </summary>
/// <returns>The bytes of the (possibly) reduced image.</returns>
internal static byte[] GetBytesOfReducedImage(string filePath)
{
using (var originalImage = PalasoImage.FromFileRobustly(filePath))
{
var image = originalImage.Image;
int originalWidth = image.Width;
int originalHeight = image.Height;
var appearsToBeJpeg = ImageUtils.AppearsToBeJpeg(originalImage);
if (originalWidth > kMaxWidth || originalHeight > kMaxHeight || !appearsToBeJpeg)
{
// Preserve the aspect ratio
float scaleX = (float)kMaxWidth / (float)originalWidth;
float scaleY = (float)kMaxHeight / (float)originalHeight;
// no point in ever expanding, even if we're making a new image just for transparency.
float scale = Math.Min(1.0f, Math.Min(scaleX, scaleY));
// New width and height maintaining the aspect ratio
int newWidth = (int)(originalWidth * scale);
int newHeight = (int)(originalHeight * scale);
var imagePixelFormat = image.PixelFormat;
switch (imagePixelFormat)
{
// These three formats are not supported for bitmaps to be drawn on using Graphics.FromImage.
// So use the default bitmap format.
// Enhance: if these are common it may be worth research to find out whether there are better options.
// - possibly the 'reduced' image might not be reduced...even though smaller, the indexed format
// might be so much more efficient that it is smaller. However, even if that is true, it doesn't
// necessarily follow that it takes less memory to render on the device. So it's not obvious that
// we should keep the original just because it's a smaller file.
// - possibly we don't need a 32-bit bitmap? Unfortunately the 1bpp/4bpp/8bpp only tells us
// that the image uses two, 16, or 256 distinct colors, not what they are or what precision they have.
case PixelFormat.Format1bppIndexed:
case PixelFormat.Format4bppIndexed:
case PixelFormat.Format8bppIndexed:
imagePixelFormat = PixelFormat.Format32bppArgb;
break;
}
// OTOH, always using 32-bit format for .png files keeps us from having problems in BloomReader
// like BL-5740 (where 24bit format files came out in BR with black backgrounds).
if (!appearsToBeJpeg)
{
imagePixelFormat = PixelFormat.Format32bppArgb;
}
using (var newImage = new Bitmap(newWidth, newHeight, imagePixelFormat))
{
// Draws the image in the specified size with quality mode set to HighQuality
using (Graphics graphics = Graphics.FromImage(newImage))
{
graphics.CompositingQuality = CompositingQuality.HighQuality;
graphics.InterpolationMode = InterpolationMode.HighQualityBicubic;
graphics.SmoothingMode = SmoothingMode.HighQuality;
using (var imageAttributes = new ImageAttributes())
{
// See https://stackoverflow.com/a/11850971/7442826
// Fixes the 50% gray border issue on bright white or dark images
imageAttributes.SetWrapMode(WrapMode.TileFlipXY);
// In addition to possibly scaling, we want PNG images to have transparent backgrounds.
if (!appearsToBeJpeg)
{
// This specifies that all white or very-near-white pixels (all color components at least 253/255)
// will be made transparent.
imageAttributes.SetColorKey(Color.FromArgb(253, 253, 253), Color.White);
}
var destRect = new Rectangle(0, 0, newWidth, newHeight);
graphics.DrawImage(image, destRect, 0, 0, image.Width, image.Height,
GraphicsUnit.Pixel, imageAttributes);
}
}
// Save the file in the same format as the original, and return its bytes.
using (var tempFile = TempFile.WithExtension(Path.GetExtension(filePath)))
{
// This uses default quality settings for jpgs...one site says this is
// 75 quality on a scale that runs from 0-100. For most images, this
// should give a quality barely distinguishable from uncompressed and still save
// about 7/8 of the file size. Lower quality settings rapidly lose quality
// while only saving a little space; higher ones rapidly use more space
// with only marginal quality improvement.
// See https://photo.stackexchange.com/questions/30243/what-quality-to-choose-when-converting-to-jpg
// for more on quality and https://docs.microsoft.com/en-us/dotnet/framework/winforms/advanced/how-to-set-jpeg-compression-level
// for how to control the quality setting if we decide to (RobustImageIO has
// suitable overloads).
RobustImageIO.SaveImage(newImage, tempFile.Path, image.RawFormat);
// Copy the metadata from the original file to the new file.
var metadata = SIL.Windows.Forms.ClearShare.Metadata.FromFile(filePath);
if (!metadata.IsEmpty)
{
metadata.Write(tempFile.Path);
}
return(RobustFile.ReadAllBytes(tempFile.Path));
}
}
}
}
return(RobustFile.ReadAllBytes(filePath));
}
/// <summary>
/// Adds a directory, along with all files and subdirectories, to the ZipStream.
/// </summary>
/// <param name="directoryToCompress">The directory to add recursively</param>
/// <param name="zipStream">The ZipStream to which the files and directories will be added</param>
/// <param name="dirNameOffset">This number of characters will be removed from the full directory or file name
/// before creating the zip entry name</param>
/// <param name="dirNamePrefix">string to prefix to the zip entry name</param>
/// <param name="depthFromCollection">int with the number of folders away it is from the collection folder. The collection folder itself is 0,
/// a book is 1, a subfolder of the book is 2, etc.</param>
/// <param name="forReaderTools">If True, then some pre-processing will be done to the contents of decodable
/// and leveled readers before they are added to the ZipStream</param>
/// <param name="excludeAudio">If true, the contents of the audio directory will not be included</param>
/// <param name="reduceImages">If true, image files are reduced in size to no larger than the max size before saving</para>
/// <param name="omitMetaJson">If true, meta.json is excluded (typically for HTML readers).</param>
private static void CompressDirectory(string directoryToCompress, ZipOutputStream zipStream, int dirNameOffset, string dirNamePrefix,
int depthFromCollection, bool forReaderTools, bool excludeAudio, bool reduceImages, bool omitMetaJson = false, string pathToFileForSha = null)
{
if (excludeAudio && Path.GetFileName(directoryToCompress).ToLowerInvariant() == "audio")
{
return;
}
var files = Directory.GetFiles(directoryToCompress);
// Don't get distracted by HTML files in any folder other than the book folder.
// These HTML files in other locations aren't generated by Bloom. They may not have the format Bloom expects,
// causing needless parsing errors to be thrown if we attempt to read them using Bloom code.
bool shouldScanHtml = depthFromCollection == 1; // 1 means 1 level below the collection level, i.e. this is the book level
var bookFile = shouldScanHtml ? BookStorage.FindBookHtmlInFolder(directoryToCompress) : null;
XmlDocument dom = null;
List <string> imagesToGiveTransparentBackgrounds = null;
List <string> imagesToPreserveResolution = null;
// Tests can also result in bookFile being null.
if (!String.IsNullOrEmpty(bookFile))
{
var originalContent = File.ReadAllText(bookFile, Encoding.UTF8);
dom = XmlHtmlConverter.GetXmlDomFromHtml(originalContent);
var fullScreenAttr = dom.GetElementsByTagName("body").Cast <XmlElement>().First().Attributes["data-bffullscreenpicture"]?.Value;
if (fullScreenAttr != null && fullScreenAttr.IndexOf("bloomReader", StringComparison.InvariantCulture) >= 0)
{
// This feature (currently used for motion books in landscape mode) triggers an all-black background,
// due to a rule in bookFeatures.less.
// Making white pixels transparent on an all-black background makes line-art disappear,
// which is bad (BL-6564), so just make an empty list in this case.
imagesToGiveTransparentBackgrounds = new List <string>();
}
else
{
imagesToGiveTransparentBackgrounds = FindCoverImages(dom);
}
imagesToPreserveResolution = FindImagesToPreserveResolution(dom);
FindBackgroundAudioFiles(dom);
}
else
{
imagesToGiveTransparentBackgrounds = new List <string>();
imagesToPreserveResolution = new List <string>();
}
// Some of the knowledge about ExcludedFileExtensions might one day move into this method.
// But we'd have to check carefully the other places it is used.
var localOnlyFiles = BookStorage.LocalOnlyFiles(directoryToCompress);
foreach (var filePath in files)
{
if (ExcludedFileExtensionsLowerCase.Contains(Path.GetExtension(filePath.ToLowerInvariant())))
{
continue; // BL-2246: skip putting this one into the BloomPack
}
if (IsUnneededWaveFile(filePath, depthFromCollection))
{
continue;
}
if (localOnlyFiles.Contains(filePath))
{
continue;
}
var fileName = Path.GetFileName(filePath).ToLowerInvariant();
if (fileName.StartsWith(BookStorage.PrefixForCorruptHtmFiles))
{
continue;
}
// Various stuff we keep in the book folder that is useful for editing or bloom library
// or displaying collections but not needed by the reader. The most important is probably
// eliminating the pdf, which can be very large. Note that we do NOT eliminate the
// basic thumbnail.png, as we want eventually to extract that to use in the Reader UI.
if (fileName == "thumbnail-70.png" || fileName == "thumbnail-256.png")
{
continue;
}
if (fileName == "meta.json" && omitMetaJson)
{
continue;
}
FileInfo fi = new FileInfo(filePath);
var entryName = dirNamePrefix + filePath.Substring(dirNameOffset); // Makes the name in zip based on the folder
entryName = ZipEntry.CleanName(entryName); // Removes drive from name and fixes slash direction
ZipEntry newEntry = new ZipEntry(entryName)
{
DateTime = fi.LastWriteTime,
IsUnicodeText = true
};
// encode filename and comment in UTF8
byte[] modifiedContent = {};
// if this is a ReaderTools book, call GetBookReplacedWithTemplate() to get the contents
if (forReaderTools && (bookFile == filePath))
{
modifiedContent = Encoding.UTF8.GetBytes(GetBookReplacedWithTemplate(filePath));
newEntry.Size = modifiedContent.Length;
}
else if (forReaderTools && (Path.GetFileName(filePath) == "meta.json"))
{
modifiedContent = Encoding.UTF8.GetBytes(GetMetaJsonModfiedForTemplate(filePath));
newEntry.Size = modifiedContent.Length;
}
else if (reduceImages && ImageFileExtensions.Contains(Path.GetExtension(filePath.ToLowerInvariant())))
{
fileName = Path.GetFileName(filePath); // restore original capitalization
if (imagesToPreserveResolution.Contains(fileName))
{
modifiedContent = RobustFile.ReadAllBytes(filePath);
}
else
{
// Cover images should be transparent if possible. Others don't need to be.
var makeBackgroundTransparent = imagesToGiveTransparentBackgrounds.Contains(fileName);
modifiedContent = GetImageBytesForElectronicPub(filePath, makeBackgroundTransparent);
}
newEntry.Size = modifiedContent.Length;
}
else if (Path.GetExtension(filePath).ToLowerInvariant() == ".bloomcollection")
{
modifiedContent = Encoding.UTF8.GetBytes(GetBloomCollectionModifiedForTemplate(filePath));
newEntry.Size = modifiedContent.Length;
}
// CompressBookForDevice is always called with reduceImages set.
else if (reduceImages && bookFile == filePath)
{
SignLanguageApi.ProcessVideos(HtmlDom.SelectChildVideoElements(dom.DocumentElement).Cast <XmlElement>(), directoryToCompress);
var newContent = XmlHtmlConverter.ConvertDomToHtml5(dom);
modifiedContent = Encoding.UTF8.GetBytes(newContent);
newEntry.Size = modifiedContent.Length;
if (pathToFileForSha != null)
{
// Make an extra entry containing the sha
var sha = Book.ComputeHashForAllBookRelatedFiles(pathToFileForSha);
var name = "version.txt"; // must match what BloomReader is looking for in NewBookListenerService.IsBookUpToDate()
MakeExtraEntry(zipStream, name, sha);
LastVersionCode = sha;
}
}
else
{
newEntry.Size = fi.Length;
}
zipStream.PutNextEntry(newEntry);
if (modifiedContent.Length > 0)
{
using (var memStream = new MemoryStream(modifiedContent))
{
// There is some minimum buffer size (44 was too small); I don't know exactly what it is,
// but 1024 makes it happy.
StreamUtils.Copy(memStream, zipStream, new byte[Math.Max(modifiedContent.Length, 1024)]);
}
}
else
{
// Zip the file in buffered chunks
byte[] buffer = new byte[4096];
using (var streamReader = RobustFile.OpenRead(filePath))
{
StreamUtils.Copy(streamReader, zipStream, buffer);
}
}
zipStream.CloseEntry();
}
var folders = Directory.GetDirectories(directoryToCompress);
foreach (var folder in folders)
{
var dirName = Path.GetFileName(folder);
if ((dirName == null) || (dirName.ToLowerInvariant() == "sample texts"))
{
continue; // Don't want to bundle these up
}
CompressDirectory(folder, zipStream, dirNameOffset, dirNamePrefix, depthFromCollection + 1, forReaderTools, excludeAudio, reduceImages);
}
}
