async Task <bool> ExtractPartAndValidate(PartialZipDownload part, Stream partInputStream, string cacheDirectory)
{
string fileHash = null;
var outputPath = GetOutputPath(cacheDirectory, part);
using (var iis = new System.IO.Compression.DeflateStream(partInputStream, System.IO.Compression.CompressionMode.Decompress))
//using (var iis = new ICSharpCode.SharpZipLib.Zip.Compression.Streams.InflaterInputStream (partInputStream, new ICSharpCode.SharpZipLib.Zip.Compression.Inflater (true)))
using (var fs = File.Open(outputPath, FileMode.Create)) {
await iis.CopyToAsync(fs).ConfigureAwait(false);
await fs.FlushAsync().ConfigureAwait(false);
fs.Seek(0, SeekOrigin.Begin);
fileHash = DownloadUtils.HashMd5(fs);
LogDebugMessage("Hash of Downloaded File: {0}", fileHash);
fs.Close();
}
if (!string.IsNullOrEmpty(part.Md5) && !part.Md5.Equals(fileHash, StringComparison.InvariantCultureIgnoreCase))
{
// TODO: HANDLE
LogMessage("File MD5 Hash was invalid, deleting file: {0}", part.ToFile);
File.Delete(outputPath);
return(false);
}
return(true);
}
// We intentionally won't call the base implementation in this override
// since other tasks should handle the restores
// This task is just responsible for extracting proguard config files
// from the .aar input files so we are reusing the base task to help
// track down the .aar files themselves.
public override bool Execute()
{
// Get the dir to store proguard config files in
proguardIntermediateOutputPath = Path.Combine(MergeOutputDir, "proguard");
if (!Directory.Exists(proguardIntermediateOutputPath))
{
Directory.CreateDirectory(proguardIntermediateOutputPath);
}
var additionalFileWrites = new List <ITaskItem> ();
// Make sure our XbdMerge directory exists
var outputDir = MergeOutputDir;
Directory.CreateDirectory(outputDir);
// Get our assembly restore map
var restoreMap = BuildRestoreMap(RestoreAssemblyResources);
if (restoreMap == null)
{
return(false);
}
// Look through all the assemblies we would restore for
foreach (var asm in restoreMap)
{
// We only want to find proguard files in .aar files referenced
// for assemblies we actually have referenced and have them mapped to
var asmName = new AssemblyName(asm.Key);
ITaskItem item = FindMatchingAssembly(InputReferencePaths, asmName);
if (item == null)
{
if (ThrowOnMissingAssembly)
{
return(false);
}
else
{
continue;
}
}
// Use a hash for the assembly name to keep paths shorter
var saveNameHash = DownloadUtils.HashMd5(asmName.Name)?.Substring(0, 8);
// We keep a stamp file around to avoid reprocessing, so skip if it exists
var stampPath = Path.Combine(outputDir, saveNameHash + ".proguard.stamp");
if (File.Exists(stampPath))
{
continue;
}
// Get all the mapped .aar files
var resourceItems = asm.Value;
// We want to increment on the hash name in case there are multiple .aar files and/or proguard config
// files for a given assembly, so we use them all and not overwrite the same name
var entryCount = 0;
// In theory we could have multiple .aar files? Probably never happen...
foreach (var resourceItem in resourceItems)
{
// Full path to .aar file
var resourceFullPath = resourceItem.GetMetadata("FullPath");
using (var fileStream = File.OpenRead(resourceFullPath))
using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Read)) {
// Look for proguard config files in the archive
foreach (var entry in zipArchive.Entries)
{
// Skip entries which are not proguard configs
if (!entry.Name.Equals("proguard.txt", StringComparison.OrdinalIgnoreCase) &&
!entry.Name.Equals("proguard.cfg", StringComparison.OrdinalIgnoreCase))
{
continue;
}
// Figure out our destination filename
var proguardSaveFilename = Path.Combine(proguardIntermediateOutputPath, saveNameHash + entryCount + ".txt");
// Add this to our file writes
additionalFileWrites.Add(new TaskItem(proguardSaveFilename));
// Save out the proguard file
using (var entryStream = entry.Open())
using (var fs = File.Create(proguardSaveFilename)) {
entryStream.CopyTo(fs);
fs.Flush();
fs.Close();
}
entryCount++;
}
}
}
// *.proguard.stamp files are additional file writes
File.WriteAllText(stampPath, string.Empty);
additionalFileWrites.Add(new TaskItem(stampPath));
}
AdditionalFileWrites = additionalFileWrites.ToArray();
return(true);
}
async Task DownloadAll(string cacheDirectory, List <PartialZipDownload> parts)
{
// Get the parts all grouped by their URL so we can batch requests with multiple ranges
// instead of making a request to the same url for each part
// also only grab the parts that don't already locally exist
var uniqueUrls = parts
.Where(p => !File.Exists(Path.Combine(cacheDirectory, p.Id, p.ToFile)))
.GroupBy(p => p.Url);
// For each unique url...
foreach (var partsByUrl in uniqueUrls)
{
var downloadUrl = partsByUrl.Key;
LogMessage("Downloading Partial Zip parts from: " + downloadUrl);
try {
// Create a lock file based on the hash of the URL we are downloading from
// Since we could download a multipart request, we are locking on the url from any other process downloading from it
var lockFile = Path.Combine(cacheDirectory, DownloadUtils.HashMd5(downloadUrl) + ".locked");
using (var lockStream = DownloadUtils.ObtainExclusiveFileLock(lockFile, base.Token, TimeSpan.FromSeconds(30))) {
if (lockStream == null)
{
LogCodedError(ErrorCodes.ExclusiveLockTimeout, "Timed out waiting for exclusive file lock on: {0}", lockFile);
LogMessage("Timed out waiting for an exclusive file lock on: " + lockFile, MessageImportance.High);
// Log Custom error if one was specified in the partial download info
var firstPart = partsByUrl.FirstOrDefault();
if (!string.IsNullOrEmpty(firstPart?.CustomErrorCode) && !string.IsNullOrEmpty(firstPart?.CustomErrorMessage))
{
LogCodedError(firstPart.CustomErrorCode, firstPart.CustomErrorMessage);
}
return;
}
try {
await Download(cacheDirectory, partsByUrl.Key, partsByUrl.ToList()).ConfigureAwait(false);
} catch (Exception ex) {
LogCodedError(ErrorCodes.PartialDownloadFailed, "Partial Download Failed for one or more parts");
LogErrorFromException(ex);
// Log Custom error if one was specified in the partial download info
var firstPart = partsByUrl.FirstOrDefault();
if (!string.IsNullOrEmpty(firstPart?.CustomErrorCode) && !string.IsNullOrEmpty(firstPart?.CustomErrorMessage))
{
LogCodedError(firstPart.CustomErrorCode, firstPart.CustomErrorMessage);
}
}
}
try {
if (File.Exists(lockFile))
{
File.Delete(lockFile);
}
} catch { }
} catch (Exception ex) {
LogCodedError(ErrorCodes.PartialDownloadFailed, "Partial Download Failed for one or more parts");
LogErrorFromException(ex);
// Log Custom error if one was specified in the partial download info
var firstPart = partsByUrl.FirstOrDefault();
if (!string.IsNullOrEmpty(firstPart?.CustomErrorCode) && !string.IsNullOrEmpty(firstPart?.CustomErrorMessage))
{
LogCodedError(firstPart.CustomErrorCode, firstPart.CustomErrorMessage);
}
}
}
}