/// <summary>
/// Synchronously downloads and assembles a multi-part file as specified by a <see cref="Neon.Deployment.DownloadManifest"/>.
/// </summary>
/// <param name="download">The download details.</param>
/// <param name="targetPath">The target file path.</param>
/// <param name="progressAction">Optionally specifies an action to be called with the the percentage downloaded.</param>
/// <param name="retry">Optionally specifies the retry policy. This defaults to a reasonable policy.</param>
/// <param name="partTimeout">Optionally specifies the HTTP download timeout for each part (defaults to 10 minutes).</param>
/// <exception cref="IOException">Thrown when the download is corrupt.</exception>
/// <exception cref="SocketException">Thrown for network errors.</exception>
/// <exception cref="HttpException">Thrown for HTTP network errors.</exception>
/// <exception cref="OperationCanceledException">Thrown when the operation was cancelled.</exception>
public static void DownloadMultiPart(
DownloadManifest download,
string targetPath,
DownloadProgressDelegate progressAction = null,
IRetryPolicy retry = null,
TimeSpan partTimeout = default)
{
DownloadMultiPartAsync(download, targetPath, progressAction, partTimeout, retry).WaitWithoutAggregate();
}
/// <summary>
/// Uploads a multi-part download to a release and then publishes the release.
/// </summary>
/// <param name="repo">Identifies the target repository.</param>
/// <param name="release">The target release.</param>
/// <param name="sourcePath">Path to the file being uploaded.</param>
/// <param name="version">The download version.</param>
/// <param name="name">Optionally overrides the download file name specified by <paramref name="sourcePath"/> to initialize <see cref="DownloadManifest.Name"/>.</param>
/// <param name="filename">Optionally overrides the download file name specified by <paramref name="sourcePath"/> to initialize <see cref="DownloadManifest.Filename"/>.</param>
/// <param name="noMd5File">
/// This method creates a file named [<paramref name="sourcePath"/>.md5] with the MD5 hash for the entire
/// uploaded file by default. You may override this behavior by passing <paramref name="noMd5File"/>=<c>true</c>.
/// </param>
/// <param name="maxPartSize">Optionally overrides the maximum part size (defailts to 100 MiB).</param>d
/// <returns>The <see cref="DownloadManifest"/>.</returns>
/// <remarks>
/// <para>
/// The release passed must be unpublished and you may upload other assets before calling this.
/// </para>
/// <note>
/// Take care that any assets already published have names that won't conflict with the asset
/// part names, which will be formatted like: <b>part-##</b>
/// </note>
/// </remarks>
public DownloadManifest UploadMultipartAsset(
string repo,
Release release,
string sourcePath,
string version,
string name = null,
string filename = null,
bool noMd5File = false,
long maxPartSize = (long)(100 * ByteUnits.MebiBytes))
{
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(repo), nameof(repo));
Covenant.Requires <ArgumentNullException>(release != null, nameof(release));
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(sourcePath), nameof(sourcePath));
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(version), nameof(version));
name = name ?? Path.GetFileName(sourcePath);
filename = filename ?? Path.GetFileName(sourcePath);
using (var input = File.OpenRead(sourcePath))
{
if (input.Length == 0)
{
throw new IOException($"Asset at [{sourcePath}] cannot be empty.");
}
var assetPartMap = new List <Tuple <ReleaseAsset, DownloadPart> >();
var manifest = new DownloadManifest()
{
Name = name, Version = version, Filename = filename
};
var partCount = NeonHelper.PartitionCount(input.Length, maxPartSize);
var partNumber = 0;
var partStart = 0L;
var cbRemaining = input.Length;
manifest.Md5 = CryptoHelper.ComputeMD5String(input);
input.Position = 0;
while (cbRemaining > 0)
{
var partSize = Math.Min(cbRemaining, maxPartSize);
var part = new DownloadPart()
{
Number = partNumber,
Size = partSize,
};
// We're going to use a substream to compute the MD5 hash for the part
// as well as to actually upload the part to the GitHub release.
using (var partStream = new SubStream(input, partStart, partSize))
{
part.Md5 = CryptoHelper.ComputeMD5String(partStream);
partStream.Position = 0;
var asset = GitHub.Releases.UploadAsset(repo, release, partStream, $"part-{partNumber:0#}");
assetPartMap.Add(new Tuple <ReleaseAsset, DownloadPart>(asset, part));
}
manifest.Parts.Add(part);
// Loop to handle the next part (if any).
partNumber++;
partStart += partSize;
cbRemaining -= partSize;
}
manifest.Size = manifest.Parts.Sum(part => part.Size);
// Publish the release.
var releaseUpdate = release.ToUpdate();
releaseUpdate.Draft = false;
release = GitHub.Releases.Update(repo, release, releaseUpdate);
// Now that the release has been published, we can go back and fill in
// the asset URIs for each of the download parts.
foreach (var item in assetPartMap)
{
item.Item2.Uri = GitHub.Releases.GetAssetUri(release, item.Item1);
}
// Write the MD5 file unless disabled.
if (!noMd5File)
{
File.WriteAllText($"{sourcePath}.md5", manifest.Md5);
}
return(manifest);
}
}
/// <summary>
/// Asynchronously downloads and assembles a multi-part file as specified by a <see cref="Neon.Deployment.DownloadManifest"/>.
/// </summary>
/// <param name="manifest">The download details.</param>
/// <param name="targetPath">The target file path.</param>
/// <param name="progressAction">Optionally specifies an action to be called with the the percentage downloaded.</param>
/// <param name="partTimeout">Optionally specifies the HTTP download timeout for each part (defaults to 10 minutes).</param>
/// <param name="retry">Optionally specifies the retry policy. This defaults to a reasonable policy.</param>
/// <param name="cancellationToken">Optionally specifies the operation cancellation token.</param>
/// <returns>The path to the downloaded file.</returns>
/// <exception cref="IOException">Thrown when the download is corrupt.</exception>
/// <exception cref="SocketException">Thrown for network errors.</exception>
/// <exception cref="HttpException">Thrown for HTTP network errors.</exception>
/// <exception cref="OperationCanceledException">Thrown when the operation was cancelled.</exception>
/// <remarks>
/// <para>
/// This method downloads the file specified by <paramref name="manifest"/> to the folder specified, creating
/// the folder first when required. The file will be downloaded in parts, where each part will be validated
/// by comparing the part's MD5 hash (when present) with the computed value. The output file will be named
/// <see cref="DownloadManifest.Name"/> and the overall MD5 hash will also be saved using the same file name but
/// <b>adding</b> the <b>.md5</b> extension.
/// </para>
/// <para>
/// This method will continue downloading a partially downloaded file. This works by validating the already
/// downloaded parts against their MD5 hashes and then continuing part downloads after the last valid part.
/// Nothing will be downloaded when the existing file is fully formed.
/// </para>
/// <note>
/// The target files (output and MD5) will be deleted when download appears to be corrupt.
/// </note>
/// </remarks>
public static async Task <string> DownloadMultiPartAsync(
DownloadManifest manifest,
string targetPath,
DownloadProgressDelegate progressAction = null,
TimeSpan partTimeout = default,
IRetryPolicy retry = null,
CancellationToken cancellationToken = default)
{
await SyncContext.Clear;
Covenant.Requires <ArgumentNullException>(manifest != null, nameof(manifest));
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(targetPath), nameof(targetPath));
retry = retry ?? new ExponentialRetryPolicy(TransientDetector.NetworkOrHttp, maxAttempts: 5);
if (partTimeout <= TimeSpan.Zero)
{
partTimeout = TimeSpan.FromMinutes(10);
}
var targetFolder = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(targetFolder);
var targetMd5Path = Path.Combine(Path.GetDirectoryName(targetPath), Path.GetFileName(targetPath) + ".md5");
var nextPartNumber = 0;
// If the target file already exists along with its MD5 hash file:
//
// 1. Compare the manifest MD5 with the local MD5 hash file and
// quickly continue with the download when these don't match.
//
// 2. When the MD5 files match, compute the MD5 of the downloaded
// file and compare that with the manifest and continue with
// the download when these don't match.
if (File.Exists(targetPath) && File.Exists(targetMd5Path) && File.ReadAllText(targetMd5Path).Trim() == manifest.Md5)
{
if (File.ReadAllText(targetMd5Path).Trim() == manifest.Md5)
{
using (var downloadStream = File.OpenRead(targetPath))
{
if (CryptoHelper.ComputeMD5String(downloadStream) == manifest.Md5)
{
return(targetPath);
}
}
}
}
NeonHelper.DeleteFile(targetMd5Path); // We'll recompute this below
// Validate the parts of any existing target file to determine where
// to start downloading missing parts.
if (File.Exists(targetPath))
{
using (var output = new FileStream(targetPath, System.IO.FileMode.Open, FileAccess.ReadWrite))
{
var pos = 0L;
foreach (var part in manifest.Parts.OrderBy(part => part.Number))
{
progressAction?.Invoke(DownloadProgressType.Check, (int)((double)pos / (double)manifest.Size * 100.0));
// Handle a partially downloaded part. We're going to truncate the file to
// remove the partial part and then break to start re-downloading the part.
if (output.Length < pos + part.Size)
{
output.SetLength(pos);
nextPartNumber = part.Number;
break;
}
// Validate the part MD5. We're going to truncate the file to remove the
// partial part and then break to start re-downloading the part.
using (var partStream = new SubStream(output, pos, part.Size))
{
if (CryptoHelper.ComputeMD5String(partStream) != part.Md5)
{
output.SetLength(pos);
nextPartNumber = part.Number;
break;
}
}
pos += part.Size;
nextPartNumber = part.Number + 1;
}
}
}
// Download any remaining parts.
if (progressAction != null && !progressAction.Invoke(DownloadProgressType.Download, 0))
{
return(targetPath);
}
if (nextPartNumber == manifest.Parts.Count)
{
progressAction?.Invoke(DownloadProgressType.Download, 100);
return(targetPath);
}
try
{
using (var httpClient = new HttpClient())
{
httpClient.Timeout = partTimeout;
using (var output = new FileStream(targetPath, System.IO.FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
// Determine the starting position of the next part to be downloaded.
var pos = manifest.Parts
.Where(part => part.Number < nextPartNumber)
.Sum(part => part.Size);
// Download the remaining parts.
foreach (var part in manifest.Parts
.Where(part => part.Number >= nextPartNumber)
.OrderBy(part => part.Number))
{
await retry.InvokeAsync(
async() =>
{
output.Position = pos;
var response = await httpClient.GetAsync(part.Uri, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
response.EnsureSuccessStatusCode();
using (var contentStream = await response.Content.ReadAsStreamAsync())
{
await contentStream.CopyToAsync(output, cancellationToken);
}
});
// Ensure that the downloaded part size matches the specification.
if (output.Position - pos != part.Size)
{
throw new IOException($"[{manifest.Name}]: Part [{part.Number}] actual size [{output.Position - pos}] does not match the expected size [{part.Size}].");
}
// Ensure that the downloaded part MD5 matches the specification.
using (var subStream = new SubStream(output, pos, part.Size))
{
var actualMd5 = CryptoHelper.ComputeMD5String(subStream);
if (actualMd5 != part.Md5)
{
throw new IOException($"[{manifest.Name}]: Part [{part.Number}] actual MD5 [{actualMd5}] does not match the expected MD5 [{part.Md5}].");
}
}
pos += part.Size;
if (progressAction != null && !progressAction.Invoke(DownloadProgressType.Download, (int)(100.0 * ((double)part.Number / (double)manifest.Parts.Count))))
{
return(targetPath);
}
}
if (output.Length != manifest.Size)
{
throw new IOException($"[{manifest.Name}]: Expected size [{manifest.Size}] got [{output.Length}].");
}
}
progressAction?.Invoke(DownloadProgressType.Download, 100);
File.WriteAllText(targetMd5Path, manifest.Md5, Encoding.ASCII);
return(targetPath);
}
}
catch (IOException)
{
NeonHelper.DeleteFile(targetPath);
NeonHelper.DeleteFile(targetMd5Path);
throw;
}
}
/// <summary>
/// <para>
/// Uploads a file in multiple parts from the local workstation to S3, returning the
/// <see cref="DownloadManifest"/> details. required by <see cref="DeploymentHelper.DownloadMultiPart(DownloadManifest, string, DownloadProgressDelegate, IRetryPolicy, TimeSpan)"/>
/// and <see cref="DeploymentHelper.DownloadMultiPartAsync(DownloadManifest, string, DownloadProgressDelegate, TimeSpan, IRetryPolicy, System.Threading.CancellationToken)"/>
/// to actually download the entire file. The URI to the uploaded <see cref="DownloadManifest"/> details is also returned.
/// </para>
/// <para>
/// See the remarks for details about how this works.
/// </para>
/// </summary>
/// <param name="sourcePath">Path to the file being uploaded.</param>
/// <param name="targetFolderUri">
/// <para>
/// The target S3 URI structured like <b>https://s3.REGION.amazonaws.com/BUCKET/...</b>
/// URI referencing an S3 bucket and the optional folder where the file's download information
/// and parts will be uploaded.
/// </para>
/// <note>
/// The <b>s3://</b> URI scheme is not supported.
/// </note>
/// </param>
/// <param name="version">Optionally specifies the download file version.</param>
/// <param name="name">Optionally overrides the download file name specified by <paramref name="sourcePath"/> to initialize <see cref="DownloadManifest.Name"/>.</param>
/// <param name="filename">Optionally overrides the download file name specified by <paramref name="sourcePath"/> to initialize <see cref="DownloadManifest.Filename"/>.</param>
/// <param name="noMd5File">
/// This method creates a file named [<paramref name="sourcePath"/>.md5] with the MD5 hash for the entire
/// uploaded file by default. You may override this behavior by passing <paramref name="noMd5File"/>=<c>true</c>.
/// </param>
/// <param name="maxPartSize">Optionally overrides the maximum part size (defailts to 100 MiB).</param>
/// <param name="publicReadAccess">Optionally grant the upload public read access.</param>
/// <param name="progressAction">Optional action called as the file is uploaded, passing the <c>long</c> percent complete.</param>
/// <returns>The <see cref="DownloadManifest"/> information.</returns>
/// <returns>The <see cref="DownloadManifest"/> information as well as the URI to the uploaded manifest.</returns>
/// <remarks>
/// <para>
/// This method works by splitting the <paramref name="sourcePath"/> file into parts no larger than
/// <paramref name="maxPartSize"/> bytes each and the uploading these parts to the specified bucket
/// and path along with a file holding <see cref="DownloadManifest"/> information describing the download
/// and its constituent parts. This information includes details about the download including the
/// overall MD5 and size as well records describing each part including their URIs, sizes and MD5.
/// </para>
/// <para>
/// The <see cref="DownloadManifest"/> details returned include all of the information required by
/// <see cref="DeploymentHelper.DownloadMultiPart(DownloadManifest, string, DownloadProgressDelegate, IRetryPolicy, TimeSpan)"/> and
/// <see cref="DeploymentHelper.DownloadMultiPartAsync(DownloadManifest, string, DownloadProgressDelegate, TimeSpan, IRetryPolicy, System.Threading.CancellationToken)"/>
/// to actually download the entire file and the URI returned references these msame details as
/// uploaded to S3.
/// </para>
/// <para>
/// You'll need to pass <paramref name="sourcePath"/> as the path to the file being uploaded
/// and <paramref name="targetFolderUri"/> as the S3 location where the download information and the
/// file parts will be uploaded. <paramref name="targetFolderUri"/> may use with the <b>https://</b>
/// or <b>s3://</b> URI scheme.
/// </para>
/// <para>
/// By default the uploaded file and parts names will be based on the filename part of <paramref name="sourcePath"/>,
/// but this can be overridden via <paramref name="filename"/>. The <see cref="DownloadManifest"/> information for the
/// file will be uploaded as <b>FILENAME.manifest</b> and the parts will be written to a subfolder named
/// <b>FILENAME.parts</b>. For example, uploading a large file named <b>myfile.json</b> to <b>https://s3.uswest.amazonaws.com/mybucket</b>
/// will result S3 file layout like:
/// </para>
/// <code>
/// https://s3.uswest.amazonaws.com/mybucket
/// myfile.json.manifest
/// myfile.json.parts/
/// part-0000
/// part-0001
/// part-0002
/// ...
/// </code>
/// <para>
/// The URI returned in this case will be <b>https://s3.uswest.amazonaws.com/mybucket/myfile.json.manifest</b>.
/// </para>
/// </remarks>
public static (DownloadManifest manifest, string manifestUri) S3UploadMultiPart(
string sourcePath,
string targetFolderUri,
string version = null,
string name = null,
string filename = null,
bool noMd5File = false,
long maxPartSize = (long)(100 * ByteUnits.MebiBytes),
bool publicReadAccess = false,
Action <long> progressAction = null)
{
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(sourcePath), nameof(sourcePath));
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(targetFolderUri), nameof(targetFolderUri));
if (!Uri.TryCreate(targetFolderUri, UriKind.Absolute, out var uriCheck))
{
Covenant.Assert(false, $"Invalid [{nameof(targetFolderUri)}={targetFolderUri}].");
}
Covenant.Assert(uriCheck.Scheme == "https", $"Invalid scheme in [{nameof(targetFolderUri)}={targetFolderUri}]. Only [https://] is supported.");
name = name ?? Path.GetFileName(sourcePath);
filename = filename ?? Path.GetFileName(sourcePath);
// Determine the base URI for the download manifest and parts on S3.
var baseUri = targetFolderUri;
if (!baseUri.EndsWith('/'))
{
baseUri += '/';
}
baseUri += filename;
// Remove any existing manifest object as well as any parts.
var manifestUri = $"{baseUri}.manifest";
var partsFolder = $"{baseUri}.parts/";
if (progressAction != null)
{
progressAction(0L);
}
S3Remove(manifestUri);
S3Remove(partsFolder, recursive: true, include: $"{partsFolder}*");
// We're going to upload the parts first, while initializing the download manifest as we go.
var manifest = new DownloadManifest()
{
Name = name, Version = version, Filename = filename
};
using (var input = File.OpenRead(sourcePath))
{
var partCount = NeonHelper.PartitionCount(input.Length, maxPartSize);
var partNumber = 0;
var partStart = 0L;
var cbRemaining = input.Length;
manifest.Md5 = CryptoHelper.ComputeMD5String(input);
input.Position = 0;
while (cbRemaining > 0)
{
var partSize = Math.Min(cbRemaining, maxPartSize);
var part = new DownloadPart()
{
Uri = $"{partsFolder}part-{partNumber:000#}",
Number = partNumber,
Size = partSize,
};
// We're going to use a substream to compute the MD5 hash for the part
// as well as to actually upload the part to S3.
using (var partStream = new SubStream(input, partStart, partSize))
{
part.Md5 = CryptoHelper.ComputeMD5String(partStream);
partStream.Position = 0;
S3Upload(partStream, part.Uri, publicReadAccess: publicReadAccess);
}
manifest.Parts.Add(part);
// Loop to handle the next part (if any).
partNumber++;
partStart += partSize;
cbRemaining -= partSize;
if (progressAction != null)
{
progressAction(Math.Min(99L, (long)(100.0 * (double)partNumber / (double)partCount)));
}
}
manifest.Size = manifest.Parts.Sum(part => part.Size);
}
// Upload the manifest.
S3UploadText(NeonHelper.JsonSerialize(manifest, Formatting.Indented), manifestUri, metadata: $"Content-Type={DeploymentHelper.DownloadManifestContentType}", publicReadAccess: publicReadAccess);
// Write the MD5 file unless disabled.
if (!noMd5File)
{
File.WriteAllText($"{sourcePath}.md5", manifest.Md5);
}
if (progressAction != null)
{
progressAction(100L);
}
return(manifest : manifest, manifestUri : manifestUri);
}