Exemple #1
0
 /// <summary>
 /// Synchronously downloads and assembles a multi-part file as specified by a <see cref="Neon.Deployment.DownloadManifest"/>.
 /// </summary>
 /// <param name="download">The download details.</param>
 /// <param name="targetPath">The target file path.</param>
 /// <param name="progressAction">Optionally specifies an action to be called with the the percentage downloaded.</param>
 /// <param name="retry">Optionally specifies the retry policy.  This defaults to a reasonable policy.</param>
 /// <param name="partTimeout">Optionally specifies the HTTP download timeout for each part (defaults to 10 minutes).</param>
 /// <exception cref="IOException">Thrown when the download is corrupt.</exception>
 /// <exception cref="SocketException">Thrown for network errors.</exception>
 /// <exception cref="HttpException">Thrown for HTTP network errors.</exception>
 /// <exception cref="OperationCanceledException">Thrown when the operation was cancelled.</exception>
 public static void DownloadMultiPart(
     DownloadManifest download,
     string targetPath,
     DownloadProgressDelegate progressAction = null,
     IRetryPolicy retry   = null,
     TimeSpan partTimeout = default)
 {
     DownloadMultiPartAsync(download, targetPath, progressAction, partTimeout, retry).WaitWithoutAggregate();
 }
Exemple #2
0
        /// <summary>
        /// Uploads a multi-part download to a release and then publishes the release.
        /// </summary>
        /// <param name="repo">Identifies the target repository.</param>
        /// <param name="release">The target release.</param>
        /// <param name="sourcePath">Path to the file being uploaded.</param>
        /// <param name="version">The download version.</param>
        /// <param name="name">Optionally overrides the download file name specified by <paramref name="sourcePath"/> to initialize <see cref="DownloadManifest.Name"/>.</param>
        /// <param name="filename">Optionally overrides the download file name specified by <paramref name="sourcePath"/> to initialize <see cref="DownloadManifest.Filename"/>.</param>
        /// <param name="noMd5File">
        /// This method creates a file named [<paramref name="sourcePath"/>.md5] with the MD5 hash for the entire
        /// uploaded file by default.  You may override this behavior by passing <paramref name="noMd5File"/>=<c>true</c>.
        /// </param>
        /// <param name="maxPartSize">Optionally overrides the maximum part size (defailts to 100 MiB).</param>d
        /// <returns>The <see cref="DownloadManifest"/>.</returns>
        /// <remarks>
        /// <para>
        /// The release passed must be unpublished and you may upload other assets before calling this.
        /// </para>
        /// <note>
        /// Take care that any assets already published have names that won't conflict with the asset
        /// part names, which will be formatted like: <b>part-##</b>
        /// </note>
        /// </remarks>
        public DownloadManifest UploadMultipartAsset(
            string repo,
            Release release,
            string sourcePath,
            string version,
            string name      = null,
            string filename  = null,
            bool noMd5File   = false,
            long maxPartSize = (long)(100 * ByteUnits.MebiBytes))
        {
            Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(repo), nameof(repo));
            Covenant.Requires <ArgumentNullException>(release != null, nameof(release));
            Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(sourcePath), nameof(sourcePath));
            Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(version), nameof(version));

            name     = name ?? Path.GetFileName(sourcePath);
            filename = filename ?? Path.GetFileName(sourcePath);

            using (var input = File.OpenRead(sourcePath))
            {
                if (input.Length == 0)
                {
                    throw new IOException($"Asset at [{sourcePath}] cannot be empty.");
                }

                var assetPartMap = new List <Tuple <ReleaseAsset, DownloadPart> >();
                var manifest     = new DownloadManifest()
                {
                    Name = name, Version = version, Filename = filename
                };
                var partCount   = NeonHelper.PartitionCount(input.Length, maxPartSize);
                var partNumber  = 0;
                var partStart   = 0L;
                var cbRemaining = input.Length;

                manifest.Md5   = CryptoHelper.ComputeMD5String(input);
                input.Position = 0;

                while (cbRemaining > 0)
                {
                    var partSize = Math.Min(cbRemaining, maxPartSize);
                    var part     = new DownloadPart()
                    {
                        Number = partNumber,
                        Size   = partSize,
                    };

                    // We're going to use a substream to compute the MD5 hash for the part
                    // as well as to actually upload the part to the GitHub release.

                    using (var partStream = new SubStream(input, partStart, partSize))
                    {
                        part.Md5            = CryptoHelper.ComputeMD5String(partStream);
                        partStream.Position = 0;

                        var asset = GitHub.Releases.UploadAsset(repo, release, partStream, $"part-{partNumber:0#}");

                        assetPartMap.Add(new Tuple <ReleaseAsset, DownloadPart>(asset, part));
                    }

                    manifest.Parts.Add(part);

                    // Loop to handle the next part (if any).

                    partNumber++;
                    partStart   += partSize;
                    cbRemaining -= partSize;
                }

                manifest.Size = manifest.Parts.Sum(part => part.Size);

                // Publish the release.

                var releaseUpdate = release.ToUpdate();

                releaseUpdate.Draft = false;

                release = GitHub.Releases.Update(repo, release, releaseUpdate);

                // Now that the release has been published, we can go back and fill in
                // the asset URIs for each of the download parts.

                foreach (var item in assetPartMap)
                {
                    item.Item2.Uri = GitHub.Releases.GetAssetUri(release, item.Item1);
                }

                // Write the MD5 file unless disabled.

                if (!noMd5File)
                {
                    File.WriteAllText($"{sourcePath}.md5", manifest.Md5);
                }

                return(manifest);
            }
        }
Exemple #3
0
        /// <summary>
        /// Asynchronously downloads and assembles a multi-part file  as specified by a <see cref="Neon.Deployment.DownloadManifest"/>.
        /// </summary>
        /// <param name="manifest">The download details.</param>
        /// <param name="targetPath">The target file path.</param>
        /// <param name="progressAction">Optionally specifies an action to be called with the the percentage downloaded.</param>
        /// <param name="partTimeout">Optionally specifies the HTTP download timeout for each part (defaults to 10 minutes).</param>
        /// <param name="retry">Optionally specifies the retry policy.  This defaults to a reasonable policy.</param>
        /// <param name="cancellationToken">Optionally specifies the operation cancellation token.</param>
        /// <returns>The path to the downloaded file.</returns>
        /// <exception cref="IOException">Thrown when the download is corrupt.</exception>
        /// <exception cref="SocketException">Thrown for network errors.</exception>
        /// <exception cref="HttpException">Thrown for HTTP network errors.</exception>
        /// <exception cref="OperationCanceledException">Thrown when the operation was cancelled.</exception>
        /// <remarks>
        /// <para>
        /// This method downloads the file specified by <paramref name="manifest"/> to the folder specified, creating
        /// the folder first when required.  The file will be downloaded in parts, where each part will be validated
        /// by comparing the part's MD5 hash (when present) with the computed value.  The output file will be named
        /// <see cref="DownloadManifest.Name"/> and the overall MD5 hash will also be saved using the same file name but
        /// <b>adding</b> the <b>.md5</b> extension.
        /// </para>
        /// <para>
        /// This method will continue downloading a partially downloaded file.  This works by validating the already
        /// downloaded parts against their MD5 hashes and then continuing part downloads after the last valid part.
        /// Nothing will be downloaded when the existing file is fully formed.
        /// </para>
        /// <note>
        /// The target files (output and MD5) will be deleted when download appears to be corrupt.
        /// </note>
        /// </remarks>
        public static async Task <string> DownloadMultiPartAsync(
            DownloadManifest manifest,
            string targetPath,
            DownloadProgressDelegate progressAction = null,
            TimeSpan partTimeout = default,
            IRetryPolicy retry   = null,
            CancellationToken cancellationToken = default)
        {
            await SyncContext.Clear;

            Covenant.Requires <ArgumentNullException>(manifest != null, nameof(manifest));
            Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(targetPath), nameof(targetPath));

            retry = retry ?? new ExponentialRetryPolicy(TransientDetector.NetworkOrHttp, maxAttempts: 5);

            if (partTimeout <= TimeSpan.Zero)
            {
                partTimeout = TimeSpan.FromMinutes(10);
            }

            var targetFolder = Path.GetDirectoryName(targetPath);

            Directory.CreateDirectory(targetFolder);

            var targetMd5Path  = Path.Combine(Path.GetDirectoryName(targetPath), Path.GetFileName(targetPath) + ".md5");
            var nextPartNumber = 0;

            // If the target file already exists along with its MD5 hash file:
            //
            //      1. Compare the manifest MD5 with the local MD5 hash file and
            //         quickly continue with the download when these don't match.
            //
            //      2. When the MD5 files match, compute the MD5 of the downloaded
            //         file and compare that with the manifest and continue with
            //         the download when these don't match.

            if (File.Exists(targetPath) && File.Exists(targetMd5Path) && File.ReadAllText(targetMd5Path).Trim() == manifest.Md5)
            {
                if (File.ReadAllText(targetMd5Path).Trim() == manifest.Md5)
                {
                    using (var downloadStream = File.OpenRead(targetPath))
                    {
                        if (CryptoHelper.ComputeMD5String(downloadStream) == manifest.Md5)
                        {
                            return(targetPath);
                        }
                    }
                }
            }

            NeonHelper.DeleteFile(targetMd5Path);   // We'll recompute this below

            // Validate the parts of any existing target file to determine where
            // to start downloading missing parts.

            if (File.Exists(targetPath))
            {
                using (var output = new FileStream(targetPath, System.IO.FileMode.Open, FileAccess.ReadWrite))
                {
                    var pos = 0L;

                    foreach (var part in manifest.Parts.OrderBy(part => part.Number))
                    {
                        progressAction?.Invoke(DownloadProgressType.Check, (int)((double)pos / (double)manifest.Size * 100.0));

                        // Handle a partially downloaded part.  We're going to truncate the file to
                        // remove the partial part and then break to start re-downloading the part.

                        if (output.Length < pos + part.Size)
                        {
                            output.SetLength(pos);

                            nextPartNumber = part.Number;
                            break;
                        }

                        // Validate the part MD5.  We're going to truncate the file to remove the
                        // partial part and then break to start re-downloading the part.

                        using (var partStream = new SubStream(output, pos, part.Size))
                        {
                            if (CryptoHelper.ComputeMD5String(partStream) != part.Md5)
                            {
                                output.SetLength(pos);

                                nextPartNumber = part.Number;
                                break;
                            }
                        }

                        pos           += part.Size;
                        nextPartNumber = part.Number + 1;
                    }
                }
            }

            // Download any remaining parts.

            if (progressAction != null && !progressAction.Invoke(DownloadProgressType.Download, 0))
            {
                return(targetPath);
            }

            if (nextPartNumber == manifest.Parts.Count)
            {
                progressAction?.Invoke(DownloadProgressType.Download, 100);
                return(targetPath);
            }

            try
            {
                using (var httpClient = new HttpClient())
                {
                    httpClient.Timeout = partTimeout;

                    using (var output = new FileStream(targetPath, System.IO.FileMode.OpenOrCreate, FileAccess.ReadWrite))
                    {
                        // Determine the starting position of the next part to be downloaded.

                        var pos = manifest.Parts
                                  .Where(part => part.Number < nextPartNumber)
                                  .Sum(part => part.Size);

                        // Download the remaining parts.

                        foreach (var part in manifest.Parts
                                 .Where(part => part.Number >= nextPartNumber)
                                 .OrderBy(part => part.Number))
                        {
                            await retry.InvokeAsync(
                                async() =>
                            {
                                output.Position = pos;

                                var response = await httpClient.GetAsync(part.Uri, HttpCompletionOption.ResponseHeadersRead, cancellationToken);

                                response.EnsureSuccessStatusCode();

                                using (var contentStream = await response.Content.ReadAsStreamAsync())
                                {
                                    await contentStream.CopyToAsync(output, cancellationToken);
                                }
                            });

                            // Ensure that the downloaded part size matches the specification.

                            if (output.Position - pos != part.Size)
                            {
                                throw new IOException($"[{manifest.Name}]: Part [{part.Number}] actual size [{output.Position - pos}] does not match the expected size [{part.Size}].");
                            }

                            // Ensure that the downloaded part MD5 matches the specification.

                            using (var subStream = new SubStream(output, pos, part.Size))
                            {
                                var actualMd5 = CryptoHelper.ComputeMD5String(subStream);

                                if (actualMd5 != part.Md5)
                                {
                                    throw new IOException($"[{manifest.Name}]: Part [{part.Number}] actual MD5 [{actualMd5}] does not match the expected MD5 [{part.Md5}].");
                                }
                            }

                            pos += part.Size;

                            if (progressAction != null && !progressAction.Invoke(DownloadProgressType.Download, (int)(100.0 * ((double)part.Number / (double)manifest.Parts.Count))))
                            {
                                return(targetPath);
                            }
                        }

                        if (output.Length != manifest.Size)
                        {
                            throw new IOException($"[{manifest.Name}]: Expected size [{manifest.Size}] got [{output.Length}].");
                        }
                    }

                    progressAction?.Invoke(DownloadProgressType.Download, 100);
                    File.WriteAllText(targetMd5Path, manifest.Md5, Encoding.ASCII);

                    return(targetPath);
                }
            }
            catch (IOException)
            {
                NeonHelper.DeleteFile(targetPath);
                NeonHelper.DeleteFile(targetMd5Path);

                throw;
            }
        }
Exemple #4
0
        /// <summary>
        /// <para>
        /// Uploads a file in multiple parts from the local workstation to S3, returning the
        /// <see cref="DownloadManifest"/> details. required by <see cref="DeploymentHelper.DownloadMultiPart(DownloadManifest, string, DownloadProgressDelegate, IRetryPolicy, TimeSpan)"/>
        /// and <see cref="DeploymentHelper.DownloadMultiPartAsync(DownloadManifest, string, DownloadProgressDelegate, TimeSpan, IRetryPolicy, System.Threading.CancellationToken)"/>
        /// to actually download the entire file.  The URI to the uploaded <see cref="DownloadManifest"/> details is also returned.
        /// </para>
        /// <para>
        /// See the remarks for details about how this works.
        /// </para>
        /// </summary>
        /// <param name="sourcePath">Path to the file being uploaded.</param>
        /// <param name="targetFolderUri">
        /// <para>
        /// The target S3 URI structured like <b>https://s3.REGION.amazonaws.com/BUCKET/...</b>
        /// URI referencing an S3 bucket and the optional folder where the file's download information
        /// and parts will be uploaded.
        /// </para>
        /// <note>
        /// The <b>s3://</b> URI scheme is not supported.
        /// </note>
        /// </param>
        /// <param name="version">Optionally specifies the download file version.</param>
        /// <param name="name">Optionally overrides the download file name specified by <paramref name="sourcePath"/> to initialize <see cref="DownloadManifest.Name"/>.</param>
        /// <param name="filename">Optionally overrides the download file name specified by <paramref name="sourcePath"/> to initialize <see cref="DownloadManifest.Filename"/>.</param>
        /// <param name="noMd5File">
        /// This method creates a file named [<paramref name="sourcePath"/>.md5] with the MD5 hash for the entire
        /// uploaded file by default.  You may override this behavior by passing <paramref name="noMd5File"/>=<c>true</c>.
        /// </param>
        /// <param name="maxPartSize">Optionally overrides the maximum part size (defailts to 100 MiB).</param>
        /// <param name="publicReadAccess">Optionally grant the upload public read access.</param>
        /// <param name="progressAction">Optional action called as the file is uploaded, passing the <c>long</c> percent complete.</param>
        /// <returns>The <see cref="DownloadManifest"/> information.</returns>
        /// <returns>The <see cref="DownloadManifest"/> information as well as the URI to the uploaded manifest.</returns>
        /// <remarks>
        /// <para>
        /// This method works by splitting the <paramref name="sourcePath"/> file into parts no larger than
        /// <paramref name="maxPartSize"/> bytes each and the uploading these parts to the specified bucket
        /// and path along with a file holding <see cref="DownloadManifest"/> information describing the download
        /// and its constituent parts.  This information includes details about the download including the
        /// overall MD5 and size as well records describing each part including their URIs, sizes and MD5.
        /// </para>
        /// <para>
        /// The <see cref="DownloadManifest"/> details returned include all of the information required by
        /// <see cref="DeploymentHelper.DownloadMultiPart(DownloadManifest, string, DownloadProgressDelegate, IRetryPolicy, TimeSpan)"/> and
        /// <see cref="DeploymentHelper.DownloadMultiPartAsync(DownloadManifest, string, DownloadProgressDelegate, TimeSpan, IRetryPolicy, System.Threading.CancellationToken)"/>
        /// to actually download the entire file and the URI returned references these msame details as
        /// uploaded to S3.
        /// </para>
        /// <para>
        /// You'll need to pass <paramref name="sourcePath"/> as the path to the file being uploaded
        /// and <paramref name="targetFolderUri"/> as the S3 location where the download information and the
        /// file parts will be uploaded.  <paramref name="targetFolderUri"/> may use with the <b>https://</b>
        /// or <b>s3://</b> URI scheme.
        /// </para>
        /// <para>
        /// By default the uploaded file and parts names will be based on the filename part of <paramref name="sourcePath"/>,
        /// but this can be overridden via <paramref name="filename"/>.  The <see cref="DownloadManifest"/> information for the
        /// file will be uploaded as <b>FILENAME.manifest</b> and the parts will be written to a subfolder named
        /// <b>FILENAME.parts</b>.  For example, uploading a large file named <b>myfile.json</b> to <b>https://s3.uswest.amazonaws.com/mybucket</b>
        /// will result S3 file layout like:
        /// </para>
        /// <code>
        /// https://s3.uswest.amazonaws.com/mybucket
        ///     myfile.json.manifest
        ///     myfile.json.parts/
        ///         part-0000
        ///         part-0001
        ///         part-0002
        ///         ...
        /// </code>
        /// <para>
        /// The URI returned in this case will be <b>https://s3.uswest.amazonaws.com/mybucket/myfile.json.manifest</b>.
        /// </para>
        /// </remarks>
        public static (DownloadManifest manifest, string manifestUri) S3UploadMultiPart(
            string sourcePath,
            string targetFolderUri,
            string version               = null,
            string name                  = null,
            string filename              = null,
            bool noMd5File               = false,
            long maxPartSize             = (long)(100 * ByteUnits.MebiBytes),
            bool publicReadAccess        = false,
            Action <long> progressAction = null)
        {
            Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(sourcePath), nameof(sourcePath));
            Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(targetFolderUri), nameof(targetFolderUri));

            if (!Uri.TryCreate(targetFolderUri, UriKind.Absolute, out var uriCheck))
            {
                Covenant.Assert(false, $"Invalid [{nameof(targetFolderUri)}={targetFolderUri}].");
            }

            Covenant.Assert(uriCheck.Scheme == "https", $"Invalid scheme in [{nameof(targetFolderUri)}={targetFolderUri}].  Only [https://] is supported.");

            name     = name ?? Path.GetFileName(sourcePath);
            filename = filename ?? Path.GetFileName(sourcePath);

            // Determine the base URI for the download manifest and parts on S3.

            var baseUri = targetFolderUri;

            if (!baseUri.EndsWith('/'))
            {
                baseUri += '/';
            }

            baseUri += filename;

            // Remove any existing manifest object as well as any parts.

            var manifestUri = $"{baseUri}.manifest";
            var partsFolder = $"{baseUri}.parts/";

            if (progressAction != null)
            {
                progressAction(0L);
            }

            S3Remove(manifestUri);
            S3Remove(partsFolder, recursive: true, include: $"{partsFolder}*");

            // We're going to upload the parts first, while initializing the download manifest as we go.

            var manifest = new DownloadManifest()
            {
                Name = name, Version = version, Filename = filename
            };

            using (var input = File.OpenRead(sourcePath))
            {
                var partCount   = NeonHelper.PartitionCount(input.Length, maxPartSize);
                var partNumber  = 0;
                var partStart   = 0L;
                var cbRemaining = input.Length;

                manifest.Md5   = CryptoHelper.ComputeMD5String(input);
                input.Position = 0;

                while (cbRemaining > 0)
                {
                    var partSize = Math.Min(cbRemaining, maxPartSize);
                    var part     = new DownloadPart()
                    {
                        Uri    = $"{partsFolder}part-{partNumber:000#}",
                        Number = partNumber,
                        Size   = partSize,
                    };

                    // We're going to use a substream to compute the MD5 hash for the part
                    // as well as to actually upload the part to S3.

                    using (var partStream = new SubStream(input, partStart, partSize))
                    {
                        part.Md5            = CryptoHelper.ComputeMD5String(partStream);
                        partStream.Position = 0;

                        S3Upload(partStream, part.Uri, publicReadAccess: publicReadAccess);
                    }

                    manifest.Parts.Add(part);

                    // Loop to handle the next part (if any).

                    partNumber++;
                    partStart   += partSize;
                    cbRemaining -= partSize;

                    if (progressAction != null)
                    {
                        progressAction(Math.Min(99L, (long)(100.0 * (double)partNumber / (double)partCount)));
                    }
                }

                manifest.Size = manifest.Parts.Sum(part => part.Size);
            }

            // Upload the manifest.

            S3UploadText(NeonHelper.JsonSerialize(manifest, Formatting.Indented), manifestUri, metadata: $"Content-Type={DeploymentHelper.DownloadManifestContentType}", publicReadAccess: publicReadAccess);

            // Write the MD5 file unless disabled.

            if (!noMd5File)
            {
                File.WriteAllText($"{sourcePath}.md5", manifest.Md5);
            }

            if (progressAction != null)
            {
                progressAction(100L);
            }

            return(manifest : manifest, manifestUri : manifestUri);
        }