/// <summary>
/// Asynchronously downloads and assembles a multi-part file as specified by a <see cref="Neon.Deployment.DownloadManifest"/>.
/// </summary>
/// <param name="uri">The URI for the source URI holding the <see cref="DownloadManifest"/> details as JSON.</param>
/// <param name="targetPath">The target file path.</param>
/// <param name="progressAction">Optionally specifies an action to be called with the the percentage downloaded.</param>
/// <param name="partTimeout">Optionally specifies the HTTP download timeout for each part (defaults to 10 minutes).</param>
/// <param name="retry">Optionally specifies the retry policy. This defaults to a reasonable policy.</param>
/// <param name="cancellationToken">Optionally specifies the operation cancellation token.</param>
/// <returns>The path to the downloaded file.</returns>
/// <exception cref="IOException">Thrown when the download is corrupt.</exception>
/// <exception cref="SocketException">Thrown for network errors.</exception>
/// <exception cref="HttpException">Thrown for HTTP network errors.</exception>
/// <exception cref="OperationCanceledException">Thrown when the operation was cancelled.</exception>
/// <exception cref="FormatException">Thrown when the object retrieved from <paramref name="uri"/> doesn't have the <see cref="DeploymentHelper.DownloadManifestContentType"/> content type.</exception>
/// <remarks>
/// <para>
/// This method downloads the file specified by <paramref name="uri"/> to the folder specified, creating
/// the folder first when required. The file will be downloaded in parts, where each part will be validated
/// by comparing the part's MD5 hash (when present) with the computed value. The output file will be named
/// <see cref="DownloadManifest.Name"/> and the overall MD5 hash will also be saved using the same file name but
/// <b>adding</b> the <b>.md5</b> extension.
/// </para>
/// <para>
/// This method will continue downloading a partially downloaded file. This works by validating the already
/// downloaded parts against their MD5 hashes and then continuing part downloads after the last valid part.
/// Nothing will be downloaded when the existing file is fully formed.
/// </para>
/// <note>
/// The target files (output and MD5) will be deleted when download appears to be corrupt.
/// </note>
/// </remarks>
public static async Task <string> DownloadMultiPartAsync(
string uri,
string targetPath,
DownloadProgressDelegate progressAction = null,
TimeSpan partTimeout = default,
IRetryPolicy retry = null,
CancellationToken cancellationToken = default)
{
await SyncContext.Clear;
DownloadManifest manifest;
using (var httpClient = new HttpClient())
{
var response = await httpClient.GetSafeAsync(uri);
if (!response.Content.Headers.ContentType.MediaType.Equals(DeploymentHelper.DownloadManifestContentType))
{
throw new FormatException($"The content type for [{uri}] is [{response.Content.Headers.ContentType.MediaType}]. [{DeploymentHelper.DownloadManifestContentType}] was expected.");
}
manifest = NeonHelper.JsonDeserialize <DownloadManifest>(await response.Content.ReadAsStringAsync());
}
return(await DownloadMultiPartAsync(manifest, targetPath, progressAction, partTimeout, retry, cancellationToken));
}
public static void Download(string uri, string localPath, DownloadProgressDelegate progressDelegate)
{
long remoteSize;
string fullLocalPath; // Full local path including file name if only directory was provided.
try
{
/// Get the name of the remote file.
Uri remoteUri = new Uri(uri);
string fileName = Path.GetFileName(remoteUri.LocalPath);
if (Path.GetFileName(localPath).Length == 0)
{
fullLocalPath = Path.Combine(localPath, fileName);
}
else
{
fullLocalPath = localPath;
}
/// Have to get size of remote object through the webrequest as not available on remote files,
/// although it does work on local files.
using (WebResponse response = WebRequest.Create(uri).GetResponse())
using (Stream stream = response.GetResponseStream())
remoteSize = response.ContentLength;
}
catch (Exception ex)
{
throw new ApplicationException(string.Format("Error connecting to URI (Exception={0})", ex.Message), ex);
}
int bytesRead = 0, bytesReadTotal = 0;
try
{
using (WebClient client = new WebClient())
using (Stream streamRemote = client.OpenRead(new Uri(uri)))
using (Stream streamLocal = new FileStream(fullLocalPath, FileMode.Create, FileAccess.Write, FileShare.None))
{
byte[] byteBuffer = new byte[1024 * 1024 * 2]; // 2 meg buffer although in testing only got to 10k max usage.
int perc = 0;
while ((bytesRead = streamRemote.Read(byteBuffer, 0, byteBuffer.Length)) > 0)
{
bytesReadTotal += bytesRead;
streamLocal.Write(byteBuffer, 0, bytesRead);
int newPerc = (int)((double)bytesReadTotal / (double)remoteSize * 100);
if (newPerc > perc)
{
perc = newPerc;
progressDelegate?.Invoke(perc, bytesReadTotal, (int)remoteSize);
}
}
}
}
catch (Exception ex)
{
throw new ApplicationException(string.Format("Error downloading file (Exception={0})", ex.Message), ex);
}
}
/// <summary>
/// Synchronously downloads and assembles a multi-part file as specified by a <see cref="Neon.Deployment.DownloadManifest"/>.
/// </summary>
/// <param name="download">The download details.</param>
/// <param name="targetPath">The target file path.</param>
/// <param name="progressAction">Optionally specifies an action to be called with the the percentage downloaded.</param>
/// <param name="retry">Optionally specifies the retry policy. This defaults to a reasonable policy.</param>
/// <param name="partTimeout">Optionally specifies the HTTP download timeout for each part (defaults to 10 minutes).</param>
/// <exception cref="IOException">Thrown when the download is corrupt.</exception>
/// <exception cref="SocketException">Thrown for network errors.</exception>
/// <exception cref="HttpException">Thrown for HTTP network errors.</exception>
/// <exception cref="OperationCanceledException">Thrown when the operation was cancelled.</exception>
public static void DownloadMultiPart(
DownloadManifest download,
string targetPath,
DownloadProgressDelegate progressAction = null,
IRetryPolicy retry = null,
TimeSpan partTimeout = default)
{
DownloadMultiPartAsync(download, targetPath, progressAction, partTimeout, retry).WaitWithoutAggregate();
}
private void bwUpdateProgram_DoWork(object sender, DoWorkEventArgs e)
{
//Create workdir
string workDir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
try
{
Directory.CreateDirectory(workDir);
}
catch (Exception ex)
{
e.Result = new object[] { BwResult.Error, ex };
return;
}
//Download the updatefile
DownloadProgressDelegate progressUpdate = new DownloadProgressDelegate(downloadFileProgressChanged);
bool result;
try
{
string url;
if (CurrentUpdateInfo.CompleteDownloadURL != string.Empty)
{
url = CurrentUpdateInfo.CompleteDownloadURL;
}
else
{
url = CurrentUpdateInfo.DownloadURL + CurrentUpdateInfo.Version + ".exe";
}
result = Download(url, Path.Combine(workDir, CurrentUpdateInfo.Version + ".exe"), progressUpdate);
}
catch (Exception ex)
{
e.Result = new object[] { workDir, BwResult.Error, ex };
return;
}
if (result)
{
//Download succeeded, notify the user and start the setup
e.Result = new object[] { workDir, BwResult.Success, Path.Combine(workDir, CurrentUpdateInfo.Version + ".exe") };
}
else
{
bwUpdateProgram.ReportProgress(99);
//Something went wrong OR the user cancelled
e.Result = new object[] { workDir, BwResult.Cancelled };
}
}
public async Task <string> AddFileAsync(string filePath, bool pin, DownloadProgressDelegate progressCallBack)
{
using (var fileStream = new FileStreamWithProgress(filePath, FileMode.Open, FileAccess.Read))
{
fileStream.SetTotalLength(fileStream.Length);
fileStream.SetCaption("upload progress");
fileStream.ProgressCallBack += progressCallBack;
var task = await Ipfs.FileSystem.AddAsync(fileStream, Path.GetFileName(filePath), new AddFileOptions { Pin = pin, Wrap = true });
fileStream.ProgressCallBack = null;
return(task.Id.Hash.ToString());
}
}
public static void Download(string uri, string localPath, DownloadProgressDelegate progressDelegate)
{
long remoteSize;
string fullLocalPath; // Full local path including file name if only directory was provided.
//log.InfoFormat("Attempting to download file (Uri={0}, LocalPath={1})", uri, localPath);
//try
//{
/// Get the name of the remote file.
Uri remoteUri = new Uri(uri);
string fileName = Path.GetFileName(remoteUri.LocalPath);
Console.WriteLine(fileName);
//if (Path.GetFileName(localPath).Length == 0)
fullLocalPath = Path.Combine(localPath, fileName);
//else
// fullLocalPath = localPath;
//Create a new subfolder under the current active folder
//fullLocalPath = System.IO.Path.Combine(fullLocalPath, "test");
// Create the subfolder
//DirectoryInfo directory = System.IO.Directory.CreateDirectory(fullLocalPath);
//directory.Attributes = FileAttributes.Normal;
//DirectorySecurity security = directory.GetAccessControl();
//security.SetAccessRule(new FileSystemAccessRule(Environment.UserDomainName.ToString() + "\\" + Environment.UserName.ToString(), FileSystemRights.Write, AccessControlType.Allow));
//directory.SetAccessControl(security);
/// Have to get size of remote object through the webrequest as not available on remote files,
/// although it does work on local files.
using (WebResponse response = WebRequest.Create(uri).GetResponse())
using (Stream stream = response.GetResponseStream())
remoteSize = response.ContentLength;
//log.InfoFormat("Downloading file (Uri={0}, Size={1}, FullLocalPath={2}).",
// uri, remoteSize, fullLocalPath);
/*}
catch (Exception ex)
{
throw new ApplicationException(string.Format("Error connecting to URI (Exception={0})", ex.Message), ex);
}*/
int bytesRead = 0, bytesReadTotal = 0;
Console.WriteLine(fullLocalPath);
try
{
using (WebClient client = new WebClient())
using (Stream streamRemote = client.OpenRead(new Uri(uri)))
using (Stream streamLocal = new FileStream(fullLocalPath, FileMode.Create, FileAccess.Write, FileShare.None))
{
byte[] byteBuffer = new byte[1024 * 1024 * 2]; // 2 meg buffer although in testing only got to 10k max usage.
int perc = 0;
while ((bytesRead = streamRemote.Read(byteBuffer, 0, byteBuffer.Length)) > 0)
{
bytesReadTotal += bytesRead;
streamLocal.Write(byteBuffer, 0, bytesRead);
int newPerc = (int)((double)bytesReadTotal / (double)remoteSize * 100);
if (newPerc > perc)
{
//log.InfoFormat("...Downloading (BytesRead={0}, Perc={1})...", bytesReadTotal, perc);
perc = newPerc;
if (progressDelegate != null)
progressDelegate(perc);
}
}
}
}
catch (Exception ex)
{
throw new ApplicationException(string.Format("Error downloading file (Exception={0})", ex.Message), ex);
}
//log.InfoFormat("File successfully downloaded (Uri={0}, BytesDownloaded={1}/{2}, FullLocalPath={3}).",
// uri, bytesReadTotal, remoteSize, fullLocalPath);
}
/// <summary>
/// Connects to the server and creates the local resource
/// </summary>
public void ReceiveFile()
{
Thread.Sleep(1000);
Console.WriteLine("Client Receiving Started");
TcpClient TestClient = new TcpClient();
IPEndPoint ClientIPEndPoint = new IPEndPoint(IPAddress.Parse(ServerIP), Port);
TestClient.Connect(ClientIPEndPoint);
NetworkStream TestStream = TestClient.GetStream();
//StreamReader TestReader = new StreamReader(TestStream);
BinaryReader TestReader = new BinaryReader(TestStream);
FileStream TestFileStream = new FileStream(FileName, FileMode.Create);
BinaryWriter TestWriter = new BinaryWriter(TestFileStream);
//char[] CharrBuff = new char[1024];
byte[] Buffer = new byte[1024];
int ByteCounter = 0;
long BytesReceived = 0;
//while (!(Nexus._Interface.DownloadCompleted(FileName)))
DownloadProgressDelegate dpd =
new DownloadProgressDelegate(Nexus._Interface.Progress1KinFileDownload);
while (BytesReceived < FileLength)
{
try
{
TestReader.Read(Buffer, 0, Buffer.Length);
TestWriter.Write(Buffer, 0, Buffer.Length);
BytesReceived = BytesReceived + Buffer.Length;
//TestWriter.Flush();
ByteCounter++;
if (ByteCounter == 100)
{
Nexus._Interface.Invoke(dpd, new Object[] { FileName });
ByteCounter = 0;
}
}
catch (Exception e)
{
Console.WriteLine("IN CLIENT CATCH!");
Console.WriteLine(e);
TestWriter.Write(Buffer, 0, Buffer.Length);
TestWriter.Flush();
Nexus._Interface.Invoke(dpd, new Object[] { FileName });
break;
}
}
Nexus._Interface.Invoke(dpd, new Object[] { FileName });
TestClient.Close();
TestReader.Close();
TestFileStream.Close();
TestWriter.Close();
Console.WriteLine("File Received!");
}
/// <summary>
/// Asynchronously downloads and assembles a multi-part file as specified by a <see cref="Neon.Deployment.DownloadManifest"/>.
/// </summary>
/// <param name="manifest">The download details.</param>
/// <param name="targetPath">The target file path.</param>
/// <param name="progressAction">Optionally specifies an action to be called with the the percentage downloaded.</param>
/// <param name="partTimeout">Optionally specifies the HTTP download timeout for each part (defaults to 10 minutes).</param>
/// <param name="retry">Optionally specifies the retry policy. This defaults to a reasonable policy.</param>
/// <param name="cancellationToken">Optionally specifies the operation cancellation token.</param>
/// <returns>The path to the downloaded file.</returns>
/// <exception cref="IOException">Thrown when the download is corrupt.</exception>
/// <exception cref="SocketException">Thrown for network errors.</exception>
/// <exception cref="HttpException">Thrown for HTTP network errors.</exception>
/// <exception cref="OperationCanceledException">Thrown when the operation was cancelled.</exception>
/// <remarks>
/// <para>
/// This method downloads the file specified by <paramref name="manifest"/> to the folder specified, creating
/// the folder first when required. The file will be downloaded in parts, where each part will be validated
/// by comparing the part's MD5 hash (when present) with the computed value. The output file will be named
/// <see cref="DownloadManifest.Name"/> and the overall MD5 hash will also be saved using the same file name but
/// <b>adding</b> the <b>.md5</b> extension.
/// </para>
/// <para>
/// This method will continue downloading a partially downloaded file. This works by validating the already
/// downloaded parts against their MD5 hashes and then continuing part downloads after the last valid part.
/// Nothing will be downloaded when the existing file is fully formed.
/// </para>
/// <note>
/// The target files (output and MD5) will be deleted when download appears to be corrupt.
/// </note>
/// </remarks>
public static async Task <string> DownloadMultiPartAsync(
DownloadManifest manifest,
string targetPath,
DownloadProgressDelegate progressAction = null,
TimeSpan partTimeout = default,
IRetryPolicy retry = null,
CancellationToken cancellationToken = default)
{
await SyncContext.Clear;
Covenant.Requires <ArgumentNullException>(manifest != null, nameof(manifest));
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(targetPath), nameof(targetPath));
retry = retry ?? new ExponentialRetryPolicy(TransientDetector.NetworkOrHttp, maxAttempts: 5);
if (partTimeout <= TimeSpan.Zero)
{
partTimeout = TimeSpan.FromMinutes(10);
}
var targetFolder = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(targetFolder);
var targetMd5Path = Path.Combine(Path.GetDirectoryName(targetPath), Path.GetFileName(targetPath) + ".md5");
var nextPartNumber = 0;
// If the target file already exists along with its MD5 hash file:
//
// 1. Compare the manifest MD5 with the local MD5 hash file and
// quickly continue with the download when these don't match.
//
// 2. When the MD5 files match, compute the MD5 of the downloaded
// file and compare that with the manifest and continue with
// the download when these don't match.
if (File.Exists(targetPath) && File.Exists(targetMd5Path) && File.ReadAllText(targetMd5Path).Trim() == manifest.Md5)
{
if (File.ReadAllText(targetMd5Path).Trim() == manifest.Md5)
{
using (var downloadStream = File.OpenRead(targetPath))
{
if (CryptoHelper.ComputeMD5String(downloadStream) == manifest.Md5)
{
return(targetPath);
}
}
}
}
NeonHelper.DeleteFile(targetMd5Path); // We'll recompute this below
// Validate the parts of any existing target file to determine where
// to start downloading missing parts.
if (File.Exists(targetPath))
{
using (var output = new FileStream(targetPath, System.IO.FileMode.Open, FileAccess.ReadWrite))
{
var pos = 0L;
foreach (var part in manifest.Parts.OrderBy(part => part.Number))
{
progressAction?.Invoke(DownloadProgressType.Check, (int)((double)pos / (double)manifest.Size * 100.0));
// Handle a partially downloaded part. We're going to truncate the file to
// remove the partial part and then break to start re-downloading the part.
if (output.Length < pos + part.Size)
{
output.SetLength(pos);
nextPartNumber = part.Number;
break;
}
// Validate the part MD5. We're going to truncate the file to remove the
// partial part and then break to start re-downloading the part.
using (var partStream = new SubStream(output, pos, part.Size))
{
if (CryptoHelper.ComputeMD5String(partStream) != part.Md5)
{
output.SetLength(pos);
nextPartNumber = part.Number;
break;
}
}
pos += part.Size;
nextPartNumber = part.Number + 1;
}
}
}
// Download any remaining parts.
if (progressAction != null && !progressAction.Invoke(DownloadProgressType.Download, 0))
{
return(targetPath);
}
if (nextPartNumber == manifest.Parts.Count)
{
progressAction?.Invoke(DownloadProgressType.Download, 100);
return(targetPath);
}
try
{
using (var httpClient = new HttpClient())
{
httpClient.Timeout = partTimeout;
using (var output = new FileStream(targetPath, System.IO.FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
// Determine the starting position of the next part to be downloaded.
var pos = manifest.Parts
.Where(part => part.Number < nextPartNumber)
.Sum(part => part.Size);
// Download the remaining parts.
foreach (var part in manifest.Parts
.Where(part => part.Number >= nextPartNumber)
.OrderBy(part => part.Number))
{
await retry.InvokeAsync(
async() =>
{
output.Position = pos;
var response = await httpClient.GetAsync(part.Uri, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
response.EnsureSuccessStatusCode();
using (var contentStream = await response.Content.ReadAsStreamAsync())
{
await contentStream.CopyToAsync(output, cancellationToken);
}
});
// Ensure that the downloaded part size matches the specification.
if (output.Position - pos != part.Size)
{
throw new IOException($"[{manifest.Name}]: Part [{part.Number}] actual size [{output.Position - pos}] does not match the expected size [{part.Size}].");
}
// Ensure that the downloaded part MD5 matches the specification.
using (var subStream = new SubStream(output, pos, part.Size))
{
var actualMd5 = CryptoHelper.ComputeMD5String(subStream);
if (actualMd5 != part.Md5)
{
throw new IOException($"[{manifest.Name}]: Part [{part.Number}] actual MD5 [{actualMd5}] does not match the expected MD5 [{part.Md5}].");
}
}
pos += part.Size;
if (progressAction != null && !progressAction.Invoke(DownloadProgressType.Download, (int)(100.0 * ((double)part.Number / (double)manifest.Parts.Count))))
{
return(targetPath);
}
}
if (output.Length != manifest.Size)
{
throw new IOException($"[{manifest.Name}]: Expected size [{manifest.Size}] got [{output.Length}].");
}
}
progressAction?.Invoke(DownloadProgressType.Download, 100);
File.WriteAllText(targetMd5Path, manifest.Md5, Encoding.ASCII);
return(targetPath);
}
}
catch (IOException)
{
NeonHelper.DeleteFile(targetPath);
NeonHelper.DeleteFile(targetMd5Path);
throw;
}
}
public static string Download(string uri, string localPath, DownloadProgressDelegate progressDelegate, string fileName)
{
long remoteSize;
string fullLocalPath; // Full local path including file name if only directory was provided.
Console.WriteLine("Attempting to download file (Uri={0}, LocalPath={1})", uri, localPath);
try
{
/// Get the name of the remote file.
Uri remoteUri = new Uri(uri);
if (string.IsNullOrEmpty(fileName))
{
fileName = Path.GetFileName(remoteUri.LocalPath);
}
if (Path.GetFileName(localPath).Length == 0) // Path.GetFileName(@"c:\temp").Length == 4 Path.GetFileName(@"c:\temp\").Length == 0
{
fullLocalPath = Path.Combine(localPath, fileName);
}
else
{
fullLocalPath = localPath;
}
/// Have to get size of remote object through the webrequest as not available on remote files,
/// although it does work on local files.
using (WebResponse response = WebRequest.Create(uri).GetResponse())
using (Stream stream = response.GetResponseStream())
remoteSize = response.ContentLength;
Console.WriteLine("Downloading file (Uri={0}, Size={1}, FullLocalPath={2}).",
uri, remoteSize, fullLocalPath);
}
catch (Exception ex)
{
throw new ApplicationException(string.Format("Error connecting to URI (Exception={0})", ex.Message), ex);
}
int bytesRead = 0, bytesReadTotal = 0;
try
{
using (WebClient client = new WebClient())
using (Stream streamRemote = client.OpenRead(new Uri(uri)))
using (Stream streamLocal = new FileStream(fullLocalPath, FileMode.Create, FileAccess.Write, FileShare.None))
{
byte[] byteBuffer = new byte[1024 * 1024 * 2]; // 2 meg buffer although in testing only got to 10k max usage.
int perc = 0;
while ((bytesRead = streamRemote.Read(byteBuffer, 0, byteBuffer.Length)) > 0)
{
bytesReadTotal += bytesRead;
streamLocal.Write(byteBuffer, 0, bytesRead);
int newPerc = (int)((double)bytesReadTotal / (double)remoteSize * 100);
if (newPerc > perc)
{
Console.WriteLine("...Downloading (BytesRead={0}, Perc={1})...", bytesReadTotal, perc);
perc = newPerc;
if (progressDelegate != null)
{
progressDelegate(perc);
}
}
}
}
}
catch (Exception ex)
{
throw new ApplicationException(string.Format("Error downloading file (Exception={0})", ex.Message), ex);
}
Console.WriteLine("File successfully downloaded (Uri={0}, BytesDownloaded={1}/{2}, FullLocalPath={3}).",
uri, bytesReadTotal, remoteSize, fullLocalPath);
return(fullLocalPath);
}
public async Task DownloadFileAsync(string ipfsPath, string path, long fileSize, bool pin, DownloadProgressDelegate progressCallBack)
{
// using (var stream = await Ipfs.DownloadAsync("object/get", default(CancellationToken), ipfsPath))
using (var stream = await Ipfs.FileSystem.ReadFileAsync(ipfsPath))
{
using (var fileStream = new FileStreamWithProgress(path, FileMode.Create))//File.Create(path))
{
fileStream.SetTotalLength(fileSize);
fileStream.ProgressCallBack += progressCallBack;
stream.CopyTo(fileStream);
fileStream.ProgressCallBack = null;
if (pin)
{
await Ipfs.Pin.AddAsync(ipfsPath);
}
}
}
}
