public void DownloadFile()
{
if (this.fileLocalPath == null)
{
return;
}
try
{
// create the download buffer
byte[] buffer = new byte[downloadBlockSize];
int readCount;
// read a block of bytes and get the number of bytes read
while ((int)(readCount = DownloadStream.Read(buffer, 0, downloadBlockSize)) > 0)
{
// save block to end of file
SaveToFile(buffer, readCount, this.fileLocalPath);
// update total bytes read
totalDownloaded += readCount;
}
}
finally
{
if (response != null)
{
response.Close();
}
this.isFileDownload = true;
}
//return true;
}
public Stream GetDownloadStream(File afile, long?start = null, long?end = null)
{
CustomDisposable <HttpWebResponse> ResponseGenerator(long instart, long inend, File file)
{
//var urldata = new YadGetResourceUrlRequest(HttpSettings, (YadWebAuth)Authent, file.FullPath)
// .MakeRequestAsync()
// .Result;
var _ = new YaDCommonRequest(HttpSettings, (YadWebAuth)Authent)
.With(new YadGetResourceUrlPostModel(file.FullPath),
out YadResponseModel <ResourceUrlData, ResourceUrlParams> itemInfo)
.MakeRequestAsync().Result;
var url = "https:" + itemInfo.Data.File;
HttpWebRequest request = new YadDownloadRequest(HttpSettings, (YadWebAuth)Authent, url, instart, inend);
var response = (HttpWebResponse)request.GetResponse();
return(new CustomDisposable <HttpWebResponse>
{
Value = response,
OnDispose = () => {}
});
}
var stream = new DownloadStream(ResponseGenerator, afile, start, end);
return(stream);
}
/// <summary>
/// Downloads the content of a file and copies it to the specified stream if the request succeeds.
/// </summary>
/// <param name="client">Http client.</param>
/// <param name="requestUri">Download uri.</param>
/// <param name="stream">Stream to write file content to.</param>
/// <returns>Http response message.</returns>
public static async Task <HttpResponseMessage> DownloadFileFromTfsAsync(this HttpClient client, Uri requestUri, Stream stream)
{
TFCommonUtil.CheckForNull(client, "client");
TFCommonUtil.CheckForNull(requestUri, "requestUri");
TFCommonUtil.CheckForNull(stream, "stream");
HttpResponseMessage response = await client.GetAsync(requestUri.ToString());
if (response.IsSuccessStatusCode && response.StatusCode != HttpStatusCode.NoContent)
{
bool decompress;
if (StringComparer.OrdinalIgnoreCase.Equals(response.Content.Headers.ContentType.MediaType, "application/octet-stream"))
{
decompress = false;
}
else if (StringComparer.OrdinalIgnoreCase.Equals(response.Content.Headers.ContentType.MediaType, "application/gzip"))
{
decompress = true;
}
else
{
throw new Exception(string.Format("Unsupported Content Type {0}", response.Content.Headers.ContentType.MediaType));
}
using (DownloadStream downloadStream = new DownloadStream(stream, decompress, response.Content.Headers.ContentMD5))
{
await response.Content.CopyToAsync(downloadStream);
downloadStream.ValidateHash();
}
}
return(response);
}
async Task <string> DoRun(HttpResponseMessage response, CancellationToken cancellationToken)
{
var length = response.Content.Headers.ContentLength;
if (length == null)
{
var msg = "Response did not contain Content-Length!";
if (AppDelegate.Instance.Settings.DownloadWithoutLength)
{
OnMessageEvent(msg);
}
else
{
return(msg);
}
}
else
{
OnProgressChangedEvent(0, length.Value);
}
cancellationToken.ThrowIfCancellationRequested();
if (!response.IsSuccessStatusCode)
{
return(string.Format("ERROR: {0}", response.ReasonPhrase));
}
var mystream = new DownloadStream(this, length);
await response.Content.CopyToAsync(mystream);
return(string.Format("Download complete ({0} bytes).", mystream.Position));
}
/// <summary>
/// GenerateICPacketReport callback method
/// </summary>
/// <param name="result">IC Packet file byte stream</param>
private void GenerateICPacketReportCallbackMethod(Byte[] result)
{
string methodNamespace = String.Format("{0}.{1}", GetType().FullName, System.Reflection.MethodInfo.GetCurrentMethod().Name);
Logging.LogBeginMethod(logger, methodNamespace);
try
{
if (result != null)
{
Logging.LogMethodParameter(logger, methodNamespace, result, 1);
DownloadStream.Write(result, 0, result.Length);
DownloadStream.Close();
DownloadStream = null;
}
else
{
Prompt.ShowDialog("An Error ocurred while downloading the preview report from server.");
Logging.LogMethodParameterNull(logger, methodNamespace, 1);
}
}
catch (Exception ex)
{
Prompt.ShowDialog("Message: " + ex.Message + "\nStackTrace: " + Logging.StackTraceToString(ex), "Exception", MessageBoxButton.OK);
Logging.LogException(logger, ex);
}
finally
{
Logging.LogEndMethod(logger, methodNamespace);
BusyIndicatorNotification();
}
}
public async Task DownloadStream_Supports_Seeking()
{
string bucketname = "downloadstreamtest2";
var result = await _bucketService.CreateBucketAsync(bucketname);
var bucket = await _bucketService.GetBucketAsync(bucketname);
byte[] bytesToUpload = new byte[250];
for (int i = 0; i < 250; i++)
{
bytesToUpload[i] = Convert.ToByte(i);
}
var uploadOperation = await _objectService.UploadObjectAsync(bucket, "myfile.txt", new UploadOptions(), bytesToUpload, false);
await uploadOperation.StartUploadAsync();
var stream = new DownloadStream(bucket, bytesToUpload.Length, "myfile.txt");
byte[] bytesReceived = new byte[50];
stream.Seek(100, System.IO.SeekOrigin.Begin);
await stream.ReadAsync(bytesReceived, 0, 50);
for (int i = 0; i < 50; i++)
{
Assert.AreEqual(100 + i, Convert.ToInt32(bytesReceived[i]));
}
}
/// <summary>
/// Downloads the content of a file and copies it to the specified stream if the request succeeds.
/// </summary>
/// <param name="client">Http client.</param>
/// <param name="requestUri">Download uri.</param>
/// <param name="stream">Stream to write file content to.</param>
/// <returns>Http response message.</returns>
public static async Task<HttpResponseMessage> DownloadFileFromTfsAsync(this HttpClient client, Uri requestUri, Stream stream)
{
TFCommonUtil.CheckForNull(client, "client");
TFCommonUtil.CheckForNull(requestUri, "requestUri");
TFCommonUtil.CheckForNull(stream, "stream");
HttpResponseMessage response = await client.GetAsync(requestUri.ToString());
if (response.IsSuccessStatusCode && response.StatusCode != HttpStatusCode.NoContent)
{
bool decompress;
if (StringComparer.OrdinalIgnoreCase.Equals(response.Content.Headers.ContentType.MediaType, "application/octet-stream"))
{
decompress = false;
}
else if (StringComparer.OrdinalIgnoreCase.Equals(response.Content.Headers.ContentType.MediaType, "application/gzip"))
{
decompress = true;
}
else
{
throw new Exception(string.Format("Unsupported Content Type {0}", response.Content.Headers.ContentType.MediaType));
}
using (DownloadStream downloadStream = new DownloadStream(stream, decompress, response.Content.Headers.ContentMD5))
{
await response.Content.CopyToAsync(downloadStream);
downloadStream.ValidateHash();
}
}
return response;
}
public async Task DownloadStream_Provides_First50Bytes()
{
string bucketname = "downloadstreamtest1";
var result = await _bucketService.CreateBucketAsync(bucketname);
var bucket = await _bucketService.GetBucketAsync(bucketname);
byte[] bytesToUpload = new byte[250];
for (int i = 0; i < 250; i++)
{
bytesToUpload[i] = Convert.ToByte(i);
}
var uploadOperation = await _objectService.UploadObjectAsync(bucket, "myfile.txt", new UploadOptions(), bytesToUpload, false);
await uploadOperation.StartUploadAsync();
var stream = new DownloadStream(bucket, bytesToUpload.Length, "myfile.txt", _access);
byte[] bytesReceived = new byte[50];
await stream.ReadAsync(bytesReceived, 0, 50);
for (int i = 0; i < 50; i++)
{
Assert.AreEqual(i, Convert.ToInt32(bytesReceived[i]));
}
}
private async Task DownloadAsync(string file, Manifest manifest, Action <int> progress)
{
Directory.CreateDirectory(Path.GetDirectoryName(file));
using (var downloadStream = new DownloadStream(file, manifest, false, Client))
using (var fileStream = File.OpenWrite(file))
using (var semaphore = new SemaphoreSlim(20))
{
Log($"Downloading {file} in {downloadStream.ChunkCount} chunks (Size: {GetReadableSize(downloadStream.Length)})");
var tasks = new Task[downloadStream.ChunkCount];
var writeChunkInd = 0;
for (var i = 0; i < downloadStream.ChunkCount; i++)
{
await semaphore.WaitAsync().ConfigureAwait(false);
tasks[i] = downloadStream.GetChunk(i).ContinueWith(async(t, nObj) =>
{
var n = (int)nObj;
while (n != writeChunkInd)
{
await Task.Delay(25);
}
await fileStream.WriteAsync(t.Result, 0, t.Result.Length).ConfigureAwait(false);
writeChunkInd++;
semaphore.Release();
progress(t.Result.Length);
}, i).Unwrap();
}
await Task.WhenAll(tasks).ConfigureAwait(false);
}
Log($"Downloaded {file}");
}
public Stream GetDownloadStream(File afile, long?start = null, long?end = null)
{
CustomDisposable <HttpWebResponse> ResponseGenerator(long instart, long inend, File file)
{
bool isLinked = !string.IsNullOrEmpty(file.PublicLink);
string downloadkey = isLinked
? Authent.DownloadToken
: Authent.AccessToken;
var shard = isLinked
? _cachedShards.Value[ShardType.WeblinkGet]
: _cachedShards.Value[ShardType.Get];
string url = !isLinked
? $"{shard.Url}{Uri.EscapeDataString(file.FullPath)}"
: $"{shard.Url}{new Uri(ConstSettings.PublishFileLink + file.PublicLink).PathAndQuery.Remove(0, "/public".Length)}?key={downloadkey}";
var request = (HttpWebRequest)WebRequest.Create(url);
request.Headers.Add("Accept-Ranges", "bytes");
request.AddRange(instart, inend);
request.Proxy = HttpSettings.Proxy;
request.CookieContainer = Authent.Cookies;
request.Method = "GET";
request.ContentType = MediaTypeNames.Application.Octet;
request.Accept = "*/*";
request.UserAgent = HttpSettings.UserAgent;
request.AllowReadStreamBuffering = false;
request.Timeout = 15 * 1000;
var response = (HttpWebResponse)request.GetResponse();
return(new CustomDisposable <HttpWebResponse>
{
Value = response,
OnDispose = () =>
{
//_shardManager.DownloadServersPending.Free(downServer);
//watch.Stop();
//Logger.Debug($"HTTP:{request.Method}:{request.RequestUri.AbsoluteUri} ({watch.Elapsed.Milliseconds} ms)");
}
});
}
var stream = new DownloadStream(ResponseGenerator, afile, start, end);
return(stream);
}
/// <summary>
/// Copy the uploaded stream for download.
/// </summary>
/// <param name="stream">stream to copy to</param>
/// <param name="timeout">seconds to wait</param>
/// <returns></returns>
/// <exception cref="TimeoutException"></exception>
public async Task CopyDownLoadStream(Stream stream, int timeout)
{
var count = 0;
var maxCount = timeout * 10;
while (DownloadStream == null)
{
await Task.Delay(100);
if (++count > maxCount)
{
throw new TimeoutException("Timeout occurred waiting for download stream");
}
}
await DownloadStream.CopyToAsync(stream);
}
/// <summary>
///
/// </summary>
/// <param name="result"></param>
private void CreatePresentationCallBackMethod(PresentationFile result)
{
string methodNamespace = String.Format("{0}.{1}", GetType().FullName, System.Reflection.MethodInfo.GetCurrentMethod().Name);
Logging.LogBeginMethod(logger, methodNamespace);
try
{
if (result != null && result.PresentationId > 0)
{
Logging.LogMethodParameter(logger, methodNamespace, result, 1);
/* eventAggregator.GetEvent<ToolboxUpdateEvent>().Publish(DashboardCategoryType.INVESTMENT_COMMITTEE_PRESENTATIONS);
* ICNavigation.Update(ICNavigationInfo.MeetingInfo, iCPresentationOverviewInfo);
* regionManager.RequestNavigate(RegionNames.MAIN_REGION, "ViewDashboardICPresentation", UriKind.Relative);*/
iCPresentationOverviewInfo.PresentationID = result.PresentationId;
DownloadStream.Write(result.FileStream, 0, result.FileStream.Length);
DownloadStream.Close();
DownloadStream = null;
ICNavigation.Update(ICNavigationInfo.PresentationOverviewInfo, iCPresentationOverviewInfo);
eventAggregator.GetEvent <ToolboxUpdateEvent>().Publish(DashboardCategoryType.INVESTMENT_COMMITTEE_IC_PRESENTATION);
regionManager.RequestNavigate(RegionNames.MAIN_REGION, new Uri("ViewDashboardICPresentation", UriKind.Relative));
}
else
{
Logging.LogMethodParameterNull(logger, methodNamespace, 1);
}
}
catch (Exception ex)
{
Prompt.ShowDialog("Message: " + ex.Message + "\nStackTrace: " + Logging.StackTraceToString(ex), "Exception", MessageBoxButton.OK);
Logging.LogException(logger, ex);
}
finally
{
Logging.LogEndMethod(logger, methodNamespace);
BusyIndicatorNotification();
}
}
//public HttpWebRequest UploadRequest(File file, UploadMultipartBoundary boundary)
//{
// var shard = GetShardInfo(ShardType.Upload).Result;
// var url = new Uri($"{shard.Url}?cloud_domain=2&{Authent.Login}");
// var result = new UploadRequest(url.OriginalString, file, Authent, HttpSettings);
// return result;
//}
public Stream GetDownloadStream(File afile, long?start = null, long?end = null)
{
CustomDisposable <HttpWebResponse> ResponseGenerator(long instart, long inend, File file)
{
HttpWebRequest request = new DownloadRequest(file, instart, inend, Authent, HttpSettings, _cachedShards);
var response = (HttpWebResponse)request.GetResponse();
return(new CustomDisposable <HttpWebResponse>
{
Value = response,
OnDispose = () =>
{
//_shardManager.DownloadServersPending.Free(downServer);
//watch.Stop();
//Logger.Debug($"HTTP:{request.Method}:{request.RequestUri.AbsoluteUri} ({watch.Elapsed.Milliseconds} ms)");
}
});
}
var stream = new DownloadStream(ResponseGenerator, afile, start, end);
return(stream);
}
async Task<string> DoRun (HttpResponseMessage response, CancellationToken cancellationToken)
{
var length = response.Content.Headers.ContentLength;
if (length == null) {
var msg = "Response did not contain Content-Length!";
if (AppDelegate.Instance.Settings.DownloadWithoutLength)
OnMessageEvent (msg);
else
return msg;
} else {
OnProgressChangedEvent (0, length.Value);
}
cancellationToken.ThrowIfCancellationRequested ();
if (!response.IsSuccessStatusCode)
return string.Format ("ERROR: {0}", response.ReasonPhrase);
var mystream = new DownloadStream (this, length);
await response.Content.CopyToAsync (mystream);
return string.Format ("Download complete ({0} bytes).", mystream.Position);
}
async void start_dowork(Action <string> UpdateText)
{
Directory.CreateDirectory(installlocation);
var tmp = installlocation + Path.DirectorySeparatorChar;
var checkfile = tmp + "checksums.txt";
Regex regex = new Regex(@"([^\s]+)\s+[^/]+/(.*)", RegexOptions.IgnoreCase);
if (Download.getFilefromNet(md5s, checkfile))
{
List <string> filestoget = new List <string>();
var sums = File.ReadAllLines(checkfile);
Parallel.ForEach(sums, sum =>
{
var match = regex.Match(sum);
if (match.Success)
{
var file = tmp + match.Groups[2].Value;
if (File.Exists(file))
{
if (!MD5File(file, match.Groups[1].Value))
{
lock (filestoget)
filestoget.Add(match.Groups[2].Value);
}
}
else
{
lock (filestoget)
filestoget.Add(match.Groups[2].Value);
}
}
});
DownloadStream ds = new DownloadStream(zip);
// length / 100 = part size
// part size rounded to closest 100kb
//ds.chunksize = (int) (Math.Floor((ds.Length/100.0) / 100000.0) * 100000.0);
//Console.WriteLine("chunk size {0}", ds.chunksize);
int got = 0;
using (ZipArchive zip = new ZipArchive(ds))
{
foreach (var file in filestoget)
{
var entry = zip.GetEntry(file);
UpdateText(String.Format("Getting {0}\nFile {1} of {2}\nCompressed size {3}\nSize {4}", file, got, filestoget.Count,
entry?.CompressedLength, entry?.Length));
ds.chunksize = (int)entry.CompressedLength;
var output = tmp + file.Replace('/', Path.DirectorySeparatorChar);
var dir = Path.GetDirectoryName(output);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
if (Application.ExecutablePath.ToLower() == output.ToLower())
{
continue;
}
await Task.Run(() => { entry.ExtractToFile(output, true); });
got++;
}
}
UpdateText("Done");
}
else
{
MessageBox.Show("Failed to get checksum file");
}
}
private DownloadStream GetDownloadStreamInternal(File afile, long?start = null, long?end = null)
{
bool isLinked = afile.PublicLinks.Any();
Cached <ServerRequestResult> downServer = null;
var pendingServers = isLinked
? ShardManager.WeblinkDownloadServersPending
: ShardManager.DownloadServersPending;
Stopwatch watch = new Stopwatch();
HttpWebRequest request = null;
CustomDisposable <HttpWebResponse> ResponseGenerator(long instart, long inend, File file)
{
var resp = Retry.Do(() =>
{
downServer = pendingServers.Next(downServer);
string url = (isLinked
? $"{downServer.Value.Url}{WebDavPath.EscapeDataString(file.PublicLinks.First().Uri.PathAndQuery)}"
: $"{downServer.Value.Url}{Uri.EscapeDataString(file.FullPath.TrimStart('/'))}") +
$"?client_id={HttpSettings.ClientId}&token={Authent.AccessToken}";
var uri = new Uri(url);
request = (HttpWebRequest)WebRequest.Create(uri.OriginalString);
request.AddRange(instart, inend);
request.Proxy = HttpSettings.Proxy;
request.CookieContainer = Authent.Cookies;
request.Method = "GET";
request.Accept = "*/*";
request.UserAgent = HttpSettings.UserAgent;
request.Host = uri.Host;
request.AllowWriteStreamBuffering = false;
if (isLinked)
{
request.Headers.Add("Accept-Ranges", "bytes");
request.ContentType = MediaTypeNames.Application.Octet;
request.Referer = $"{ConstSettings.CloudDomain}/home/{Uri.EscapeDataString(file.Path)}";
request.Headers.Add("Origin", ConstSettings.CloudDomain);
}
request.Timeout = 15 * 1000;
request.ReadWriteTimeout = 15 * 1000;
watch.Start();
var response = (HttpWebResponse)request.GetResponse();
return(new CustomDisposable <HttpWebResponse>
{
Value = response,
OnDispose = () =>
{
pendingServers.Free(downServer);
watch.Stop();
Logger.Debug($"HTTP:{request.Method}:{request.RequestUri.AbsoluteUri} ({watch.Elapsed.Milliseconds} ms)");
}
});
},
exception =>
((exception as WebException)?.Response as HttpWebResponse)?.StatusCode == HttpStatusCode.NotFound,
exception =>
{
pendingServers.Free(downServer);
Logger.Warn($"Retrying HTTP:{request.Method}:{request.RequestUri.AbsoluteUri} on exception {exception.Message}");
},
TimeSpan.FromSeconds(1), 2);
return(resp);
}
var stream = new DownloadStream(ResponseGenerator, afile, start, end);
return(stream);
}
public void Dispose()
{
DownloadStream?.Dispose();
}
protected virtual void Dispose(bool disposing)
{
DownloadStream?.Dispose();
}
