public void DownloadFile() { if (this.fileLocalPath == null) { return; } try { // create the download buffer byte[] buffer = new byte[downloadBlockSize]; int readCount; // read a block of bytes and get the number of bytes read while ((int)(readCount = DownloadStream.Read(buffer, 0, downloadBlockSize)) > 0) { // save block to end of file SaveToFile(buffer, readCount, this.fileLocalPath); // update total bytes read totalDownloaded += readCount; } } finally { if (response != null) { response.Close(); } this.isFileDownload = true; } //return true; }
public Stream GetDownloadStream(File afile, long?start = null, long?end = null) { CustomDisposable <HttpWebResponse> ResponseGenerator(long instart, long inend, File file) { //var urldata = new YadGetResourceUrlRequest(HttpSettings, (YadWebAuth)Authent, file.FullPath) // .MakeRequestAsync() // .Result; var _ = new YaDCommonRequest(HttpSettings, (YadWebAuth)Authent) .With(new YadGetResourceUrlPostModel(file.FullPath), out YadResponseModel <ResourceUrlData, ResourceUrlParams> itemInfo) .MakeRequestAsync().Result; var url = "https:" + itemInfo.Data.File; HttpWebRequest request = new YadDownloadRequest(HttpSettings, (YadWebAuth)Authent, url, instart, inend); var response = (HttpWebResponse)request.GetResponse(); return(new CustomDisposable <HttpWebResponse> { Value = response, OnDispose = () => {} }); } var stream = new DownloadStream(ResponseGenerator, afile, start, end); return(stream); }
/// <summary> /// Downloads the content of a file and copies it to the specified stream if the request succeeds. /// </summary> /// <param name="client">Http client.</param> /// <param name="requestUri">Download uri.</param> /// <param name="stream">Stream to write file content to.</param> /// <returns>Http response message.</returns> public static async Task <HttpResponseMessage> DownloadFileFromTfsAsync(this HttpClient client, Uri requestUri, Stream stream) { TFCommonUtil.CheckForNull(client, "client"); TFCommonUtil.CheckForNull(requestUri, "requestUri"); TFCommonUtil.CheckForNull(stream, "stream"); HttpResponseMessage response = await client.GetAsync(requestUri.ToString()); if (response.IsSuccessStatusCode && response.StatusCode != HttpStatusCode.NoContent) { bool decompress; if (StringComparer.OrdinalIgnoreCase.Equals(response.Content.Headers.ContentType.MediaType, "application/octet-stream")) { decompress = false; } else if (StringComparer.OrdinalIgnoreCase.Equals(response.Content.Headers.ContentType.MediaType, "application/gzip")) { decompress = true; } else { throw new Exception(string.Format("Unsupported Content Type {0}", response.Content.Headers.ContentType.MediaType)); } using (DownloadStream downloadStream = new DownloadStream(stream, decompress, response.Content.Headers.ContentMD5)) { await response.Content.CopyToAsync(downloadStream); downloadStream.ValidateHash(); } } return(response); }
async Task <string> DoRun(HttpResponseMessage response, CancellationToken cancellationToken) { var length = response.Content.Headers.ContentLength; if (length == null) { var msg = "Response did not contain Content-Length!"; if (AppDelegate.Instance.Settings.DownloadWithoutLength) { OnMessageEvent(msg); } else { return(msg); } } else { OnProgressChangedEvent(0, length.Value); } cancellationToken.ThrowIfCancellationRequested(); if (!response.IsSuccessStatusCode) { return(string.Format("ERROR: {0}", response.ReasonPhrase)); } var mystream = new DownloadStream(this, length); await response.Content.CopyToAsync(mystream); return(string.Format("Download complete ({0} bytes).", mystream.Position)); }
/// <summary> /// GenerateICPacketReport callback method /// </summary> /// <param name="result">IC Packet file byte stream</param> private void GenerateICPacketReportCallbackMethod(Byte[] result) { string methodNamespace = String.Format("{0}.{1}", GetType().FullName, System.Reflection.MethodInfo.GetCurrentMethod().Name); Logging.LogBeginMethod(logger, methodNamespace); try { if (result != null) { Logging.LogMethodParameter(logger, methodNamespace, result, 1); DownloadStream.Write(result, 0, result.Length); DownloadStream.Close(); DownloadStream = null; } else { Prompt.ShowDialog("An Error ocurred while downloading the preview report from server."); Logging.LogMethodParameterNull(logger, methodNamespace, 1); } } catch (Exception ex) { Prompt.ShowDialog("Message: " + ex.Message + "\nStackTrace: " + Logging.StackTraceToString(ex), "Exception", MessageBoxButton.OK); Logging.LogException(logger, ex); } finally { Logging.LogEndMethod(logger, methodNamespace); BusyIndicatorNotification(); } }
public async Task DownloadStream_Supports_Seeking() { string bucketname = "downloadstreamtest2"; var result = await _bucketService.CreateBucketAsync(bucketname); var bucket = await _bucketService.GetBucketAsync(bucketname); byte[] bytesToUpload = new byte[250]; for (int i = 0; i < 250; i++) { bytesToUpload[i] = Convert.ToByte(i); } var uploadOperation = await _objectService.UploadObjectAsync(bucket, "myfile.txt", new UploadOptions(), bytesToUpload, false); await uploadOperation.StartUploadAsync(); var stream = new DownloadStream(bucket, bytesToUpload.Length, "myfile.txt"); byte[] bytesReceived = new byte[50]; stream.Seek(100, System.IO.SeekOrigin.Begin); await stream.ReadAsync(bytesReceived, 0, 50); for (int i = 0; i < 50; i++) { Assert.AreEqual(100 + i, Convert.ToInt32(bytesReceived[i])); } }
/// <summary> /// Downloads the content of a file and copies it to the specified stream if the request succeeds. /// </summary> /// <param name="client">Http client.</param> /// <param name="requestUri">Download uri.</param> /// <param name="stream">Stream to write file content to.</param> /// <returns>Http response message.</returns> public static async Task<HttpResponseMessage> DownloadFileFromTfsAsync(this HttpClient client, Uri requestUri, Stream stream) { TFCommonUtil.CheckForNull(client, "client"); TFCommonUtil.CheckForNull(requestUri, "requestUri"); TFCommonUtil.CheckForNull(stream, "stream"); HttpResponseMessage response = await client.GetAsync(requestUri.ToString()); if (response.IsSuccessStatusCode && response.StatusCode != HttpStatusCode.NoContent) { bool decompress; if (StringComparer.OrdinalIgnoreCase.Equals(response.Content.Headers.ContentType.MediaType, "application/octet-stream")) { decompress = false; } else if (StringComparer.OrdinalIgnoreCase.Equals(response.Content.Headers.ContentType.MediaType, "application/gzip")) { decompress = true; } else { throw new Exception(string.Format("Unsupported Content Type {0}", response.Content.Headers.ContentType.MediaType)); } using (DownloadStream downloadStream = new DownloadStream(stream, decompress, response.Content.Headers.ContentMD5)) { await response.Content.CopyToAsync(downloadStream); downloadStream.ValidateHash(); } } return response; }
public async Task DownloadStream_Provides_First50Bytes() { string bucketname = "downloadstreamtest1"; var result = await _bucketService.CreateBucketAsync(bucketname); var bucket = await _bucketService.GetBucketAsync(bucketname); byte[] bytesToUpload = new byte[250]; for (int i = 0; i < 250; i++) { bytesToUpload[i] = Convert.ToByte(i); } var uploadOperation = await _objectService.UploadObjectAsync(bucket, "myfile.txt", new UploadOptions(), bytesToUpload, false); await uploadOperation.StartUploadAsync(); var stream = new DownloadStream(bucket, bytesToUpload.Length, "myfile.txt", _access); byte[] bytesReceived = new byte[50]; await stream.ReadAsync(bytesReceived, 0, 50); for (int i = 0; i < 50; i++) { Assert.AreEqual(i, Convert.ToInt32(bytesReceived[i])); } }
private async Task DownloadAsync(string file, Manifest manifest, Action <int> progress) { Directory.CreateDirectory(Path.GetDirectoryName(file)); using (var downloadStream = new DownloadStream(file, manifest, false, Client)) using (var fileStream = File.OpenWrite(file)) using (var semaphore = new SemaphoreSlim(20)) { Log($"Downloading {file} in {downloadStream.ChunkCount} chunks (Size: {GetReadableSize(downloadStream.Length)})"); var tasks = new Task[downloadStream.ChunkCount]; var writeChunkInd = 0; for (var i = 0; i < downloadStream.ChunkCount; i++) { await semaphore.WaitAsync().ConfigureAwait(false); tasks[i] = downloadStream.GetChunk(i).ContinueWith(async(t, nObj) => { var n = (int)nObj; while (n != writeChunkInd) { await Task.Delay(25); } await fileStream.WriteAsync(t.Result, 0, t.Result.Length).ConfigureAwait(false); writeChunkInd++; semaphore.Release(); progress(t.Result.Length); }, i).Unwrap(); } await Task.WhenAll(tasks).ConfigureAwait(false); } Log($"Downloaded {file}"); }
public Stream GetDownloadStream(File afile, long?start = null, long?end = null) { CustomDisposable <HttpWebResponse> ResponseGenerator(long instart, long inend, File file) { bool isLinked = !string.IsNullOrEmpty(file.PublicLink); string downloadkey = isLinked ? Authent.DownloadToken : Authent.AccessToken; var shard = isLinked ? _cachedShards.Value[ShardType.WeblinkGet] : _cachedShards.Value[ShardType.Get]; string url = !isLinked ? $"{shard.Url}{Uri.EscapeDataString(file.FullPath)}" : $"{shard.Url}{new Uri(ConstSettings.PublishFileLink + file.PublicLink).PathAndQuery.Remove(0, "/public".Length)}?key={downloadkey}"; var request = (HttpWebRequest)WebRequest.Create(url); request.Headers.Add("Accept-Ranges", "bytes"); request.AddRange(instart, inend); request.Proxy = HttpSettings.Proxy; request.CookieContainer = Authent.Cookies; request.Method = "GET"; request.ContentType = MediaTypeNames.Application.Octet; request.Accept = "*/*"; request.UserAgent = HttpSettings.UserAgent; request.AllowReadStreamBuffering = false; request.Timeout = 15 * 1000; var response = (HttpWebResponse)request.GetResponse(); return(new CustomDisposable <HttpWebResponse> { Value = response, OnDispose = () => { //_shardManager.DownloadServersPending.Free(downServer); //watch.Stop(); //Logger.Debug($"HTTP:{request.Method}:{request.RequestUri.AbsoluteUri} ({watch.Elapsed.Milliseconds} ms)"); } }); } var stream = new DownloadStream(ResponseGenerator, afile, start, end); return(stream); }
/// <summary> /// Copy the uploaded stream for download. /// </summary> /// <param name="stream">stream to copy to</param> /// <param name="timeout">seconds to wait</param> /// <returns></returns> /// <exception cref="TimeoutException"></exception> public async Task CopyDownLoadStream(Stream stream, int timeout) { var count = 0; var maxCount = timeout * 10; while (DownloadStream == null) { await Task.Delay(100); if (++count > maxCount) { throw new TimeoutException("Timeout occurred waiting for download stream"); } } await DownloadStream.CopyToAsync(stream); }
/// <summary> /// /// </summary> /// <param name="result"></param> private void CreatePresentationCallBackMethod(PresentationFile result) { string methodNamespace = String.Format("{0}.{1}", GetType().FullName, System.Reflection.MethodInfo.GetCurrentMethod().Name); Logging.LogBeginMethod(logger, methodNamespace); try { if (result != null && result.PresentationId > 0) { Logging.LogMethodParameter(logger, methodNamespace, result, 1); /* eventAggregator.GetEvent<ToolboxUpdateEvent>().Publish(DashboardCategoryType.INVESTMENT_COMMITTEE_PRESENTATIONS); * ICNavigation.Update(ICNavigationInfo.MeetingInfo, iCPresentationOverviewInfo); * regionManager.RequestNavigate(RegionNames.MAIN_REGION, "ViewDashboardICPresentation", UriKind.Relative);*/ iCPresentationOverviewInfo.PresentationID = result.PresentationId; DownloadStream.Write(result.FileStream, 0, result.FileStream.Length); DownloadStream.Close(); DownloadStream = null; ICNavigation.Update(ICNavigationInfo.PresentationOverviewInfo, iCPresentationOverviewInfo); eventAggregator.GetEvent <ToolboxUpdateEvent>().Publish(DashboardCategoryType.INVESTMENT_COMMITTEE_IC_PRESENTATION); regionManager.RequestNavigate(RegionNames.MAIN_REGION, new Uri("ViewDashboardICPresentation", UriKind.Relative)); } else { Logging.LogMethodParameterNull(logger, methodNamespace, 1); } } catch (Exception ex) { Prompt.ShowDialog("Message: " + ex.Message + "\nStackTrace: " + Logging.StackTraceToString(ex), "Exception", MessageBoxButton.OK); Logging.LogException(logger, ex); } finally { Logging.LogEndMethod(logger, methodNamespace); BusyIndicatorNotification(); } }
//public HttpWebRequest UploadRequest(File file, UploadMultipartBoundary boundary) //{ // var shard = GetShardInfo(ShardType.Upload).Result; // var url = new Uri($"{shard.Url}?cloud_domain=2&{Authent.Login}"); // var result = new UploadRequest(url.OriginalString, file, Authent, HttpSettings); // return result; //} public Stream GetDownloadStream(File afile, long?start = null, long?end = null) { CustomDisposable <HttpWebResponse> ResponseGenerator(long instart, long inend, File file) { HttpWebRequest request = new DownloadRequest(file, instart, inend, Authent, HttpSettings, _cachedShards); var response = (HttpWebResponse)request.GetResponse(); return(new CustomDisposable <HttpWebResponse> { Value = response, OnDispose = () => { //_shardManager.DownloadServersPending.Free(downServer); //watch.Stop(); //Logger.Debug($"HTTP:{request.Method}:{request.RequestUri.AbsoluteUri} ({watch.Elapsed.Milliseconds} ms)"); } }); } var stream = new DownloadStream(ResponseGenerator, afile, start, end); return(stream); }
async Task<string> DoRun (HttpResponseMessage response, CancellationToken cancellationToken) { var length = response.Content.Headers.ContentLength; if (length == null) { var msg = "Response did not contain Content-Length!"; if (AppDelegate.Instance.Settings.DownloadWithoutLength) OnMessageEvent (msg); else return msg; } else { OnProgressChangedEvent (0, length.Value); } cancellationToken.ThrowIfCancellationRequested (); if (!response.IsSuccessStatusCode) return string.Format ("ERROR: {0}", response.ReasonPhrase); var mystream = new DownloadStream (this, length); await response.Content.CopyToAsync (mystream); return string.Format ("Download complete ({0} bytes).", mystream.Position); }
async void start_dowork(Action <string> UpdateText) { Directory.CreateDirectory(installlocation); var tmp = installlocation + Path.DirectorySeparatorChar; var checkfile = tmp + "checksums.txt"; Regex regex = new Regex(@"([^\s]+)\s+[^/]+/(.*)", RegexOptions.IgnoreCase); if (Download.getFilefromNet(md5s, checkfile)) { List <string> filestoget = new List <string>(); var sums = File.ReadAllLines(checkfile); Parallel.ForEach(sums, sum => { var match = regex.Match(sum); if (match.Success) { var file = tmp + match.Groups[2].Value; if (File.Exists(file)) { if (!MD5File(file, match.Groups[1].Value)) { lock (filestoget) filestoget.Add(match.Groups[2].Value); } } else { lock (filestoget) filestoget.Add(match.Groups[2].Value); } } }); DownloadStream ds = new DownloadStream(zip); // length / 100 = part size // part size rounded to closest 100kb //ds.chunksize = (int) (Math.Floor((ds.Length/100.0) / 100000.0) * 100000.0); //Console.WriteLine("chunk size {0}", ds.chunksize); int got = 0; using (ZipArchive zip = new ZipArchive(ds)) { foreach (var file in filestoget) { var entry = zip.GetEntry(file); UpdateText(String.Format("Getting {0}\nFile {1} of {2}\nCompressed size {3}\nSize {4}", file, got, filestoget.Count, entry?.CompressedLength, entry?.Length)); ds.chunksize = (int)entry.CompressedLength; var output = tmp + file.Replace('/', Path.DirectorySeparatorChar); var dir = Path.GetDirectoryName(output); if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); } if (Application.ExecutablePath.ToLower() == output.ToLower()) { continue; } await Task.Run(() => { entry.ExtractToFile(output, true); }); got++; } } UpdateText("Done"); } else { MessageBox.Show("Failed to get checksum file"); } }
private DownloadStream GetDownloadStreamInternal(File afile, long?start = null, long?end = null) { bool isLinked = afile.PublicLinks.Any(); Cached <ServerRequestResult> downServer = null; var pendingServers = isLinked ? ShardManager.WeblinkDownloadServersPending : ShardManager.DownloadServersPending; Stopwatch watch = new Stopwatch(); HttpWebRequest request = null; CustomDisposable <HttpWebResponse> ResponseGenerator(long instart, long inend, File file) { var resp = Retry.Do(() => { downServer = pendingServers.Next(downServer); string url = (isLinked ? $"{downServer.Value.Url}{WebDavPath.EscapeDataString(file.PublicLinks.First().Uri.PathAndQuery)}" : $"{downServer.Value.Url}{Uri.EscapeDataString(file.FullPath.TrimStart('/'))}") + $"?client_id={HttpSettings.ClientId}&token={Authent.AccessToken}"; var uri = new Uri(url); request = (HttpWebRequest)WebRequest.Create(uri.OriginalString); request.AddRange(instart, inend); request.Proxy = HttpSettings.Proxy; request.CookieContainer = Authent.Cookies; request.Method = "GET"; request.Accept = "*/*"; request.UserAgent = HttpSettings.UserAgent; request.Host = uri.Host; request.AllowWriteStreamBuffering = false; if (isLinked) { request.Headers.Add("Accept-Ranges", "bytes"); request.ContentType = MediaTypeNames.Application.Octet; request.Referer = $"{ConstSettings.CloudDomain}/home/{Uri.EscapeDataString(file.Path)}"; request.Headers.Add("Origin", ConstSettings.CloudDomain); } request.Timeout = 15 * 1000; request.ReadWriteTimeout = 15 * 1000; watch.Start(); var response = (HttpWebResponse)request.GetResponse(); return(new CustomDisposable <HttpWebResponse> { Value = response, OnDispose = () => { pendingServers.Free(downServer); watch.Stop(); Logger.Debug($"HTTP:{request.Method}:{request.RequestUri.AbsoluteUri} ({watch.Elapsed.Milliseconds} ms)"); } }); }, exception => ((exception as WebException)?.Response as HttpWebResponse)?.StatusCode == HttpStatusCode.NotFound, exception => { pendingServers.Free(downServer); Logger.Warn($"Retrying HTTP:{request.Method}:{request.RequestUri.AbsoluteUri} on exception {exception.Message}"); }, TimeSpan.FromSeconds(1), 2); return(resp); } var stream = new DownloadStream(ResponseGenerator, afile, start, end); return(stream); }
public void Dispose() { DownloadStream?.Dispose(); }
protected virtual void Dispose(bool disposing) { DownloadStream?.Dispose(); }