private async Task DownloadObjectAsyncImpl( string baseUri, Stream destination, DownloadObjectOptions options, CancellationToken cancellationToken, IProgress <IDownloadProgress> progress) { Preconditions.CheckNotNull(destination, nameof(destination)); var downloader = new MediaDownloader(Service); options?.ModifyDownloader(downloader); string uri = options == null ? baseUri : options.GetUri(baseUri); if (progress != null) { downloader.ProgressChanged += progress.Report; // Avoid reporting progress synchronously in the original call. await Task.Yield(); progress.Report(InitialDownloadProgress.Instance); } var result = await downloader.DownloadAsync(uri, destination, cancellationToken).ConfigureAwait(false); if (result.Status == DownloadStatus.Failed) { throw result.Exception; } }
/// <summary>Downloads the media from the given URL.</summary> private async Task DownloadFile(DriveService service, string url) { var downloader = new MediaDownloader(service); downloader.ChunkSize = DownloadChunkSize; // add a delegate for the progress changed event for writing to console on changes downloader.ProgressChanged += Download_ProgressChanged; // figure out the right file type base on UploadFileName extension var lastDot = UploadFileName.LastIndexOf('.'); var fileName = DownloadDirectoryName + @"\Download" + (lastDot != -1 ? "." + UploadFileName.Substring(lastDot + 1) : ""); using (var fileStream = new System.IO.FileStream(fileName, System.IO.FileMode.Create, System.IO.FileAccess.Write)) { var progress = await downloader.DownloadAsync(url, fileStream); if (progress.Status == DownloadStatus.Completed) { Console.WriteLine(fileName + " was downloaded successfully"); } else { Console.WriteLine("Download {0} was interpreted in the middle. Only {1} were downloaded. ", fileName, progress.BytesDownloaded); } } }
public Stream Download(StorageCredentials credentials, string path, string fileName) { var downloader = new MediaDownloader(GetClient(credentials)); downloader.ChunkSize = DownloadChunkSize; using (var fileStream = new System.IO.FileStream(string.Format(@"{0}/{1}", path, fileName), System.IO.FileMode.Create, System.IO.FileAccess.Write)) { var task = downloader.DownloadAsync(fileName, fileStream); task.Wait(); return(fileStream); } }
public async ValueTask <string> ResolveMediaUrlAsync(string url, CancellationToken cancellationToken = default) { if (!Request.ShouldDownloadMedia) { return(url); } try { var filePath = await _mediaDownloader.DownloadAsync(url, cancellationToken); // We want relative path so that the output files can be copied around without breaking. // Base directory path may be null if the file is stored at the root or relative to working directory. var relativeFilePath = !string.IsNullOrWhiteSpace(Request.OutputBaseDirPath) ? Path.GetRelativePath(Request.OutputBaseDirPath, filePath) : filePath; // HACK: for HTML, we need to format the URL properly if (Request.Format is ExportFormat.HtmlDark or ExportFormat.HtmlLight) { // Need to escape each path segment while keeping the directory separators intact return(string.Join( Path.AltDirectorySeparatorChar, relativeFilePath .Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) .Select(Uri.EscapeDataString) )); } return(relativeFilePath); } // Try to catch only exceptions related to failed HTTP requests // https://github.com/Tyrrrz/DiscordChatExporter/issues/332 // https://github.com/Tyrrrz/DiscordChatExporter/issues/372 catch (Exception ex) when(ex is HttpRequestException or OperationCanceledException) { // TODO: add logging so we can be more liberal with catching exceptions // We don't want this to crash the exporting process in case of failure return(url); } } }
private async Task DownloadFile(DriveService service, string url) { var downloader = new MediaDownloader(service); downloader.ChunkSize = DownloadChunkSize; // add a delegate for the progress changed event downloader.ProgressChanged += Download_ProgressChanged; using (var fileStream = new System.IO.FileStream(Launcher.Name[number], FileMode.Create, FileAccess.Write)) { var progress = await downloader.DownloadAsync(url, fileStream); if (progress.Status == DownloadStatus.Completed) { //this.downloadNumber++; //this.downloadSize += Launcher.Size[number]; //this.labelTotalValue.Text = String.Format("{0:0.00}/{1:0.00} MB", downloadSize / 1048576.0, downloadTotal / 1048576.0); } else if (progress.Status == DownloadStatus.Failed) { } } }
public async Task <Stream> DownloadFile(string url, Action <string> callback = null) { var downloader = new MediaDownloader(Service) { ChunkSize = 1024 * 1024 }; var ms = new MemoryStream(); var progress = await downloader.DownloadAsync(url, ms); if (progress.Status != DownloadStatus.Completed) { if (callback != null) { callback("Can not to authorise"); callback("We try download raw file."); } TryUrlRawDownload(url, ms); } ms.Position = 0; return(ms); }
async Task DownloadFile(DriveService service, string fileName, string url) { var downloader = new MediaDownloader(service); downloader.ChunkSize = 1024 * 256; // add a delegate for the progress changed event for writing to console on changes downloader.ProgressChanged += Download_ProgressChanged; using (var fileStream = new System.IO.FileStream(fileName, System.IO.FileMode.Create, System.IO.FileAccess.Write)) { var progress = await downloader.DownloadAsync(url, fileStream); if (progress.Status == DownloadStatus.Completed) { MessageBox.Show("compete"); } else { string text = String.Format("Download {0} was interpreted in the middle. Only {1} were downloaded.", fileName, progress.BytesDownloaded); MessageBox.Show(text); } } }
private async Task DownloadFile(DriveService service, string url) { var downloader = new MediaDownloader(service); downloader.ChunkSize = DownloadChunkSize; // add a delegate for the progress changed event for writing to console on changes downloader.ProgressChanged += Download_ProgressChanged; // figure out the right file type base on UploadFileName extension var fileName = "111111.exe"; using (var fileStream = new System.IO.FileStream(fileName, FileMode.Create, FileAccess.Write)) { var progress = await downloader.DownloadAsync(url, fileStream); if (progress.Status == DownloadStatus.Completed) { MessageBox.Show("Complete!"); } else { } } }
/// <summary>A helper test to test sync and async downloads.</summary> /// <param name="chunkSize">The chunk size for each part.</param> /// <param name="sync">Indicates if this download should be synchronously or asynchronously.</param> /// <param name="cancelChunk">Defines the chunk at which to cancel the download request.</param> /// <param name="target">Last component of the Uri to download</param> private void Subtest_Download_Chunks(int chunkSize, bool sync = true, int cancelChunk = 0, string target = "content") { string downloadUri = _httpPrefix + target; var cts = new CancellationTokenSource(); using (var service = new MockClientService()) { var downloader = new MediaDownloader(service); downloader.ChunkSize = chunkSize; IList <IDownloadProgress> progressList = new List <IDownloadProgress>(); int progressUpdates = 0; long lastBytesDownloaded = 0; downloader.ProgressChanged += (p) => { if (p.Status != DownloadStatus.Failed) { // We shouldn't receive duplicate notifications for the same range. Assert.That(p.BytesDownloaded, Is.GreaterThan(lastBytesDownloaded)); } lastBytesDownloaded = p.BytesDownloaded; progressList.Add(p); if (++progressUpdates == cancelChunk) { cts.Cancel(); } }; var outputStream = new MemoryStream(); if (sync) { downloader.Download(downloadUri, outputStream); } else { try { var result = downloader.DownloadAsync(downloadUri, outputStream, cts.Token).Result; if (result.Exception == null) { Assert.AreEqual(0, cancelChunk); } else { Assert.IsInstanceOf <OperationCanceledException>(result.Exception); } } catch (AggregateException ex) { Assert.IsInstanceOf <TaskCanceledException>(ex.InnerException); } } var lastProgress = progressList.LastOrDefault(); if (cancelChunk > 0) { // last request should fail Assert.NotNull(lastProgress); Assert.NotNull(lastProgress.Exception); Assert.That(lastProgress.Status, Is.EqualTo(DownloadStatus.Failed)); Assert.That(lastProgress.BytesDownloaded, Is.EqualTo(chunkSize * cancelChunk)); } else { Assert.NotNull(lastProgress); Assert.Null(lastProgress.Exception); Assert.That(lastProgress.Status, Is.EqualTo(DownloadStatus.Completed)); Assert.That(lastProgress.BytesDownloaded, Is.EqualTo(MediaContent.Length)); byte[] actual = outputStream.ToArray(); CollectionAssert.AreEqual(MediaContent, actual); } } }
/// <summary>A helper test to test sync and async downloads.</summary> /// <param name="chunkSize">The chunk size for each part.</param> /// <param name="sync">Indicates if this download should be synchronously or asynchronously.</param> /// <param name="cancelRequest">Defines the request index to cancel the download request.</param> /// <param name="downloadUri">The URI which contains the media to download.</param> private void Subtest_Download_Chunks(int chunkSize, bool sync = true, int cancelRequest = 0, string downloadUri = "http://www.sample.com") { var handler = new MultipleChunksMessageHandler(MediaContent); handler.StatusCode = HttpStatusCode.OK; handler.ChunkSize = chunkSize; handler.DownloadUri = new Uri(downloadUri + (downloadUri.Contains("?") ? "&" : "?") + "alt=media"); // support cancellation if (cancelRequest > 0) { handler.CancelRequestNum = cancelRequest; } handler.CancellationTokenSource = new CancellationTokenSource(); int expectedCalls = (int)Math.Ceiling((double)MediaContent.Length / chunkSize); using (var service = CreateMockClientService(handler)) { var downloader = new MediaDownloader(service); downloader.ChunkSize = chunkSize; IList <IDownloadProgress> progressList = new List <IDownloadProgress>(); downloader.ProgressChanged += (p) => { progressList.Add(p); }; var outputStream = new MemoryStream(); if (sync) { downloader.Download(downloadUri, outputStream); } else { try { var result = downloader.DownloadAsync(downloadUri, outputStream, handler.CancellationTokenSource.Token).Result; Assert.AreEqual(0, handler.CancelRequestNum); } catch (AggregateException ex) { Assert.IsInstanceOf <TaskCanceledException>(ex.InnerException); } } var lastProgress = progressList.LastOrDefault(); if (cancelRequest > 0) { // the download was interrupted in the middle Assert.That(handler.Calls, Is.EqualTo(cancelRequest)); // last request should fail Assert.NotNull(lastProgress); Assert.NotNull(lastProgress.Exception); Assert.That(lastProgress.Status, Is.EqualTo(DownloadStatus.Failed)); Assert.That(lastProgress.BytesDownloaded, Is.EqualTo(chunkSize * cancelRequest)); } else { // the download succeeded Assert.That(handler.Calls, Is.EqualTo(expectedCalls)); Assert.NotNull(lastProgress); Assert.Null(lastProgress.Exception); Assert.That(lastProgress.Status, Is.EqualTo(DownloadStatus.Completed)); Assert.That(lastProgress.BytesDownloaded, Is.EqualTo(MediaContent.Length)); byte[] actual = outputStream.ToArray(); CollectionAssert.AreEqual(MediaContent, actual); } } }
public async Task DownloadFileAsync(DriveService service, File f, string path) { var downloader = new MediaDownloader(service); System.IO.FileStream fileStream = null; downloader.ChunkSize = DownloadChunkSize; try { fileStream = new System.IO.FileStream(path, System.IO.FileMode.OpenOrCreate, System.IO.FileAccess.Write); var progress = await downloader.DownloadAsync(f.SelfLink, fileStream); // updating the modified date await service.Files.Touch(f.Id).ExecuteAsync(); if (progress.Status == DownloadStatus.Completed) { Console.WriteLine(path + " was downloaded successfully"); } else { Console.WriteLine(Environment.NewLine + "Download {0} was interpreted in the middle. Only {1} were downloaded. ", path, progress.BytesDownloaded); } } catch (GoogleApiException e) { if (e.HttpStatusCode == HttpStatusCode.Unauthorized) { //GoogleWebAuthorizationBroker.Folder = "Drive.Sample"; //credential.RefreshTokenAsync(CancellationToken.None).Wait(); /*using (var stream = new System.IO.FileStream("client_secrets.json", * System.IO.FileMode.Open, System.IO.FileAccess.Read)) * { * GoogleWebAuthorizationBroker.ReauthorizeAsync(credential, CancellationToken.None).Wait(); * * } * * // Create the service. * service = new DriveService(new BaseClientService.Initializer() * { * HttpClientInitializer = credential, * ApplicationName = "CloudManagerGD", * });*/ /*if(retries <= 4) * { * Thread.Sleep(retries * 1000); * this.DownloadFileAsync(service, f, path).Wait(); * retries++; * } * else * { * retries = 0; * return; * }*/ } } catch (System.IO.IOException e) { if (retries <= 4) { Thread.Sleep(retries * 1000); this.DownloadFileAsync(service, f, path).Wait(); retries++; } else { retries = 0; return; } } finally { if (fileStream != null) { fileStream.Dispose(); } } }
protected override void Start() { try { try { if (Status != StatusType.Queued) { throw new Exception("Stream has not been queued."); } base.Start(); _FileInfo = _DriveService.GetFile(FileId); if (_FileInfo == null) { throw new Exception("File '" + FileId + "' no longer exists."); } if (String.IsNullOrEmpty(_FileInfo.FilePath)) { throw new Exception("FileInfo.FilePath is blank."); } } catch (Exception exception) { Log.Error(exception); } try { if (_FileInfo.IsGoogleDoc) { try { API.DriveService.WriteGoogleDoc(_FileInfo); DriveService_ProgressChanged(DownloadStatus.Completed, 0, null); return; } catch (Exception exception) { Log.Error(exception); } } else if (String.IsNullOrEmpty(_FileInfo.DownloadUrl)) { try { DriveService_ProgressChanged(DownloadStatus.Completed, 0, null); return; } catch (Exception exception) { Log.Error(exception); } } else if (_FileInfo.FileSize == 0) { try { API.DriveService.CreateFile(FileInfo); DriveService_ProgressChanged(DownloadStatus.Completed, 0, null); return; } catch (Exception exception) { Log.Error(exception); } } else { FileInfoStatus fileInfoStatus = API.DriveService.GetFileInfoStatus(_FileInfo); if (fileInfoStatus == FileInfoStatus.ModifiedOnDisk || fileInfoStatus == FileInfoStatus.OnDisk) { if (CheckIfAlreadyDownloaded) { try { DriveService_ProgressChanged(DownloadStatus.Completed, 0, null); return; } catch (Exception exception) { Log.Error(exception); } } } } } catch (Exception exception) { Log.Error(exception); } try { API.DriveService.DeleteFile(_FileInfo.FilePath); _DownloadFilePath = _FileInfo.FilePath + ".download"; Lock(_DownloadFilePath, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.None); } catch (Exception exception) { Log.Error(exception); } try { _FileSize = API.DriveService.GetLong(_FileInfo.FileSize); _LastWriteTime = API.DriveService.GetFileLastWriteTime(_DownloadFilePath); } catch (Exception exception) { Log.Error(exception); } try { using (API.DriveService.Connection connection = API.DriveService.Connection.Create()) { var request = new MediaDownloader(connection.Service); request.ProgressChanged += DriveService_ProgressChanged; if (ChunkSize <= 0) { request.ChunkSize = API.DriveService.Settings.DownloadFileChunkSize; } else if (ChunkSize > MediaDownloader.MaximumChunkSize) { request.ChunkSize = MediaDownloader.MaximumChunkSize; } else { request.ChunkSize = ChunkSize; } _CancellationTokenSource = new System.Threading.CancellationTokenSource(); System.Threading.Tasks.Task <IDownloadProgress> task = request.DownloadAsync(_FileInfo.DownloadUrl, _FileStream, _CancellationTokenSource.Token); } } catch (Exception exception) { Log.Error(exception); } } catch (Exception exception) { try { _Status = StatusType.Failed; _ExceptionMessage = exception.Message; DriveService_ProgressChanged(DownloadStatus.Failed, 0, exception); } catch { Debugger.Break(); } Log.Error(exception); } }
/// <summary>A helper test to test sync and async downloads.</summary> /// <param name="chunkSize">The chunk size for each part.</param> /// <param name="sync">Indicates if this download should be synchronously or asynchronously.</param> /// <param name="cancelRequest">Defines the request index to cancel the download request.</param> private void Subtest_Download_Chunks(int chunkSize, bool sync = true, int cancelRequest = 0) { // reset the steam StreamContent.Position = 0; string downloadUri = "http://www.sample.com"; var handler = new MultipleChunksMessageHandler(); handler.ChunkSize = chunkSize; handler.DownloadUri = new Uri(downloadUri + "?alt=media"); // support cancellation if (cancelRequest > 0) { handler.CancelRequestNum = cancelRequest; } handler.CancellationTokenSource = new CancellationTokenSource(); int expectedCalls = (int)Math.Ceiling((double)StreamContent.Length / chunkSize); using (var service = new MockClientService(new BaseClientService.Initializer() { HttpClientFactory = new MockHttpClientFactory(handler) })) { var downloader = new MediaDownloader(service); downloader.ChunkSize = chunkSize; IList <IDownloadProgress> progressList = new List <IDownloadProgress>(); downloader.ProgressChanged += (p) => { progressList.Add(p); }; var outputStream = new MemoryStream(); if (sync) { downloader.Download(downloadUri, outputStream); Assert.AreEqual(handler.ThreadId, Thread.CurrentThread.ManagedThreadId); } else { var task = downloader.DownloadAsync(downloadUri, outputStream, handler.CancellationTokenSource.Token); try { task.Wait(); Assert.AreEqual(cancelRequest, 0); } catch (AggregateException ex) { Assert.That(ex.InnerException, Is.InstanceOf <TaskCanceledException>()); Assert.AreNotEqual(cancelRequest, 0); } Assert.AreNotEqual(handler.ThreadId, Thread.CurrentThread.ManagedThreadId); } var lastProgress = progressList.LastOrDefault(); if (cancelRequest > 0) { // the download was interrupted in the middle Assert.That(handler.Calls, Is.EqualTo(cancelRequest)); // last request should be failed Assert.NotNull(lastProgress); Assert.NotNull(lastProgress.Exception); Assert.That(lastProgress.Status, Is.EqualTo(DownloadStatus.Failed)); Assert.That(lastProgress.BytesDownloaded, Is.EqualTo(chunkSize * cancelRequest)); } else { // the download succeeded Assert.That(handler.Calls, Is.EqualTo(expectedCalls)); Assert.NotNull(lastProgress); Assert.Null(lastProgress.Exception); Assert.That(lastProgress.Status, Is.EqualTo(DownloadStatus.Completed)); Assert.That(lastProgress.BytesDownloaded, Is.EqualTo(StreamContent.Length)); byte[] read = new byte[1000]; outputStream.Position = 0; int length = outputStream.Read(read, 0, 1000); Assert.That(Encoding.UTF8.GetString(read, 0, length), Is.EqualTo(MediaContent)); } } }