/// <summary> /// Downloads meta-information about files. /// </summary> /// <param name="service">Google Drive service that is used to download files metainformation.</param> /// <returns>A list of meta-information for all files on drive.</returns> /// <exception cref="OperationCanceledException">The token has had cancellation requested.</exception> public async Task <IEnumerable <File> > GetFilesInfo(DriveService service) { this.Log.LogMessage("Downloading files info..."); var listRequest = service.Files.List(); listRequest.PageSize = 1000; listRequest.Spaces = "drive"; var filesList = new List <File>(); try { this.Status = Status.ListingFiles; var listResult = await GoogleRequestHelper.Execute(listRequest.ExecuteAsync, this.CancellationToken); ++this.Progress; filesList.AddRange(listResult.Files); while (!string.IsNullOrEmpty(listResult.NextPageToken)) { ++this.Estimation; this.CancellationToken.ThrowIfCancellationRequested(); listRequest.PageToken = listResult.NextPageToken; listResult = await GoogleRequestHelper.Execute(listRequest.ExecuteAsync, this.CancellationToken); ++this.Progress; filesList.AddRange(listResult.Files); } } catch (Exception e) { if (this.CancellationToken.IsCancellationRequested) { throw; } this.Log.LogError("Files listing error.", e); this.Error(); return(new List <File>()); } this.Log.LogMessage($"Listed {filesList.Count} files."); this.Estimation += filesList.Count; var semaphore = new AsyncSemaphore(Utils.Constants.DownloadThreadsCount); this.Status = Status.DownloadingMetaInformation; var result = await AsyncHelper.ProcessInChunks( filesList, f => this.DownloadFileMetaInformation(service, f, semaphore), Utils.Constants.TaskChunkSize, this.CancellationToken); this.Done(); return(result); }
/// <summary> /// Creates Google Drive service object with correct credentials. /// </summary> /// <param name="refreshToken">Refresh Token to use.</param> /// <returns>Google Drive service object.</returns> private async Task <DriveService> CreateService(string refreshToken) { var initializer = new AuthorizationCodeFlow.Initializer( "https://accounts.google.com/o/oauth2/auth", "https://accounts.google.com/o/oauth2/token") { ClientSecrets = new ClientSecrets { ClientId = Constants.GoogleDriveSyncClientId, ClientSecret = Constants.GoogleDriveSyncClientSecret } }; // Hardcoded Google Drive Sync secret. string[] scopes = { DriveService.Scope.DriveReadonly }; initializer.Scopes = scopes; using (var codeFlow = new AuthorizationCodeFlow(initializer)) { try { var token = await GoogleRequestHelper.Execute( ct => codeFlow.RefreshTokenAsync("user", refreshToken, ct), this.CancellationToken); var credential = new UserCredential(codeFlow, "user", token); var service = new DriveService( new BaseClientService.Initializer { HttpClientInitializer = credential, ApplicationName = Utils.Constants.ApplicationName }); return(service); } catch (Exception e) { if (this.CancellationToken.IsCancellationRequested) { this.Log.LogMessage("Operation was cancelled."); throw; } this.StatusAccumulator.FailureOccurred(); this.Log.LogError("Authorization error.", e); throw; } } }
/// <summary> /// Downloads information about user and account. /// </summary> /// <param name="service">Google Drive service used to download about info.</param> /// <param name="taskContext">Provides context for task execution, like logger object and /// cancellation token.</param> /// <returns>Information about user and account.</returns> public async Task <About> Run(DriveService service, ITaskContext taskContext) { taskContext.Log.LogMessage("Downloading user account info..."); try { var aboutRequest = service.About.Get(); aboutRequest.Fields = "*"; return(await GoogleRequestHelper.Execute(aboutRequest.ExecuteAsync, taskContext.CancellationToken)); } catch (Exception e) { if (taskContext.CancellationToken.IsCancellationRequested) { throw; } taskContext.Fail("User account info download failed.", e); return(null); } }
/// <summary> /// Asynchronously downloads meta-information for one file. /// </summary> /// <param name="service">Google Drive service that is used to download files metainformation.</param> /// <param name="file">File with filled Id field.</param> /// <param name="semaphore">Semaphore used to throttle downloading process.</param> /// <returns>Asynchronous task that downloads meta-information.</returns> private async Task <File> DownloadFileMetaInformation(DriveService service, File file, AsyncSemaphore semaphore) { var fileInfoRequest = service.Files.Get(file.Id); fileInfoRequest.Fields = "*"; await semaphore.WaitAsync(); try { var fileInfo = await GoogleRequestHelper.Execute( fileInfoRequest.ExecuteAsync, this.CancellationToken, GoogleRequestHelper.Cooldown *Utils.Constants.DownloadThreadsCount); ++this.Progress; this.StatusAccumulator.SuccessItem(); return(fileInfo); } catch (Exception e) { if (this.CancellationToken.IsCancellationRequested) { throw; } this.Log.LogError($"Downloading meta-information for file {file.Name} failed.", e); this.StatusAccumulator.FailureOccurred(); throw; } finally { semaphore.Release(); } }
/// <summary> /// Downloads Google Docs file. /// </summary> /// <param name="file">Meta-information about file to download.</param> /// <param name="context">Provides context for task execution, like logger object and /// cancellation token.</param> /// <returns>An information about downloaded file.</returns> public async Task <DownloadedFile> Run(File file, ITaskContext context) { if (!this.exportFormats.ContainsKey(file.MimeType) || (this.exportFormats[file.MimeType].Count == 0)) { context.Log.LogError($"Unknown export format for file MIME type {file.MimeType}."); return(null); } // Just take first available export format for now. var exportFormat = this.exportFormats[file.MimeType][0]; var exportRequest = this.service.Files.Export(file.Id, exportFormat); await this.semaphore.WaitAsync(); context.Log.LogDebugMessage( $"Starting to download '{file.Name}' Google Docs file, export format is {exportFormat}."); try { var resultStream = await GoogleRequestHelper.Execute( exportRequest.ExecuteAsStreamAsync, context.CancellationToken, GoogleRequestHelper.Cooldown *Utils.Constants.DownloadThreadsCount); var mimeParts = exportFormat.Split('/', '+'); var extension = mimeParts[mimeParts.Length - 1]; switch (extension) { case "vnd.openxmlformats-officedocument.presentationml.presentation": extension = "pptx"; break; case "xml": extension = "svg"; break; } var fileName = FileDownloadUtils.GetFileName(file, this.directories) + "." + extension; using (var fileStream = new FileStream(fileName, FileMode.Create)) { await resultStream.CopyToAsync(fileStream, 4096, context.CancellationToken); } var fileInfo = FileDownloadUtils.CorrectFileTimes(file, fileName); context.Log.LogDebugMessage( $"File '{file.Name}' downloading finished, saved as '{fileInfo.FullName}'."); return(new DownloadedFile(file, fileInfo.FullName)); } catch (Exception e) { if (context.CancellationToken.IsCancellationRequested) { throw; } context.Fail($"File '{file.Name}' downloading failed.", e); throw; } finally { this.semaphore.Release(); } }
/// <summary> /// Downloads binary (non-"Google Docs") file. /// </summary> /// <param name="service">Google Drive service used to download a file.</param> /// <param name="file">Meta-information about file to download.</param> /// <param name="semaphore">Semaphore used to throttle downloading process.</param> /// <param name="unitsOfWork">Abstract units of work assigned to download this file. Used in progress reporting.</param> /// <param name="directories">Dictionary that contains possible directories to save a file.</param> /// <returns>Information about downloaded file, null if downloading failed.</returns> /// <exception cref="InternalErrorException">Something is wrong with downloader code itself.</exception> public async Task <DownloadedFile> DownloadBinaryFile( DriveService service, File file, AsyncSemaphore semaphore, int unitsOfWork, IDictionary <File, IEnumerable <DirectoryInfo> > directories) { this.Info = file.Name; this.Estimation = unitsOfWork; if (file.Size.HasValue && (file.Size.Value == 0)) { try { var fileName = FileDownloadUtils.GetFileName(file, directories); System.IO.File.Create(fileName); this.Done(); return(new DownloadedFile(file, fileName)); } catch (Exception e) { this.Log.LogError($"Saving zero-length file '{file.Name}' error.", e); this.StatusAccumulator.FailureOccurred(); throw; } } var request = service.Files.Get(file.Id); request.MediaDownloader.ProgressChanged += downloadProgress => { switch (downloadProgress.Status) { case DownloadStatus.NotStarted: break; case DownloadStatus.Downloading: this.Progress = (int)(downloadProgress.BytesDownloaded * unitsOfWork / (file.Size ?? 1)); break; case DownloadStatus.Completed: this.Progress = this.Estimation; break; case DownloadStatus.Failed: this.Progress = this.Estimation; if (!this.CancellationToken.IsCancellationRequested) { this.Status = Status.Error; this.Log.LogError($"Downloading file '{file.Name}' error.", downloadProgress.Exception); this.StatusAccumulator.FailureOccurred(); } break; default: throw new InternalErrorException("DownloadStatus enum contains unknown value."); } }; await semaphore.WaitAsync(); this.Log.LogDebugMessage($"Starting to download '{file.Name}' binary file."); try { var fileName = FileDownloadUtils.GetFileName(file, directories); using (var fileStream = new FileStream(fileName, FileMode.Create)) { await GoogleRequestHelper.Execute( ct => request.DownloadAsync(fileStream, ct), this.CancellationToken, GoogleRequestHelper.Cooldown *Utils.Constants.DownloadThreadsCount); } var fileInfo = FileDownloadUtils.CorrectFileTimes(file, fileName); this.Log.LogDebugMessage($"File '{file.Name}' downloading finished, saved as {fileInfo.FullName}."); this.StatusAccumulator.SuccessItem(); this.Done(); return(new DownloadedFile(file, fileInfo.FullName)); } catch (Exception e) { if (this.CancellationToken.IsCancellationRequested) { throw; } this.Log.LogError($"Downloading file '{file.Name}' error.", e); this.StatusAccumulator.FailureOccurred(); throw; } finally { semaphore.Release(); } }