/// <summary>
/// Downloads meta-information about files.
/// </summary>
/// <param name="service">Google Drive service that is used to download files metainformation.</param>
/// <returns>A list of meta-information for all files on drive.</returns>
/// <exception cref="OperationCanceledException">The token has had cancellation requested.</exception>
public async Task <IEnumerable <File> > GetFilesInfo(DriveService service)
{
this.Log.LogMessage("Downloading files info...");
var listRequest = service.Files.List();
listRequest.PageSize = 1000;
listRequest.Spaces = "drive";
var filesList = new List <File>();
try
{
this.Status = Status.ListingFiles;
var listResult = await GoogleRequestHelper.Execute(listRequest.ExecuteAsync, this.CancellationToken);
++this.Progress;
filesList.AddRange(listResult.Files);
while (!string.IsNullOrEmpty(listResult.NextPageToken))
{
++this.Estimation;
this.CancellationToken.ThrowIfCancellationRequested();
listRequest.PageToken = listResult.NextPageToken;
listResult = await GoogleRequestHelper.Execute(listRequest.ExecuteAsync, this.CancellationToken);
++this.Progress;
filesList.AddRange(listResult.Files);
}
}
catch (Exception e)
{
if (this.CancellationToken.IsCancellationRequested)
{
throw;
}
this.Log.LogError("Files listing error.", e);
this.Error();
return(new List <File>());
}
this.Log.LogMessage($"Listed {filesList.Count} files.");
this.Estimation += filesList.Count;
var semaphore = new AsyncSemaphore(Utils.Constants.DownloadThreadsCount);
this.Status = Status.DownloadingMetaInformation;
var result = await AsyncHelper.ProcessInChunks(
filesList,
f => this.DownloadFileMetaInformation(service, f, semaphore),
Utils.Constants.TaskChunkSize,
this.CancellationToken);
this.Done();
return(result);
}
/// <summary>
/// Creates Google Drive service object with correct credentials.
/// </summary>
/// <param name="refreshToken">Refresh Token to use.</param>
/// <returns>Google Drive service object.</returns>
private async Task <DriveService> CreateService(string refreshToken)
{
var initializer =
new AuthorizationCodeFlow.Initializer(
"https://accounts.google.com/o/oauth2/auth",
"https://accounts.google.com/o/oauth2/token")
{
ClientSecrets =
new ClientSecrets
{
ClientId = Constants.GoogleDriveSyncClientId,
ClientSecret = Constants.GoogleDriveSyncClientSecret
}
};
// Hardcoded Google Drive Sync secret.
string[] scopes = { DriveService.Scope.DriveReadonly };
initializer.Scopes = scopes;
using (var codeFlow = new AuthorizationCodeFlow(initializer))
{
try
{
var token = await GoogleRequestHelper.Execute(
ct => codeFlow.RefreshTokenAsync("user", refreshToken, ct),
this.CancellationToken);
var credential = new UserCredential(codeFlow, "user", token);
var service =
new DriveService(
new BaseClientService.Initializer
{
HttpClientInitializer = credential,
ApplicationName = Utils.Constants.ApplicationName
});
return(service);
}
catch (Exception e)
{
if (this.CancellationToken.IsCancellationRequested)
{
this.Log.LogMessage("Operation was cancelled.");
throw;
}
this.StatusAccumulator.FailureOccurred();
this.Log.LogError("Authorization error.", e);
throw;
}
}
}
/// <summary>
/// Downloads information about user and account.
/// </summary>
/// <param name="service">Google Drive service used to download about info.</param>
/// <param name="taskContext">Provides context for task execution, like logger object and
/// cancellation token.</param>
/// <returns>Information about user and account.</returns>
public async Task <About> Run(DriveService service, ITaskContext taskContext)
{
taskContext.Log.LogMessage("Downloading user account info...");
try
{
var aboutRequest = service.About.Get();
aboutRequest.Fields = "*";
return(await GoogleRequestHelper.Execute(aboutRequest.ExecuteAsync, taskContext.CancellationToken));
}
catch (Exception e)
{
if (taskContext.CancellationToken.IsCancellationRequested)
{
throw;
}
taskContext.Fail("User account info download failed.", e);
return(null);
}
}
/// <summary>
/// Asynchronously downloads meta-information for one file.
/// </summary>
/// <param name="service">Google Drive service that is used to download files metainformation.</param>
/// <param name="file">File with filled Id field.</param>
/// <param name="semaphore">Semaphore used to throttle downloading process.</param>
/// <returns>Asynchronous task that downloads meta-information.</returns>
private async Task <File> DownloadFileMetaInformation(DriveService service, File file, AsyncSemaphore semaphore)
{
var fileInfoRequest = service.Files.Get(file.Id);
fileInfoRequest.Fields = "*";
await semaphore.WaitAsync();
try
{
var fileInfo = await GoogleRequestHelper.Execute(
fileInfoRequest.ExecuteAsync,
this.CancellationToken,
GoogleRequestHelper.Cooldown *Utils.Constants.DownloadThreadsCount);
++this.Progress;
this.StatusAccumulator.SuccessItem();
return(fileInfo);
}
catch (Exception e)
{
if (this.CancellationToken.IsCancellationRequested)
{
throw;
}
this.Log.LogError($"Downloading meta-information for file {file.Name} failed.", e);
this.StatusAccumulator.FailureOccurred();
throw;
}
finally
{
semaphore.Release();
}
}
/// <summary>
/// Downloads Google Docs file.
/// </summary>
/// <param name="file">Meta-information about file to download.</param>
/// <param name="context">Provides context for task execution, like logger object and
/// cancellation token.</param>
/// <returns>An information about downloaded file.</returns>
public async Task <DownloadedFile> Run(File file, ITaskContext context)
{
if (!this.exportFormats.ContainsKey(file.MimeType) || (this.exportFormats[file.MimeType].Count == 0))
{
context.Log.LogError($"Unknown export format for file MIME type {file.MimeType}.");
return(null);
}
// Just take first available export format for now.
var exportFormat = this.exportFormats[file.MimeType][0];
var exportRequest = this.service.Files.Export(file.Id, exportFormat);
await this.semaphore.WaitAsync();
context.Log.LogDebugMessage(
$"Starting to download '{file.Name}' Google Docs file, export format is {exportFormat}.");
try
{
var resultStream = await GoogleRequestHelper.Execute(
exportRequest.ExecuteAsStreamAsync,
context.CancellationToken,
GoogleRequestHelper.Cooldown *Utils.Constants.DownloadThreadsCount);
var mimeParts = exportFormat.Split('/', '+');
var extension = mimeParts[mimeParts.Length - 1];
switch (extension)
{
case "vnd.openxmlformats-officedocument.presentationml.presentation":
extension = "pptx";
break;
case "xml":
extension = "svg";
break;
}
var fileName = FileDownloadUtils.GetFileName(file, this.directories) + "." + extension;
using (var fileStream = new FileStream(fileName, FileMode.Create))
{
await resultStream.CopyToAsync(fileStream, 4096, context.CancellationToken);
}
var fileInfo = FileDownloadUtils.CorrectFileTimes(file, fileName);
context.Log.LogDebugMessage(
$"File '{file.Name}' downloading finished, saved as '{fileInfo.FullName}'.");
return(new DownloadedFile(file, fileInfo.FullName));
}
catch (Exception e)
{
if (context.CancellationToken.IsCancellationRequested)
{
throw;
}
context.Fail($"File '{file.Name}' downloading failed.", e);
throw;
}
finally
{
this.semaphore.Release();
}
}
/// <summary>
/// Downloads binary (non-"Google Docs") file.
/// </summary>
/// <param name="service">Google Drive service used to download a file.</param>
/// <param name="file">Meta-information about file to download.</param>
/// <param name="semaphore">Semaphore used to throttle downloading process.</param>
/// <param name="unitsOfWork">Abstract units of work assigned to download this file. Used in progress reporting.</param>
/// <param name="directories">Dictionary that contains possible directories to save a file.</param>
/// <returns>Information about downloaded file, null if downloading failed.</returns>
/// <exception cref="InternalErrorException">Something is wrong with downloader code itself.</exception>
public async Task <DownloadedFile> DownloadBinaryFile(
DriveService service,
File file,
AsyncSemaphore semaphore,
int unitsOfWork,
IDictionary <File, IEnumerable <DirectoryInfo> > directories)
{
this.Info = file.Name;
this.Estimation = unitsOfWork;
if (file.Size.HasValue && (file.Size.Value == 0))
{
try
{
var fileName = FileDownloadUtils.GetFileName(file, directories);
System.IO.File.Create(fileName);
this.Done();
return(new DownloadedFile(file, fileName));
}
catch (Exception e)
{
this.Log.LogError($"Saving zero-length file '{file.Name}' error.", e);
this.StatusAccumulator.FailureOccurred();
throw;
}
}
var request = service.Files.Get(file.Id);
request.MediaDownloader.ProgressChanged += downloadProgress =>
{
switch (downloadProgress.Status)
{
case DownloadStatus.NotStarted:
break;
case DownloadStatus.Downloading:
this.Progress = (int)(downloadProgress.BytesDownloaded * unitsOfWork / (file.Size ?? 1));
break;
case DownloadStatus.Completed:
this.Progress = this.Estimation;
break;
case DownloadStatus.Failed:
this.Progress = this.Estimation;
if (!this.CancellationToken.IsCancellationRequested)
{
this.Status = Status.Error;
this.Log.LogError($"Downloading file '{file.Name}' error.", downloadProgress.Exception);
this.StatusAccumulator.FailureOccurred();
}
break;
default:
throw new InternalErrorException("DownloadStatus enum contains unknown value.");
}
};
await semaphore.WaitAsync();
this.Log.LogDebugMessage($"Starting to download '{file.Name}' binary file.");
try
{
var fileName = FileDownloadUtils.GetFileName(file, directories);
using (var fileStream = new FileStream(fileName, FileMode.Create))
{
await GoogleRequestHelper.Execute(
ct => request.DownloadAsync(fileStream, ct),
this.CancellationToken,
GoogleRequestHelper.Cooldown *Utils.Constants.DownloadThreadsCount);
}
var fileInfo = FileDownloadUtils.CorrectFileTimes(file, fileName);
this.Log.LogDebugMessage($"File '{file.Name}' downloading finished, saved as {fileInfo.FullName}.");
this.StatusAccumulator.SuccessItem();
this.Done();
return(new DownloadedFile(file, fileInfo.FullName));
}
catch (Exception e)
{
if (this.CancellationToken.IsCancellationRequested)
{
throw;
}
this.Log.LogError($"Downloading file '{file.Name}' error.", e);
this.StatusAccumulator.FailureOccurred();
throw;
}
finally
{
semaphore.Release();
}
}