private async Task <SyncResults> SyncFiles(SyncResultsBuilder resultsBuilder)
{
_logger.LogInformation($"Sync files starting - {_filesToDownload.Count} files to be downloaded");
// To increase the download speed we will download multiple files at once in pages
var page = GetNextDownloadPage();
while (page.downloads != null && page.downloads.Any())
{
_logger.LogInformation($"Starting download for {page.downloads.Count()} files");
var downloaders = new List <Task>();
foreach (var download in page.downloads)
{
downloaders.Add(Task.Run(() =>
{
DownloadFile(download, resultsBuilder);
}));
}
await Task.WhenAll(downloaders);
page = GetNextDownloadPage(page.nextPage);
}
return(resultsBuilder.Build());
}
public SyncResults(SyncResultsBuilder builder)
{
SuccessfulDownloads = builder.SuccessfulDownloads.AsReadOnly();
ErroredFiles = builder.ErroredFiles.AsReadOnly();
TotalDownloads = SuccessfulDownloads.Count + ErroredFiles.Count;
}
public async Task Go()
{
StartTime = DateTime.Now;
Authorize();
CreateService();
SanatizeDestinationPath();
// Start finding files to sync and track progress
// First call to FindFilesToSync should include the top level folder Id from config
var resultsBuilder = new SyncResultsBuilder();
FindFilesToSync(resultsBuilder, Config.GoogleDriveFolderId);
var results = await SyncFiles(resultsBuilder);
// Once finished display a summary of the results
EndTime = DateTime.Now;
DisplayResults(results);
}
private void CheckForLocalFile(GFile gFile, SyncResultsBuilder results)
{
var fullPath = GetFileFolderPath(gFile);
_logger.LogInformation($"New File \"{fullPath}\"");
var localPath = Path.Combine(Config.DestinationPath, fullPath);
var downloadFile = new DownloadableFile(gFile, localPath);
try
{
if (LFile.Exists(localPath))
{
_logger.LogInformation(" File exists");
if (Config.ForceDownloads)
{
_logger.LogInformation($" Force downloads enabled, will download to: {localPath}");
_filesToDownload.Add(downloadFile);
return;
}
var lFileInfo = new FileInfo(localPath);
if (lFileInfo.LastWriteTimeUtc <= (gFile.ModifiedTime ?? gFile.CreatedTime))
{
_logger.LogInformation($" Google file is newer, will download to: {localPath}");
_filesToDownload.Add(downloadFile);
}
}
else
{
// File doesn't exist, download it
_logger.LogInformation($" File does not exist, downloading to: {localPath}");
_filesToDownload.Add(downloadFile);
}
}
catch (Exception exception)
{
var error = downloadFile.ToErroredFile(exception);
results.ErroredFiles.Add(error);
}
}
private void FindFilesToSync(SyncResultsBuilder results, string folderId)
{
if (string.IsNullOrWhiteSpace(folderId))
{
return;
}
_logger.LogInformation($"Finding files in folder with id {folderId}");
var filePage = GetNextGoogleFilePage(folderId, null);
while (filePage?.Files != null && filePage.Files.Count > 0)
{
// Loop the files in this page
foreach (var gFile in filePage.Files)
{
// If this is a folder ignore it as we can't "sync" a folder (but it will be created later when we get the full path)
// save it to the file cache for reference later when getting parent path
if (gFile.MimeType == SyncConstants.FolderMimeType)
{
_fileCache.TryAdd(gFile.Id, gFile);
// Recusrively call FileFilesToSync agains the folder we just found to get the files in this folder
FindFilesToSync(results, gFile.Id);
continue;
}
// Non-folder files should be checked against the local destination for downloading
CheckForLocalFile(gFile, results);
}
// Get the next page or stop the loop
if (string.IsNullOrWhiteSpace(filePage.NextPageToken))
{
filePage = null;
}
else
{
filePage = GetNextGoogleFilePage(folderId, filePage.NextPageToken);
}
}
}
private void DownloadFile(DownloadableFile download, SyncResultsBuilder results)
{
_logger.LogInformation($"[dl: {download.GFile.Name}] download starting at {download.Destination}");
var sanitizedDestination = SanatizePath(download.Destination);
if (sanitizedDestination != download.Destination)
{
_logger.LogError($"[dl: {download.GFile.Name}] Warning: Invalid Destination path. Will use \"{sanitizedDestination}\" instead");
}
// Create the directorys (if they dont exist) to avoid exception when making file
var folderPath = Path.GetDirectoryName(sanitizedDestination);
Directory.CreateDirectory(folderPath);
// Create the file in a temp location to avoid overwriting the file and corrupting if the download is broken
var tempFile = sanitizedDestination + ".temp";
try
{
using (var stream = new FileStream(tempFile, FileMode.Create))
{
var request = _service.Files.Get(download.GFile.Id);
// Add a handler which will be notified on progress changes.
// It will notify on each chunk download and when the
// download is completed or failed.
request.MediaDownloader.ProgressChanged +=
(IDownloadProgress progress) =>
{
switch (progress.Status)
{
case DownloadStatus.Downloading:
{
string percent;
if (download.GFile.Size.HasValue)
{
percent = $"{((100 * progress.BytesDownloaded) / download.GFile.Size.Value)}%";
}
else
{
percent = "...";
}
_logger.LogInformation($"[dl: {download.GFile.Name}] {percent}");
break;
}
case DownloadStatus.Completed:
{
_logger.LogInformation($"[dl: {download.GFile.Name}] Download complete.");
results.SuccessfulDownloads.Add(download);
break;
}
case DownloadStatus.Failed:
{
_logger.LogError($"[dl: {download.GFile.Name}] Download failed. Exception: {progress.Exception.Message}");
results.ErroredFiles.Add(download.ToErroredFile(progress.Exception));
break;
}
}
};
request.DownloadWithStatus(stream);
stream.Dispose();
// Finished downloading, delete the original and rename the temp to the original
LFile.Delete(sanitizedDestination);
LFile.Move(tempFile, sanitizedDestination);
}
}
catch (Exception exception)
{
_logger.LogError($"[dl: {download?.GFile?.Name}] Download failed. Exception: {exception.Message}");
_logger.LogError(exception.Message);
if (download?.GFile != null)
{
var errorFile = download.ToErroredFile(exception);
results.ErroredFiles.Add(errorFile);
}
}
}