public async Task DownloadProductFileAsync(long id, string title, string sourceUri, string destination, IStatus status)
{
if (string.IsNullOrEmpty(sourceUri))
{
return;
}
var sourceUriSansSession = formatUriRemoveSessionDelegate.Format(sourceUri);
var destinationUri = formatValidationFileDelegate.Format(sourceUriSansSession);
// return early if validation is not expected for this file
if (!confirmValidationExpectedDelegate.Confirm(sourceUriSansSession))
{
return;
}
if (fileController.Exists(destinationUri))
{
await statusController.InformAsync(status, "Validation file already exists, will not be redownloading");
return;
}
var validationSourceUri = formatValidationUriDelegate.Format(sourceUri);
var downloadValidationFileTask = await statusController.CreateAsync(status, "Download validation file");
await downloadFromUriAsyncDelegate.DownloadFromUriAsync(
validationSourceUri,
validationDirectoryDelegate.GetDirectory(string.Empty),
downloadValidationFileTask);
await statusController.CompleteAsync(downloadValidationFileTask);
}
public string Format(string sourceUri)
{
var sourceUriSansSession = formatUriRemoveSessionDelegate.Format(sourceUri);
var sourceFilename = Path.GetFileName(sourceUriSansSession);
var validationFilename = getValidationFilenameDelegate.GetFilename(sourceFilename);
return(sourceUri.Replace(sourceFilename, validationFilename));
}
public async Task <IDictionary <long, IList <string> > > GetDownloadSourcesAsync(IStatus status)
{
var processUpdatesTask = await statusController.CreateAsync(status, "Process screenshots updates");
var screenshotsSources = new Dictionary <long, IList <string> >();
var current = 0;
var total = await screenshotsDataController.CountAsync(processUpdatesTask);
var processProductsScreenshotsTask = await statusController.CreateAsync(processUpdatesTask, "Process product screenshots");
foreach (var id in await screenshotsDataController.ItemizeAllAsync(processProductsScreenshotsTask))
{
var productScreenshots = await screenshotsDataController.GetByIdAsync(id, processProductsScreenshotsTask);
if (productScreenshots == null)
{
await statusController.WarnAsync(processProductsScreenshotsTask, $"Product {id} doesn't have screenshots");
continue;
}
await statusController.UpdateProgressAsync(
processUpdatesTask,
++current,
total,
productScreenshots.Title);
var currentProductScreenshotSources = new List <string>();
foreach (var uri in productScreenshots.Uris)
{
var sourceUri = formatScreenshotsUriDelegate.Format(uri);
var destinationUri = Path.Combine(
screenshotsDirectoryDelegate.GetDirectory(string.Empty),
Path.GetFileName(sourceUri));
if (fileController.Exists(destinationUri))
{
continue;
}
currentProductScreenshotSources.Add(sourceUri);
}
if (currentProductScreenshotSources.Any())
{
screenshotsSources.Add(id, currentProductScreenshotSources);
}
}
await statusController.CompleteAsync(processProductsScreenshotsTask);
await statusController.CompleteAsync(processUpdatesTask);
return(screenshotsSources);
}
public override async Task ProcessActivityAsync(IStatus status)
{
var cleanupTask = await statusController.CreateAsync(status, $"Cleanup {context}");
var expectedItems = await itemizeAllExpectedItemsAsyncDelegate.ItemizeAllAsync(status);
var actualItems = await itemizeAllActualItemsAsyncDelegate.ItemizeAllAsync(status);
var unexpectedItems = actualItems.Except(expectedItems);
var cleanupItems = new List <string>();
foreach (var unexpectedItem in unexpectedItems)
{
foreach (var detailedItem in itemizeDetailsDelegate.Itemize(unexpectedItem))
{
cleanupItems.Add(detailedItem);
cleanupItems.Add(formatSupplementaryItemDelegate.Format(detailedItem));
}
}
var moveToRecycleBinTask = await statusController.CreateAsync(status, "Move unexpected items to recycle bin");
var current = 0;
foreach (var item in cleanupItems)
{
await statusController.UpdateProgressAsync(
moveToRecycleBinTask,
++current,
cleanupItems.Count,
item);
recycleDelegate.Recycle(item);
}
// check if any of the directories are left empty and delete
var emptyDirectories = new List <string>();
foreach (var item in cleanupItems)
{
var directory = Path.GetDirectoryName(item);
if (!emptyDirectories.Contains(directory) &&
!directoryController.EnumerateFiles(directory).Any() &&
!directoryController.EnumerateDirectories(directory).Any())
{
emptyDirectories.Add(directory);
}
}
foreach (var directory in emptyDirectories)
{
directoryController.Delete(directory);
}
await statusController.CompleteAsync(moveToRecycleBinTask);
}
public async Task <IDictionary <long, IList <string> > > GetDownloadSourcesAsync(IStatus status)
{
var getDownloadSourcesStatus = await statusController.CreateAsync(status, "Get download sources");
var productImageSources = new Dictionary <long, IList <string> >();
var productIds = await itemizeAllProductsAsyncDelegate.ItemizeAllAsync(getDownloadSourcesStatus);
var current = 0;
foreach (var id in productIds)
{
await statusController.UpdateProgressAsync(
getDownloadSourcesStatus,
++current,
productIds.Count(),
id.ToString());
var productCore = await dataController.GetByIdAsync(id, getDownloadSourcesStatus);
// not all updated products can be found with all dataControllers
if (productCore == null)
{
continue;
}
var imageSources = new List <string> {
formatImagesUriDelegate.Format(
getImageUriDelegate.GetImageUri(productCore))
};
if (!productImageSources.ContainsKey(id))
{
productImageSources.Add(id, new List <string>());
}
foreach (var source in imageSources)
{
productImageSources[id].Add(source);
}
}
await statusController.CompleteAsync(getDownloadSourcesStatus);
return(productImageSources);
}
public async Task ConstrainAsync(int delaySeconds, IStatus status)
{
var throttleTask = await statusController.CreateAsync(
status,
$"Sleeping {formatSecondsDelegate.Format(delaySeconds)} before next operation");
for (var ii = 0; ii < delaySeconds; ii++)
{
await Task.Delay(1000);
await statusController.UpdateProgressAsync(
throttleTask,
ii + 1,
delaySeconds,
"Countdown",
TimeUnits.Seconds);
}
await statusController.CompleteAsync(throttleTask);
}
public async Task OutputContinuousAsync(IStatus status, params string[] data)
{
// Clear frame buffer
fragmentBuffer.Clear();
// Break the lines with new line separator, also wrap lines given the available console width
var fragmentWrappedLines = formatTextToFitConsoleWindowDelegate.Format(data);
// To build the buffer, we'll pad each line with spaces.
// That takes care of previous frame lines that could have been longer
foreach (var line in fragmentWrappedLines)
{
fragmentBuffer.Append(line.PadRight(consoleController.WindowWidth));
}
// Write the current frame buffer, no need to worry about line breaks or new lines:
// padded content length takes care of both
consoleController.Write(fragmentBuffer.ToString());
// For consoles that have zero cursor width make sure to forcefully break the line
if (consoleController.CursorLeft > 0)
{
consoleController.WriteLine(string.Empty);
}
// Clear remaining lines from the previous frame, if any
await ClearContinuousLinesAsync(previousFrameLines - fragmentWrappedLines.Count());
}
public IDictionary <string, string> GetViewModel(IStatus status)
{
if (status.Complete)
{
return(null);
}
// viewmodel schemas
var viewModel = new Dictionary <string, string>
{
{ StatusAppViewModelSchema.Title, "" },
{ StatusAppViewModelSchema.ContainsProgress, "" },
{ StatusAppViewModelSchema.ProgressTarget, "" },
{ StatusAppViewModelSchema.ProgressPercent, "" },
{ StatusAppViewModelSchema.ProgressCurrent, "" },
{ StatusAppViewModelSchema.ProgressTotal, "" },
{ StatusAppViewModelSchema.ContainsETA, "" },
{ StatusAppViewModelSchema.RemainingTime, "" },
{ StatusAppViewModelSchema.AverageUnitsPerSecond, "" },
{ StatusAppViewModelSchema.ContainsFailures, "" },
{ StatusAppViewModelSchema.FailuresCount, "" },
{ StatusAppViewModelSchema.ContainsWarnings, "" },
{ StatusAppViewModelSchema.WarningsCount, "" }
};
viewModel[StatusAppViewModelSchema.Title] = status.Title;
if (status.Progress != null)
{
var current = status.Progress.Current;
var total = status.Progress.Total;
viewModel[StatusAppViewModelSchema.ContainsProgress] = "true";
viewModel[StatusAppViewModelSchema.ProgressTarget] = status.Progress.Target;
viewModel[StatusAppViewModelSchema.ProgressPercent] = string.Format("{0:P1}", (double)current / total);
var currentFormatted = current.ToString();
var totalFormatted = total.ToString();
if (status.Progress.Unit == DataUnits.Bytes)
{
viewModel[StatusAppViewModelSchema.ContainsETA] = "true";
currentFormatted = formatBytesDelegate.Format(current);
totalFormatted = formatBytesDelegate.Format(total);
var estimatedTimeAvailable = formatRemainingTimeAtSpeedDelegate.Format(status);
var remainingTime = formatSecondsDelegate.Format(estimatedTimeAvailable.Item1);
var speed = formatBytesDelegate.Format((long)estimatedTimeAvailable.Item2);
viewModel[StatusAppViewModelSchema.RemainingTime] = remainingTime;
viewModel[StatusAppViewModelSchema.AverageUnitsPerSecond] = speed;
}
viewModel[StatusAppViewModelSchema.ProgressCurrent] = currentFormatted;
viewModel[StatusAppViewModelSchema.ProgressTotal] = totalFormatted;
}
if (status.Failures != null && status.Failures.Any())
{
viewModel[StatusAppViewModelSchema.ContainsFailures] = "true";
viewModel[StatusAppViewModelSchema.FailuresCount] = status.Failures.Count.ToString();
}
if (status.Warnings != null && status.Warnings.Any())
{
viewModel[StatusAppViewModelSchema.ContainsWarnings] = "true";
viewModel[StatusAppViewModelSchema.WarningsCount] = status.Warnings.Count.ToString();
}
return(viewModel);
}
public override async Task ProcessActivityAsync(IStatus status)
{
var validateProductsStatus = await statusController.CreateAsync(status, "Validate products");
var current = 0;
var validateProductsList = await itemizeAllProductsAsyncDelegate.ItemizeAllAsync(validateProductsStatus);
var validateProductsCount = validateProductsList.Count();
foreach (var id in validateProductsList)
{
var gameDetails = await gameDetailsDataController.GetByIdAsync(id, validateProductsStatus);
var validationResults = await validationResultsDataController.GetByIdAsync(id, validateProductsStatus);
if (validationResults == null)
{
validationResults = new ValidationResult
{
Id = id,
Title = gameDetails.Title
}
}
;
await statusController.UpdateProgressAsync(validateProductsStatus,
++current,
validateProductsCount,
gameDetails.Title);
var localFiles = new List <string>();
var getLocalFilesTask = await statusController.CreateAsync(validateProductsStatus, "Enumerate local product files");
foreach (var manualUrl in
await itemizeGameDetailsManualUrlsAsyncDelegate.ItemizeAsync(gameDetails, getLocalFilesTask))
{
var resolvedUri = await routingController.TraceRouteAsync(id, manualUrl, getLocalFilesTask);
// use directory from source and file from resolved URI
var localFile = Path.Combine(
productFileDirectoryDelegate.GetDirectory(manualUrl),
productFileFilenameDelegate.GetFilename(resolvedUri));
localFiles.Add(localFile);
}
await statusController.CompleteAsync(getLocalFilesTask);
// check if current validation results allow us to skip validating current product
// otherwise - validate all the files again
// ...
var fileValidationResults = new List <IFileValidationResult>(localFiles.Count);
var validateFilesTask = await statusController.CreateAsync(
validateProductsStatus,
"Validate product files");
var currentFile = 0;
foreach (var localFile in localFiles)
{
await statusController.UpdateProgressAsync(validateFilesTask,
++currentFile,
localFiles.Count,
localFile);
var validationFile = formatValidationFileDelegate.Format(localFile);
try
{
fileValidationResults.Add(await fileValidationController.ValidateFileAsync(
localFile,
validationFile,
validateFilesTask));
}
catch (Exception ex)
{
await statusController.FailAsync(validateProductsStatus,
$"{localFile}: {ex.Message}");
}
}
await statusController.CompleteAsync(validateFilesTask);
validationResults.Files = fileValidationResults.ToArray();
var updateValidationResultsTask = await statusController.CreateAsync(
validateProductsStatus,
"Update validation results");
await validationResultsDataController.UpdateAsync(validationResults, validateProductsStatus);
await statusController.CompleteAsync(updateValidationResultsTask);
}
await statusController.CompleteAsync(validateProductsStatus);
}
}
public async Task DownloadProductFileAsync(long id, string title, string sourceUri, string destination, IStatus status)
{
var downloadTask = await statusController.CreateAsync(status, "Download game details manual url");
HttpResponseMessage response;
try
{
response = await networkController.RequestResponseAsync(downloadTask, HttpMethod.Get, sourceUri);
}
catch (HttpRequestException ex)
{
await statusController.FailAsync(
downloadTask,
$"Failed to get successful response for {sourceUri} for " +
$"product {id}: {title}, message: {ex.Message}");
await statusController.CompleteAsync(downloadTask);
return;
}
using (response)
{
var resolvedUri = response.RequestMessage.RequestUri.ToString();
// GOG.com quirk
// When resolving ManualUrl from GameDetails we get CDN Uri with the session key.
// Storing this key is pointless - it expries after some time and needs to be updated.
// So here we filter our this session key and store direct file Uri
var uriSansSession = formatUriRemoveSessionDelegate.Format(resolvedUri);
await routingController.UpdateRouteAsync(
id,
title,
sourceUri,
uriSansSession,
downloadTask);
try
{
await downloadFromResponseAsyncDelegate.DownloadFromResponseAsync(
response,
destination,
downloadTask);
}
catch (Exception ex)
{
await statusController.FailAsync(
downloadTask,
$"Couldn't download {sourceUri}, resolved as {resolvedUri} to {destination} " +
$"for product {id}: {title}, error message: {ex.Message}");
}
// GOG.com quirk
// Supplementary download is a secondary download to a primary driven by download scheduling
// The example is validation file - while we can use the same pipeline, we would be
// largerly duplicating all the work to establish the session, compute the name etc.
// While the only difference validation files have - is additional extension.
// So instead we'll do a supplementary download using primary download information
await downloadValidationFileAsyncDelegate?.DownloadProductFileAsync(
id,
title,
resolvedUri,
destination,
downloadTask);
}
await statusController.CompleteAsync(downloadTask);
}
public IDictionary <string, string> GetViewModel(IStatus status)
{
// viewmodel schemas
var viewModel = new Dictionary <string, string>
{
{ StatusReportViewModelSchema.Title, "" },
{ StatusReportViewModelSchema.Complete, "" },
{ StatusReportViewModelSchema.Started, "" },
{ StatusReportViewModelSchema.Duration, "" },
{ StatusReportViewModelSchema.Result, "" },
{ StatusReportViewModelSchema.ContainsProgress, "" },
{ StatusReportViewModelSchema.ProgressTarget, "" },
{ StatusReportViewModelSchema.ProgressCurrent, "" },
{ StatusReportViewModelSchema.ProgressTotal, "" },
{ StatusReportViewModelSchema.ContainsFailures, "" },
{ StatusReportViewModelSchema.Failures, "" },
{ StatusReportViewModelSchema.ContainsWarnings, "" },
{ StatusReportViewModelSchema.Warnings, "" },
{ StatusReportViewModelSchema.ContainsInformation, "" },
{ StatusReportViewModelSchema.Information, "" }
};
viewModel[StatusReportViewModelSchema.Title] = status.Title;
viewModel[StatusReportViewModelSchema.Complete] = status.Complete ? "true" : "";
viewModel[StatusReportViewModelSchema.Started] = status.Started.ToLocalTime().ToString();
viewModel[StatusReportViewModelSchema.Duration] =
status.Complete ?
formatSecondsDelegate.Format((status.Completed - status.Started).Seconds) :
"";
var results = new List <string>();
if (status.Failures != null && status.Failures.Any())
{
results.Add("Failure(s)");
}
if (status.Warnings != null && status.Warnings.Any())
{
results.Add("Warning(s)");
}
var result = string.Join(",", results);
if (string.IsNullOrEmpty(result))
{
result = "Success";
}
viewModel[StatusReportViewModelSchema.Result] = result;
if (status.Progress != null)
{
var current = status.Progress.Current;
var total = status.Progress.Total;
viewModel[StatusReportViewModelSchema.ContainsProgress] = "true";
viewModel[StatusReportViewModelSchema.ProgressTarget] = status.Progress.Target;
var currentFormatted = current.ToString();
var totalFormatted = total.ToString();
if (status.Progress.Unit == DataUnits.Bytes)
{
currentFormatted = formatBytesDelegate.Format(current);
totalFormatted = formatBytesDelegate.Format(total);
}
viewModel[StatusReportViewModelSchema.ProgressCurrent] = currentFormatted;
viewModel[StatusReportViewModelSchema.ProgressTotal] = totalFormatted;
}
if (status.Failures != null && status.Failures.Any())
{
viewModel[StatusReportViewModelSchema.ContainsFailures] = "true";
viewModel[StatusReportViewModelSchema.Failures] = string.Join("; ", status.Failures);
}
if (status.Warnings != null && status.Warnings.Any())
{
viewModel[StatusReportViewModelSchema.ContainsWarnings] = "true";
viewModel[StatusReportViewModelSchema.Warnings] = string.Join("; ", status.Warnings);
}
if (status.Information != null && status.Information.Any())
{
viewModel[StatusReportViewModelSchema.ContainsInformation] = "true";
viewModel[StatusReportViewModelSchema.Information] = string.Join("; ", status.Information);
}
return(viewModel);
}
public async Task <GameDetails> GetDeserializedAsync(IStatus status, string uri, IDictionary <string, string> parameters = null)
{
// GOG.com quirk
// GameDetails as sent by GOG.com servers have an intersting data structure for downloads:
// it's represented as an array, where first element is a string with the language,
// followed by actual download details, something like:
// [ "English", { // download1 }, { // download2 } ]
// Which of course is not a problem for JavaScript, but is a problem for
// deserializing into strongly typed C# data structures.
// To work around this we wrapped encapsulated usual network requests,
// data transformation and desearialization in a sinlge package.
// To process downloads we do the following:
// - if response contains language downloads, signified by [[
// - extract actual language information and remove it from the string
// - deserialize downloads into OperatingSystemsDownloads collection
// - assign languages, since we know we should have as many downloads array as languages
var data = await getResourceAsyncDelegate.GetResourceAsync(status, uri, parameters);
var gameDetails = serializationController.Deserialize <GameDetails>(data);
if (gameDetails == null)
{
return(null);
}
var gameDetailsLanguageDownloads = new List <OperatingSystemsDownloads>();
if (!confirmStringContainsLanguageDownloadsDelegate.Confirm(data))
{
return(gameDetails);
}
var downloadStrings = itemizeGameDetailsDownloadsDelegate.Itemize(data);
foreach (var downloadString in downloadStrings)
{
var downloadLanguages = itemizeDownloadLanguagesDelegate.Itemize(downloadString);
if (downloadLanguages == null)
{
throw new InvalidOperationException("Cannot find any download languages or download language format changed.");
}
// ... and remove download lanugage strings from downloads
var downloadsStringSansLanguages = replaceMultipleStringsDelegate.ReplaceMultiple(
downloadString,
string.Empty,
downloadLanguages.ToArray());
// now it should be safe to deserialize langugage downloads
var downloads =
serializationController.Deserialize <OperatingSystemsDownloads[][]>(
downloadsStringSansLanguages);
// and convert GOG two-dimensional array of downloads to single-dimensional array
var languageDownloads = convert2DArrayToArrayDelegate.Convert(downloads);
if (languageDownloads.Count() != downloadLanguages.Count())
{
throw new InvalidOperationException("Number of extracted language downloads doesn't match number of languages.");
}
// map language downloads with the language code we extracted earlier
var languageDownloadIndex = 0;
collectionController.Map(downloadLanguages, language =>
{
var formattedLanguage = formatDownloadLanguageDelegate.Format(language);
var languageCode = languageController.GetLanguageCode(formattedLanguage);
languageDownloads[languageDownloadIndex++].Language = languageCode;
});
gameDetailsLanguageDownloads.AddRange(languageDownloads);
}
gameDetails.LanguageDownloads = gameDetailsLanguageDownloads.ToArray();
return(gameDetails);
}