public async Task <T> DeserializePullAsync <T>(string uri, IStatus status)
{
var started = DateTime.Now;
var deserializePullTask = await statusController.CreateAsync(status, "Reading serialized data", false);
T data = default(T);
if (fileController.Exists(uri))
{
using (var readableStream = streamController.OpenReadable(uri))
data = Serializer.Deserialize <T>(readableStream);
}
await statusController.CompleteAsync(deserializePullTask, false);
var completed = DateTime.Now;
var duration = (completed - started).TotalMilliseconds;
traceDelegate?.Trace(
"DePull",
started.ToFileTimeUtc().ToString(),
completed.ToFileTimeUtc().ToString(),
duration.ToString(),
uri);
return(data);
}
public async Task UpdateScreenshotsAsync(Product product, IStatus status)
{
var requestProductPageTask = await statusController.CreateAsync(status, "Request product page containing screenshots information");
var productPageUri = string.Format(getUpdateUriDelegate.GetUpdateUri(Entity.Screenshots), product.Url);
var productPageContent = await networkController.GetResourceAsync(requestProductPageTask, productPageUri);
await statusController.CompleteAsync(requestProductPageTask);
var extractScreenshotsTask = await statusController.CreateAsync(status, "Exract screenshots from the page");
var extractedProductScreenshots = itemizeScreenshotsDelegates.Itemize(productPageContent);
if (extractedProductScreenshots == null)
{
return;
}
var productScreenshots = new ProductScreenshots
{
Id = product.Id,
Title = product.Title,
Uris = new List <string>(extractedProductScreenshots)
};
await statusController.CompleteAsync(extractScreenshotsTask);
var updateProductScreenshotsTask = await statusController.CreateAsync(status, "Add product screenshots");
await screenshotsDataController.UpdateAsync(productScreenshots, updateProductScreenshotsTask);
await statusController.CompleteAsync(updateProductScreenshotsTask);
}
public async Task <IDictionary <long, IList <string> > > GetDownloadSourcesAsync(IStatus status)
{
var processUpdatesTask = await statusController.CreateAsync(status, "Process screenshots updates");
var screenshotsSources = new Dictionary <long, IList <string> >();
var current = 0;
var total = await screenshotsDataController.CountAsync(processUpdatesTask);
var processProductsScreenshotsTask = await statusController.CreateAsync(processUpdatesTask, "Process product screenshots");
foreach (var id in await screenshotsDataController.ItemizeAllAsync(processProductsScreenshotsTask))
{
var productScreenshots = await screenshotsDataController.GetByIdAsync(id, processProductsScreenshotsTask);
if (productScreenshots == null)
{
await statusController.WarnAsync(processProductsScreenshotsTask, $"Product {id} doesn't have screenshots");
continue;
}
await statusController.UpdateProgressAsync(
processUpdatesTask,
++current,
total,
productScreenshots.Title);
var currentProductScreenshotSources = new List <string>();
foreach (var uri in productScreenshots.Uris)
{
var sourceUri = formatScreenshotsUriDelegate.Format(uri);
var destinationUri = Path.Combine(
screenshotsDirectoryDelegate.GetDirectory(string.Empty),
Path.GetFileName(sourceUri));
if (fileController.Exists(destinationUri))
{
continue;
}
currentProductScreenshotSources.Add(sourceUri);
}
if (currentProductScreenshotSources.Any())
{
screenshotsSources.Add(id, currentProductScreenshotSources);
}
}
await statusController.CompleteAsync(processProductsScreenshotsTask);
await statusController.CompleteAsync(processUpdatesTask);
return(screenshotsSources);
}
public async Task <IEnumerable <string> > ItemizeAsync(GameDetails gameDetails, IStatus status)
{
var enumerateGameDetailsFilesStatus = await statusController.CreateAsync(status, "Enumerate game details files");
var gameDetailsFiles = new List <string>();
var gameDetailsManualUrls = await itemizeGameDetailsManualUrlsDelegate.ItemizeAsync(gameDetails, status);
var gameDetailsManualUrlsCount = gameDetailsManualUrls.Count();
var gameDetailsResolvedUris = await routingController.TraceRoutesAsync(
gameDetails.Id,
gameDetailsManualUrls,
enumerateGameDetailsFilesStatus);
// that means that routes information is incomplete and
// it's not possible to map manualUrls to resolvedUrls
if (gameDetailsManualUrlsCount != gameDetailsResolvedUris.Count)
{
await statusController.CompleteAsync(enumerateGameDetailsFilesStatus);
throw new ArgumentException($"Product {gameDetails.Id} resolvedUris count doesn't match manualUrls count");
}
for (var ii = 0; ii < gameDetailsResolvedUris.Count; ii++)
{
// since we don't download all languages and operating systems
// we don't have routes for each and every gameDetails uri
// however the ones we have should represent expected files for that product
if (string.IsNullOrEmpty(gameDetailsResolvedUris[ii]))
{
continue;
}
//var localFileUri = Path.Combine(
// getDirectoryDelegate.GetDirectory(gameDetailsManualUrls.ElementAt(ii)),
//getFilenameDelegate.GetFilename(gameDetailsResolvedUris[ii]));
var localFilePath = getPathDelegate.GetPath(
gameDetailsManualUrls.ElementAt(ii),
gameDetailsResolvedUris[ii]);
gameDetailsFiles.Add(localFilePath);
}
await statusController.CompleteAsync(enumerateGameDetailsFilesStatus);
if (gameDetailsManualUrlsCount != gameDetailsFiles.Count)
{
throw new ArgumentException($"Product {gameDetails.Id} files count doesn't match manualUrls count");
}
return(gameDetailsFiles);
}
public async Task <string> TraceRouteAsync(long id, string source, IStatus status)
{
var traceRouteStatus = await statusController.CreateAsync(status, "Trace route");
var productRoutes = await productRoutesDataController.GetByIdAsync(id, traceRouteStatus);
if (productRoutes == null)
{
return(string.Empty);
}
await statusController.CompleteAsync(traceRouteStatus);
return(TraceProductRoute(productRoutes.Routes, source));
}
public async Task DownloadProductFileAsync(long id, string title, string sourceUri, string destination, IStatus status)
{
if (string.IsNullOrEmpty(sourceUri))
{
return;
}
var sourceUriSansSession = formatUriRemoveSessionDelegate.Format(sourceUri);
var destinationUri = formatValidationFileDelegate.Format(sourceUriSansSession);
// return early if validation is not expected for this file
if (!confirmValidationExpectedDelegate.Confirm(sourceUriSansSession))
{
return;
}
if (fileController.Exists(destinationUri))
{
await statusController.InformAsync(status, "Validation file already exists, will not be redownloading");
return;
}
var validationSourceUri = formatValidationUriDelegate.Format(sourceUri);
var downloadValidationFileTask = await statusController.CreateAsync(status, "Download validation file");
await downloadFromUriAsyncDelegate.DownloadFromUriAsync(
validationSourceUri,
validationDirectoryDelegate.GetDirectory(string.Empty),
downloadValidationFileTask);
await statusController.CompleteAsync(downloadValidationFileTask);
}
public async Task ConstrainAsync(string uri, IStatus status)
{
var prefix = collectionController.Reduce(uriPrefixes, uri.StartsWith).SingleOrDefault();
if (string.IsNullOrEmpty(prefix))
{
return;
}
// don't limit rate for the first N requests, even if they match rate limit prefix
if (++rateLimitRequestsCount <= passthroughCount)
{
return;
}
var now = DateTime.UtcNow;
var elapsed = (int)(now - lastRequestToUriPrefix[prefix]).TotalSeconds;
if (elapsed < requestIntervalSeconds)
{
var limitRateTask = await statusController.CreateAsync(status, "Limit request rate to avoid temporary server block");
await constrainExecutionAsyncDelegate.ConstrainAsync(requestIntervalSeconds - elapsed, status);
await statusController.CompleteAsync(limitRateTask);
}
lastRequestToUriPrefix[prefix] = now;
}
public async Task <IEnumerable <string> > ItemizeAllAsync(IStatus status)
{
var enumerateGameDetailsDirectoriesTask = await statusController.CreateAsync(status, "Enumerate gameDetails directories");
var directories = new List <string>();
var current = 0;
var gameDetailsIds = await gameDetailsDataController.ItemizeAllAsync(enumerateGameDetailsDirectoriesTask);
var gameDetailsCount = await gameDetailsDataController.CountAsync(enumerateGameDetailsDirectoriesTask);
foreach (var id in gameDetailsIds)
{
var gameDetails = await gameDetailsDataController.GetByIdAsync(id, enumerateGameDetailsDirectoriesTask);
await statusController.UpdateProgressAsync(
enumerateGameDetailsDirectoriesTask,
++current,
gameDetailsCount,
gameDetails.Title);
directories.AddRange(
await itemizeGameDetailsDirectoriesAsyncDelegate.ItemizeAsync(
gameDetails,
enumerateGameDetailsDirectoriesTask));
}
await statusController.CompleteAsync(enumerateGameDetailsDirectoriesTask);
return(directories);
}
public async Task LoadAsync(IStatus status)
{
var loadStatus = await statusController.CreateAsync(status, "Load stored data", false);
var storedDataUri = getPathDelegate.GetPath(string.Empty, string.Empty);
storedData = await serializedStorageController.DeserializePullAsync <ModelType>(storedDataUri, loadStatus);
if (storedData == null)
{
storedData = new ModelType();
}
DataAvailable = true;
await statusController.CompleteAsync(loadStatus, false);
}
public async Task CommitAsync(IStatus status)
{
var commitTask = await statusController.CreateAsync(status, "Commit updated data");
// commit records controller
if (recordsController != null)
{
var commitRecordsTask = await statusController.CreateAsync(commitTask, "Commit records");
await recordsController.CommitAsync(status);
await statusController.CompleteAsync(commitRecordsTask);
}
var commitDataTask = await statusController.CreateAsync(commitTask, "Commit items");
await stashController.SaveAsync(commitDataTask);
await statusController.CompleteAsync(commitDataTask);
var additionalCommitsTask = await statusController.CreateAsync(commitTask, "Additional commit dependencies");
var current = 0;
foreach (var commitDelegate in additionalCommitDelegates)
{
await statusController.UpdateProgressAsync(
additionalCommitsTask,
++current,
additionalCommitDelegates.Length,
"Commit delegate");
await commitDelegate.CommitAsync(additionalCommitsTask);
}
await statusController.CompleteAsync(additionalCommitsTask);
await statusController.CompleteAsync(commitTask);
}
public async Task <IEnumerable <string> > ItemizeAllAsync(IStatus status)
{
var enumerateProductFilesDirectoriesTask = await statusController.CreateAsync(
status,
"Enumerate productFiles directories");
var directories = new List <string>();
var productFilesDirectory = productFilesDirectoryDelegate.GetDirectory(string.Empty);
directories.AddRange(directoryController.EnumerateDirectories(productFilesDirectory));
await statusController.CompleteAsync(enumerateProductFilesDirectoriesTask);
return(directories);
}
public async Task <IDictionary <long, IList <string> > > GetDownloadSourcesAsync(IStatus status)
{
var getDownloadSourcesStatus = await statusController.CreateAsync(status, "Get download sources");
var productImageSources = new Dictionary <long, IList <string> >();
var productIds = await itemizeAllProductsAsyncDelegate.ItemizeAllAsync(getDownloadSourcesStatus);
var current = 0;
foreach (var id in productIds)
{
await statusController.UpdateProgressAsync(
getDownloadSourcesStatus,
++current,
productIds.Count(),
id.ToString());
var productCore = await dataController.GetByIdAsync(id, getDownloadSourcesStatus);
// not all updated products can be found with all dataControllers
if (productCore == null)
{
continue;
}
var imageSources = new List <string> {
formatImagesUriDelegate.Format(
getImageUriDelegate.GetImageUri(productCore))
};
if (!productImageSources.ContainsKey(id))
{
productImageSources.Add(id, new List <string>());
}
foreach (var source in imageSources)
{
productImageSources[id].Add(source);
}
}
await statusController.CompleteAsync(getDownloadSourcesStatus);
return(productImageSources);
}
public async Task DownloadFromUriAsync(string sourceUri, string destination, IStatus status)
{
var downloadEntryTask = await statusController.CreateAsync(status, "Download from source");
try
{
using (var response = await requestResponseAsyncDelegate.RequestResponseAsync(downloadEntryTask, HttpMethod.Get, sourceUri))
await downloadFromResponseAsyncDelegate.DownloadFromResponseAsync(response, destination, downloadEntryTask);
}
catch (Exception ex)
{
await statusController.WarnAsync(downloadEntryTask, $"{sourceUri}: {ex.Message}");
}
finally
{
await statusController.CompleteAsync(downloadEntryTask);
}
}
public async Task ConstrainAsync(int delaySeconds, IStatus status)
{
var throttleTask = await statusController.CreateAsync(
status,
$"Sleeping {formatSecondsDelegate.Format(delaySeconds)} before next operation");
for (var ii = 0; ii < delaySeconds; ii++)
{
await Task.Delay(1000);
await statusController.UpdateProgressAsync(
throttleTask,
ii + 1,
delaySeconds,
"Countdown",
TimeUnits.Seconds);
}
await statusController.CompleteAsync(throttleTask);
}
public async Task <Settings> CorrectAsync(Settings settings, IStatus status)
{
var correctSettingsStatus = await statusController.CreateAsync(status, "Correct settings");
if (settings.DownloadsLanguages == null)
{
settings.DownloadsLanguages = new string[0];
}
if (settings.DownloadsOperatingSystems == null)
{
settings.DownloadsOperatingSystems = new string[0];
}
if (settings.Directories == null)
{
settings.Directories = new Dictionary <string, string>();
}
await statusController.CompleteAsync(correctSettingsStatus);
return(settings);
}
public async Task <bool> AuthorizedAsync(IStatus status)
{
var getUserDataTask = await statusController.CreateAsync(status, "Get userData.json");
var userDataString = await networkController.GetResourceAsync(getUserDataTask, Uris.Paths.Authentication.UserData);
if (string.IsNullOrEmpty(userDataString))
{
return(false);
}
var userData = serializationController.Deserialize <Models.UserData>(userDataString);
await statusController.CompleteAsync(getUserDataTask);
return(userData.IsLoggedIn);
}
public async Task <IDictionary <long, IList <string> > > GetDownloadSourcesAsync(IStatus status)
{
var getDownloadSourcesStatus = await statusController.CreateAsync(status, "Get download sources");
var gameDetailsDownloadSources = new Dictionary <long, IList <string> >();
var current = 0;
foreach (var id in await updatedDataController.ItemizeAllAsync(getDownloadSourcesStatus))
{
await statusController.UpdateProgressAsync(
getDownloadSourcesStatus,
++current,
await updatedDataController.CountAsync(getDownloadSourcesStatus),
id.ToString());
var gameDetails = await gameDetailsDataController.GetByIdAsync(id, getDownloadSourcesStatus);
if (!gameDetailsDownloadSources.ContainsKey(id))
{
gameDetailsDownloadSources.Add(id, new List <string>());
}
foreach (var manualUrl in
await itemizeGameDetailsManualUrlsAsyncController.ItemizeAsync(gameDetails, status))
{
if (!gameDetailsDownloadSources[id].Contains(manualUrl))
{
gameDetailsDownloadSources[id].Add(manualUrl);
}
}
}
await statusController.CompleteAsync(getDownloadSourcesStatus);
return(gameDetailsDownloadSources);
}
public async Task <IList <T> > GetPageResultsAsync(IStatus status)
{
// GOG.com quirk
// Products, AccountProducts use server-side paginated results, similar to Wikipedia.
// This class encapsulates requesting sequence of pages up to the total number of pages.
// Additionally page requests are filtered using hashes, so if response has the same hash
// we would not deserialize it again - no point passing around same data.
// Please note that ealier app versions also used heuristic optimization when
// some page was unchanged - stop requesting next pages. This leads to stale data as GOG.com
// changes information all the time updating Products and AccountProducts. It's especially
// important for AccountProducts as GOG.com can set Updates information on any AccountProduct.
// Updated is used for driving updated.json - that in turn is used for all subsequent operations
// as an optimization - we don't process all products all the time, just updated
var pageResults = new List <T>();
var currentPage = 1;
var totalPages = 1;
T pageResult = null;
var getPagesTask = await statusController.CreateAsync(status, $"Request {context}");
do
{
var response = await requestPageAsyncDelegate.RequestPageAsync(
requestUri,
requestParameters,
currentPage,
getPagesTask);
await statusController.UpdateProgressAsync(
getPagesTask,
currentPage,
totalPages,
requestUri,
PageUnits.Pages);
var requestHash = await storedHashController.GetHashAsync(requestUri + currentPage, getPagesTask);
var responseHash = await getStringHashDelegate.GetHashAsync(response, getPagesTask);
pageResult = serializationController.Deserialize <T>(response);
if (pageResult == null)
{
continue;
}
totalPages = pageResult.TotalPages;
if (responseHash == requestHash)
{
continue;
}
var setHashTask = await statusController.CreateAsync(getPagesTask, "Set hash", false);
await storedHashController.SetHashAsync(requestUri + currentPage, responseHash, setHashTask);
await statusController.CompleteAsync(setHashTask, false);
pageResults.Add(pageResult);
} while (++currentPage <= totalPages);
await statusController.CompleteAsync(getPagesTask);
return(pageResults);
}
public async Task DownloadProductFileAsync(long id, string title, string sourceUri, string destination, IStatus status)
{
var downloadTask = await statusController.CreateAsync(status, "Download game details manual url");
HttpResponseMessage response;
try
{
response = await networkController.RequestResponseAsync(downloadTask, HttpMethod.Get, sourceUri);
}
catch (HttpRequestException ex)
{
await statusController.FailAsync(
downloadTask,
$"Failed to get successful response for {sourceUri} for " +
$"product {id}: {title}, message: {ex.Message}");
await statusController.CompleteAsync(downloadTask);
return;
}
using (response)
{
var resolvedUri = response.RequestMessage.RequestUri.ToString();
// GOG.com quirk
// When resolving ManualUrl from GameDetails we get CDN Uri with the session key.
// Storing this key is pointless - it expries after some time and needs to be updated.
// So here we filter our this session key and store direct file Uri
var uriSansSession = formatUriRemoveSessionDelegate.Format(resolvedUri);
await routingController.UpdateRouteAsync(
id,
title,
sourceUri,
uriSansSession,
downloadTask);
try
{
await downloadFromResponseAsyncDelegate.DownloadFromResponseAsync(
response,
destination,
downloadTask);
}
catch (Exception ex)
{
await statusController.FailAsync(
downloadTask,
$"Couldn't download {sourceUri}, resolved as {resolvedUri} to {destination} " +
$"for product {id}: {title}, error message: {ex.Message}");
}
// GOG.com quirk
// Supplementary download is a secondary download to a primary driven by download scheduling
// The example is validation file - while we can use the same pipeline, we would be
// largerly duplicating all the work to establish the session, compute the name etc.
// While the only difference validation files have - is additional extension.
// So instead we'll do a supplementary download using primary download information
await downloadValidationFileAsyncDelegate?.DownloadProductFileAsync(
id,
title,
resolvedUri,
destination,
downloadTask);
}
await statusController.CompleteAsync(downloadTask);
}
