public async Task <IEnumerable <string> > ItemizeAllAsync(IStatus status)
{
var enumerateGameDetailsDirectoriesTask = await statusController.CreateAsync(status, "Enumerate gameDetails directories");
var directories = new List <string>();
var current = 0;
var gameDetailsIds = await gameDetailsDataController.ItemizeAllAsync(enumerateGameDetailsDirectoriesTask);
var gameDetailsCount = await gameDetailsDataController.CountAsync(enumerateGameDetailsDirectoriesTask);
foreach (var id in gameDetailsIds)
{
var gameDetails = await gameDetailsDataController.GetByIdAsync(id, enumerateGameDetailsDirectoriesTask);
await statusController.UpdateProgressAsync(
enumerateGameDetailsDirectoriesTask,
++current,
gameDetailsCount,
gameDetails.Title);
directories.AddRange(
await itemizeGameDetailsDirectoriesAsyncDelegate.ItemizeAsync(
gameDetails,
enumerateGameDetailsDirectoriesTask));
}
await statusController.CompleteAsync(enumerateGameDetailsDirectoriesTask);
return(directories);
}
public async Task <IDictionary <long, IList <string> > > GetDownloadSourcesAsync(IStatus status)
{
var processUpdatesTask = await statusController.CreateAsync(status, "Process screenshots updates");
var screenshotsSources = new Dictionary <long, IList <string> >();
var current = 0;
var total = await screenshotsDataController.CountAsync(processUpdatesTask);
var processProductsScreenshotsTask = await statusController.CreateAsync(processUpdatesTask, "Process product screenshots");
foreach (var id in await screenshotsDataController.ItemizeAllAsync(processProductsScreenshotsTask))
{
var productScreenshots = await screenshotsDataController.GetByIdAsync(id, processProductsScreenshotsTask);
if (productScreenshots == null)
{
await statusController.WarnAsync(processProductsScreenshotsTask, $"Product {id} doesn't have screenshots");
continue;
}
await statusController.UpdateProgressAsync(
processUpdatesTask,
++current,
total,
productScreenshots.Title);
var currentProductScreenshotSources = new List <string>();
foreach (var uri in productScreenshots.Uris)
{
var sourceUri = formatScreenshotsUriDelegate.Format(uri);
var destinationUri = Path.Combine(
screenshotsDirectoryDelegate.GetDirectory(string.Empty),
Path.GetFileName(sourceUri));
if (fileController.Exists(destinationUri))
{
continue;
}
currentProductScreenshotSources.Add(sourceUri);
}
if (currentProductScreenshotSources.Any())
{
screenshotsSources.Add(id, currentProductScreenshotSources);
}
}
await statusController.CompleteAsync(processProductsScreenshotsTask);
await statusController.CompleteAsync(processUpdatesTask);
return(screenshotsSources);
}
public async Task <IDictionary <long, IList <string> > > GetDownloadSourcesAsync(IStatus status)
{
var getDownloadSourcesStatus = await statusController.CreateAsync(status, "Get download sources");
var productImageSources = new Dictionary <long, IList <string> >();
var productIds = await itemizeAllProductsAsyncDelegate.ItemizeAllAsync(getDownloadSourcesStatus);
var current = 0;
foreach (var id in productIds)
{
await statusController.UpdateProgressAsync(
getDownloadSourcesStatus,
++current,
productIds.Count(),
id.ToString());
var productCore = await dataController.GetByIdAsync(id, getDownloadSourcesStatus);
// not all updated products can be found with all dataControllers
if (productCore == null)
{
continue;
}
var imageSources = new List <string> {
formatImagesUriDelegate.Format(
getImageUriDelegate.GetImageUri(productCore))
};
if (!productImageSources.ContainsKey(id))
{
productImageSources.Add(id, new List <string>());
}
foreach (var source in imageSources)
{
productImageSources[id].Add(source);
}
}
await statusController.CompleteAsync(getDownloadSourcesStatus);
return(productImageSources);
}
public async Task ConstrainAsync(int delaySeconds, IStatus status)
{
var throttleTask = await statusController.CreateAsync(
status,
$"Sleeping {formatSecondsDelegate.Format(delaySeconds)} before next operation");
for (var ii = 0; ii < delaySeconds; ii++)
{
await Task.Delay(1000);
await statusController.UpdateProgressAsync(
throttleTask,
ii + 1,
delaySeconds,
"Countdown",
TimeUnits.Seconds);
}
await statusController.CompleteAsync(throttleTask);
}
public async Task DownloadFromResponseAsync(HttpResponseMessage response, string destination, IStatus status)
{
response.EnsureSuccessStatusCode();
var filename = response.RequestMessage.RequestUri.Segments.Last();
var fullPath = Path.Combine(destination, filename);
int bufferSize = 1024 * 1024; // 1M
byte[] buffer = new byte[bufferSize];
int bytesRead = 0;
long totalBytesRead = 0;
// don't redownload file with the same name and size
if (fileController.Exists(fullPath) &&
fileController.GetSize(fullPath) == response.Content.Headers.ContentLength)
{
await statusController.InformAsync(
status,
$"File {fullPath} already exists and matches response size, will not be redownloading");
return;
}
using (var writeableStream = streamController.OpenWritable(fullPath))
using (var responseStream = await response.Content.ReadAsStreamAsync())
{
while ((bytesRead = await responseStream.ReadAsync(buffer, 0, bufferSize)) > 0)
{
totalBytesRead += bytesRead;
var contentLength = response.Content.Headers.ContentLength != null ?
(long) response.Content.Headers.ContentLength :
totalBytesRead;
await writeableStream.WriteAsync(buffer, 0, bytesRead);
await statusController.UpdateProgressAsync(
status,
totalBytesRead,
contentLength,
filename,
DataUnits.Bytes);
}
}
}
public async Task CommitAsync(IStatus status)
{
var commitTask = await statusController.CreateAsync(status, "Commit updated data");
// commit records controller
if (recordsController != null)
{
var commitRecordsTask = await statusController.CreateAsync(commitTask, "Commit records");
await recordsController.CommitAsync(status);
await statusController.CompleteAsync(commitRecordsTask);
}
var commitDataTask = await statusController.CreateAsync(commitTask, "Commit items");
await stashController.SaveAsync(commitDataTask);
await statusController.CompleteAsync(commitDataTask);
var additionalCommitsTask = await statusController.CreateAsync(commitTask, "Additional commit dependencies");
var current = 0;
foreach (var commitDelegate in additionalCommitDelegates)
{
await statusController.UpdateProgressAsync(
additionalCommitsTask,
++current,
additionalCommitDelegates.Length,
"Commit delegate");
await commitDelegate.CommitAsync(additionalCommitsTask);
}
await statusController.CompleteAsync(additionalCommitsTask);
await statusController.CompleteAsync(commitTask);
}
public async Task <IDictionary <long, IList <string> > > GetDownloadSourcesAsync(IStatus status)
{
var getDownloadSourcesStatus = await statusController.CreateAsync(status, "Get download sources");
var gameDetailsDownloadSources = new Dictionary <long, IList <string> >();
var current = 0;
foreach (var id in await updatedDataController.ItemizeAllAsync(getDownloadSourcesStatus))
{
await statusController.UpdateProgressAsync(
getDownloadSourcesStatus,
++current,
await updatedDataController.CountAsync(getDownloadSourcesStatus),
id.ToString());
var gameDetails = await gameDetailsDataController.GetByIdAsync(id, getDownloadSourcesStatus);
if (!gameDetailsDownloadSources.ContainsKey(id))
{
gameDetailsDownloadSources.Add(id, new List <string>());
}
foreach (var manualUrl in
await itemizeGameDetailsManualUrlsAsyncController.ItemizeAsync(gameDetails, status))
{
if (!gameDetailsDownloadSources[id].Contains(manualUrl))
{
gameDetailsDownloadSources[id].Add(manualUrl);
}
}
}
await statusController.CompleteAsync(getDownloadSourcesStatus);
return(gameDetailsDownloadSources);
}
public async Task <IFileValidationResult> ValidateFileAsync(string productFileUri, string validationUri, IStatus status)
{
var fileValidation = new FileValidation
{
Filename = productFileUri
};
if (string.IsNullOrEmpty(productFileUri))
{
throw new ArgumentNullException();
}
if (string.IsNullOrEmpty(validationUri))
{
throw new ArgumentNullException();
}
fileValidation.ValidationExpected = confirmValidationExpectedDelegate.Confirm(productFileUri);
if (!fileValidation.ValidationExpected)
{
return(fileValidation);
}
fileValidation.ValidationFileExists = VerifyValidationFileExists(validationUri);
fileValidation.ProductFileExists = VerifyProductFileExists(productFileUri);
if (!fileValidation.ValidationFileExists ||
!fileValidation.ProductFileExists)
{
return(fileValidation);
}
try
{
using (var xmlStream = streamController.OpenReadable(validationUri))
validationXml.Load(xmlStream);
fileValidation.ValidationFileIsValid = true;
}
catch
{
fileValidation.ValidationFileIsValid = false;
return(fileValidation);
}
var fileElement = validationXml.GetElementsByTagName("file");
if (fileElement == null ||
fileElement.Count < 1 ||
fileElement[0] == null ||
fileElement[0].Attributes == null)
{
fileValidation.ValidationFileIsValid = false;
return(fileValidation);
}
long expectedSize;
string expectedName;
int chunks;
bool available;
try
{
expectedSize = long.Parse(fileElement[0].Attributes["total_size"]?.Value);
expectedName = fileElement[0].Attributes["name"]?.Value;
chunks = int.Parse(fileElement[0].Attributes["chunks"]?.Value);
available = fileElement[0].Attributes["available"]?.Value == "1";
if (!available)
{
var notAvailableMessage = fileElement[0].Attributes["notavailablemsg"]?.Value;
throw new Exception(notAvailableMessage);
}
}
catch
{
fileValidation.ValidationFileIsValid = false;
return(fileValidation);
}
fileValidation.FilenameVerified = VerifyFilename(productFileUri, expectedName);
fileValidation.SizeVerified = VerifySize(productFileUri, expectedSize);
var chunksValidation = new List <IChunkValidation>();
using (var fileStream = streamController.OpenReadable(productFileUri))
{
long length = 0;
foreach (XmlNode chunkElement in fileElement[0].ChildNodes)
{
if (chunkElement.Name != "chunk")
{
continue;
}
long from, to = 0;
string expectedMd5 = string.Empty;
from = long.Parse(chunkElement.Attributes["from"]?.Value);
to = long.Parse(chunkElement.Attributes["to"]?.Value);
length += (to - from);
expectedMd5 = chunkElement.FirstChild.Value;
chunksValidation.Add(await VerifyChunkAsync(fileStream, from, to, expectedMd5, status));
await statusController.UpdateProgressAsync(
status,
length,
expectedSize,
productFileUri,
DataUnits.Bytes);
}
fileValidation.Chunks = chunksValidation.ToArray();
await statusController.UpdateProgressAsync(status, length, expectedSize, productFileUri);
}
await statusController.InformAsync(status, $"Validation result: {productFileUri} is valid: " +
$"{validationResultController.ProductFileIsValid(fileValidation)}");
return(fileValidation);
}
public async Task <IList <T> > GetPageResultsAsync(IStatus status)
{
// GOG.com quirk
// Products, AccountProducts use server-side paginated results, similar to Wikipedia.
// This class encapsulates requesting sequence of pages up to the total number of pages.
// Additionally page requests are filtered using hashes, so if response has the same hash
// we would not deserialize it again - no point passing around same data.
// Please note that ealier app versions also used heuristic optimization when
// some page was unchanged - stop requesting next pages. This leads to stale data as GOG.com
// changes information all the time updating Products and AccountProducts. It's especially
// important for AccountProducts as GOG.com can set Updates information on any AccountProduct.
// Updated is used for driving updated.json - that in turn is used for all subsequent operations
// as an optimization - we don't process all products all the time, just updated
var pageResults = new List <T>();
var currentPage = 1;
var totalPages = 1;
T pageResult = null;
var getPagesTask = await statusController.CreateAsync(status, $"Request {context}");
do
{
var response = await requestPageAsyncDelegate.RequestPageAsync(
requestUri,
requestParameters,
currentPage,
getPagesTask);
await statusController.UpdateProgressAsync(
getPagesTask,
currentPage,
totalPages,
requestUri,
PageUnits.Pages);
var requestHash = await storedHashController.GetHashAsync(requestUri + currentPage, getPagesTask);
var responseHash = await getStringHashDelegate.GetHashAsync(response, getPagesTask);
pageResult = serializationController.Deserialize <T>(response);
if (pageResult == null)
{
continue;
}
totalPages = pageResult.TotalPages;
if (responseHash == requestHash)
{
continue;
}
var setHashTask = await statusController.CreateAsync(getPagesTask, "Set hash", false);
await storedHashController.SetHashAsync(requestUri + currentPage, responseHash, setHashTask);
await statusController.CompleteAsync(setHashTask, false);
pageResults.Add(pageResult);
} while (++currentPage <= totalPages);
await statusController.CompleteAsync(getPagesTask);
return(pageResults);
}
