public async Task <IEnumerable <string> > ItemizeAllAsync(IStatus status) { var enumerateGameDetailsDirectoriesTask = await statusController.CreateAsync(status, "Enumerate gameDetails directories"); var directories = new List <string>(); var current = 0; var gameDetailsIds = await gameDetailsDataController.ItemizeAllAsync(enumerateGameDetailsDirectoriesTask); var gameDetailsCount = await gameDetailsDataController.CountAsync(enumerateGameDetailsDirectoriesTask); foreach (var id in gameDetailsIds) { var gameDetails = await gameDetailsDataController.GetByIdAsync(id, enumerateGameDetailsDirectoriesTask); await statusController.UpdateProgressAsync( enumerateGameDetailsDirectoriesTask, ++current, gameDetailsCount, gameDetails.Title); directories.AddRange( await itemizeGameDetailsDirectoriesAsyncDelegate.ItemizeAsync( gameDetails, enumerateGameDetailsDirectoriesTask)); } await statusController.CompleteAsync(enumerateGameDetailsDirectoriesTask); return(directories); }
public async Task <IDictionary <long, IList <string> > > GetDownloadSourcesAsync(IStatus status) { var processUpdatesTask = await statusController.CreateAsync(status, "Process screenshots updates"); var screenshotsSources = new Dictionary <long, IList <string> >(); var current = 0; var total = await screenshotsDataController.CountAsync(processUpdatesTask); var processProductsScreenshotsTask = await statusController.CreateAsync(processUpdatesTask, "Process product screenshots"); foreach (var id in await screenshotsDataController.ItemizeAllAsync(processProductsScreenshotsTask)) { var productScreenshots = await screenshotsDataController.GetByIdAsync(id, processProductsScreenshotsTask); if (productScreenshots == null) { await statusController.WarnAsync(processProductsScreenshotsTask, $"Product {id} doesn't have screenshots"); continue; } await statusController.UpdateProgressAsync( processUpdatesTask, ++current, total, productScreenshots.Title); var currentProductScreenshotSources = new List <string>(); foreach (var uri in productScreenshots.Uris) { var sourceUri = formatScreenshotsUriDelegate.Format(uri); var destinationUri = Path.Combine( screenshotsDirectoryDelegate.GetDirectory(string.Empty), Path.GetFileName(sourceUri)); if (fileController.Exists(destinationUri)) { continue; } currentProductScreenshotSources.Add(sourceUri); } if (currentProductScreenshotSources.Any()) { screenshotsSources.Add(id, currentProductScreenshotSources); } } await statusController.CompleteAsync(processProductsScreenshotsTask); await statusController.CompleteAsync(processUpdatesTask); return(screenshotsSources); }
public async Task <IDictionary <long, IList <string> > > GetDownloadSourcesAsync(IStatus status) { var getDownloadSourcesStatus = await statusController.CreateAsync(status, "Get download sources"); var productImageSources = new Dictionary <long, IList <string> >(); var productIds = await itemizeAllProductsAsyncDelegate.ItemizeAllAsync(getDownloadSourcesStatus); var current = 0; foreach (var id in productIds) { await statusController.UpdateProgressAsync( getDownloadSourcesStatus, ++current, productIds.Count(), id.ToString()); var productCore = await dataController.GetByIdAsync(id, getDownloadSourcesStatus); // not all updated products can be found with all dataControllers if (productCore == null) { continue; } var imageSources = new List <string> { formatImagesUriDelegate.Format( getImageUriDelegate.GetImageUri(productCore)) }; if (!productImageSources.ContainsKey(id)) { productImageSources.Add(id, new List <string>()); } foreach (var source in imageSources) { productImageSources[id].Add(source); } } await statusController.CompleteAsync(getDownloadSourcesStatus); return(productImageSources); }
public async Task ConstrainAsync(int delaySeconds, IStatus status) { var throttleTask = await statusController.CreateAsync( status, $"Sleeping {formatSecondsDelegate.Format(delaySeconds)} before next operation"); for (var ii = 0; ii < delaySeconds; ii++) { await Task.Delay(1000); await statusController.UpdateProgressAsync( throttleTask, ii + 1, delaySeconds, "Countdown", TimeUnits.Seconds); } await statusController.CompleteAsync(throttleTask); }
public async Task DownloadFromResponseAsync(HttpResponseMessage response, string destination, IStatus status) { response.EnsureSuccessStatusCode(); var filename = response.RequestMessage.RequestUri.Segments.Last(); var fullPath = Path.Combine(destination, filename); int bufferSize = 1024 * 1024; // 1M byte[] buffer = new byte[bufferSize]; int bytesRead = 0; long totalBytesRead = 0; // don't redownload file with the same name and size if (fileController.Exists(fullPath) && fileController.GetSize(fullPath) == response.Content.Headers.ContentLength) { await statusController.InformAsync( status, $"File {fullPath} already exists and matches response size, will not be redownloading"); return; } using (var writeableStream = streamController.OpenWritable(fullPath)) using (var responseStream = await response.Content.ReadAsStreamAsync()) { while ((bytesRead = await responseStream.ReadAsync(buffer, 0, bufferSize)) > 0) { totalBytesRead += bytesRead; var contentLength = response.Content.Headers.ContentLength != null ? (long) response.Content.Headers.ContentLength : totalBytesRead; await writeableStream.WriteAsync(buffer, 0, bytesRead); await statusController.UpdateProgressAsync( status, totalBytesRead, contentLength, filename, DataUnits.Bytes); } } }
public async Task CommitAsync(IStatus status) { var commitTask = await statusController.CreateAsync(status, "Commit updated data"); // commit records controller if (recordsController != null) { var commitRecordsTask = await statusController.CreateAsync(commitTask, "Commit records"); await recordsController.CommitAsync(status); await statusController.CompleteAsync(commitRecordsTask); } var commitDataTask = await statusController.CreateAsync(commitTask, "Commit items"); await stashController.SaveAsync(commitDataTask); await statusController.CompleteAsync(commitDataTask); var additionalCommitsTask = await statusController.CreateAsync(commitTask, "Additional commit dependencies"); var current = 0; foreach (var commitDelegate in additionalCommitDelegates) { await statusController.UpdateProgressAsync( additionalCommitsTask, ++current, additionalCommitDelegates.Length, "Commit delegate"); await commitDelegate.CommitAsync(additionalCommitsTask); } await statusController.CompleteAsync(additionalCommitsTask); await statusController.CompleteAsync(commitTask); }
public async Task <IDictionary <long, IList <string> > > GetDownloadSourcesAsync(IStatus status) { var getDownloadSourcesStatus = await statusController.CreateAsync(status, "Get download sources"); var gameDetailsDownloadSources = new Dictionary <long, IList <string> >(); var current = 0; foreach (var id in await updatedDataController.ItemizeAllAsync(getDownloadSourcesStatus)) { await statusController.UpdateProgressAsync( getDownloadSourcesStatus, ++current, await updatedDataController.CountAsync(getDownloadSourcesStatus), id.ToString()); var gameDetails = await gameDetailsDataController.GetByIdAsync(id, getDownloadSourcesStatus); if (!gameDetailsDownloadSources.ContainsKey(id)) { gameDetailsDownloadSources.Add(id, new List <string>()); } foreach (var manualUrl in await itemizeGameDetailsManualUrlsAsyncController.ItemizeAsync(gameDetails, status)) { if (!gameDetailsDownloadSources[id].Contains(manualUrl)) { gameDetailsDownloadSources[id].Add(manualUrl); } } } await statusController.CompleteAsync(getDownloadSourcesStatus); return(gameDetailsDownloadSources); }
public async Task <IFileValidationResult> ValidateFileAsync(string productFileUri, string validationUri, IStatus status) { var fileValidation = new FileValidation { Filename = productFileUri }; if (string.IsNullOrEmpty(productFileUri)) { throw new ArgumentNullException(); } if (string.IsNullOrEmpty(validationUri)) { throw new ArgumentNullException(); } fileValidation.ValidationExpected = confirmValidationExpectedDelegate.Confirm(productFileUri); if (!fileValidation.ValidationExpected) { return(fileValidation); } fileValidation.ValidationFileExists = VerifyValidationFileExists(validationUri); fileValidation.ProductFileExists = VerifyProductFileExists(productFileUri); if (!fileValidation.ValidationFileExists || !fileValidation.ProductFileExists) { return(fileValidation); } try { using (var xmlStream = streamController.OpenReadable(validationUri)) validationXml.Load(xmlStream); fileValidation.ValidationFileIsValid = true; } catch { fileValidation.ValidationFileIsValid = false; return(fileValidation); } var fileElement = validationXml.GetElementsByTagName("file"); if (fileElement == null || fileElement.Count < 1 || fileElement[0] == null || fileElement[0].Attributes == null) { fileValidation.ValidationFileIsValid = false; return(fileValidation); } long expectedSize; string expectedName; int chunks; bool available; try { expectedSize = long.Parse(fileElement[0].Attributes["total_size"]?.Value); expectedName = fileElement[0].Attributes["name"]?.Value; chunks = int.Parse(fileElement[0].Attributes["chunks"]?.Value); available = fileElement[0].Attributes["available"]?.Value == "1"; if (!available) { var notAvailableMessage = fileElement[0].Attributes["notavailablemsg"]?.Value; throw new Exception(notAvailableMessage); } } catch { fileValidation.ValidationFileIsValid = false; return(fileValidation); } fileValidation.FilenameVerified = VerifyFilename(productFileUri, expectedName); fileValidation.SizeVerified = VerifySize(productFileUri, expectedSize); var chunksValidation = new List <IChunkValidation>(); using (var fileStream = streamController.OpenReadable(productFileUri)) { long length = 0; foreach (XmlNode chunkElement in fileElement[0].ChildNodes) { if (chunkElement.Name != "chunk") { continue; } long from, to = 0; string expectedMd5 = string.Empty; from = long.Parse(chunkElement.Attributes["from"]?.Value); to = long.Parse(chunkElement.Attributes["to"]?.Value); length += (to - from); expectedMd5 = chunkElement.FirstChild.Value; chunksValidation.Add(await VerifyChunkAsync(fileStream, from, to, expectedMd5, status)); await statusController.UpdateProgressAsync( status, length, expectedSize, productFileUri, DataUnits.Bytes); } fileValidation.Chunks = chunksValidation.ToArray(); await statusController.UpdateProgressAsync(status, length, expectedSize, productFileUri); } await statusController.InformAsync(status, $"Validation result: {productFileUri} is valid: " + $"{validationResultController.ProductFileIsValid(fileValidation)}"); return(fileValidation); }
public async Task <IList <T> > GetPageResultsAsync(IStatus status) { // GOG.com quirk // Products, AccountProducts use server-side paginated results, similar to Wikipedia. // This class encapsulates requesting sequence of pages up to the total number of pages. // Additionally page requests are filtered using hashes, so if response has the same hash // we would not deserialize it again - no point passing around same data. // Please note that ealier app versions also used heuristic optimization when // some page was unchanged - stop requesting next pages. This leads to stale data as GOG.com // changes information all the time updating Products and AccountProducts. It's especially // important for AccountProducts as GOG.com can set Updates information on any AccountProduct. // Updated is used for driving updated.json - that in turn is used for all subsequent operations // as an optimization - we don't process all products all the time, just updated var pageResults = new List <T>(); var currentPage = 1; var totalPages = 1; T pageResult = null; var getPagesTask = await statusController.CreateAsync(status, $"Request {context}"); do { var response = await requestPageAsyncDelegate.RequestPageAsync( requestUri, requestParameters, currentPage, getPagesTask); await statusController.UpdateProgressAsync( getPagesTask, currentPage, totalPages, requestUri, PageUnits.Pages); var requestHash = await storedHashController.GetHashAsync(requestUri + currentPage, getPagesTask); var responseHash = await getStringHashDelegate.GetHashAsync(response, getPagesTask); pageResult = serializationController.Deserialize <T>(response); if (pageResult == null) { continue; } totalPages = pageResult.TotalPages; if (responseHash == requestHash) { continue; } var setHashTask = await statusController.CreateAsync(getPagesTask, "Set hash", false); await storedHashController.SetHashAsync(requestUri + currentPage, responseHash, setHashTask); await statusController.CompleteAsync(setHashTask, false); pageResults.Add(pageResult); } while (++currentPage <= totalPages); await statusController.CompleteAsync(getPagesTask); return(pageResults); }