public async Task <string> PullAsync(string uri) { var started = DateTime.Now; var data = string.Empty; if (fileController.Exists(uri)) { using (var stream = streamController.OpenReadable(uri)) using (StreamReader reader = new StreamReader(stream)) data = await reader.ReadToEndAsync(); } if (traceDelegate == null) { return(data); } var completed = DateTime.Now; var duration = (completed - started).TotalMilliseconds; traceDelegate?.Trace( "Pull", started.ToFileTimeUtc().ToString(), completed.ToFileTimeUtc().ToString(), duration.ToString(), uri); return(data); }
public async Task <T> DeserializePullAsync <T>(string uri, IStatus status) { var started = DateTime.Now; var deserializePullTask = await statusController.CreateAsync(status, "Reading serialized data", false); T data = default(T); if (fileController.Exists(uri)) { using (var readableStream = streamController.OpenReadable(uri)) data = Serializer.Deserialize <T>(readableStream); } await statusController.CompleteAsync(deserializePullTask, false); var completed = DateTime.Now; var duration = (completed - started).TotalMilliseconds; traceDelegate?.Trace( "DePull", started.ToFileTimeUtc().ToString(), completed.ToFileTimeUtc().ToString(), duration.ToString(), uri); return(data); }
public async Task DownloadProductFileAsync(long id, string title, string sourceUri, string destination, IStatus status) { if (string.IsNullOrEmpty(sourceUri)) { return; } var sourceUriSansSession = formatUriRemoveSessionDelegate.Format(sourceUri); var destinationUri = formatValidationFileDelegate.Format(sourceUriSansSession); // return early if validation is not expected for this file if (!confirmValidationExpectedDelegate.Confirm(sourceUriSansSession)) { return; } if (fileController.Exists(destinationUri)) { await statusController.InformAsync(status, "Validation file already exists, will not be redownloading"); return; } var validationSourceUri = formatValidationUriDelegate.Format(sourceUri); var downloadValidationFileTask = await statusController.CreateAsync(status, "Download validation file"); await downloadFromUriAsyncDelegate.DownloadFromUriAsync( validationSourceUri, validationDirectoryDelegate.GetDirectory(string.Empty), downloadValidationFileTask); await statusController.CompleteAsync(downloadValidationFileTask); }
public override async Task ProcessActivityAsync(IStatus status) { var validateDataTask = await statusController.CreateAsync(status, "Validate data"); var dataFiles = await storedHashController.ItemizeAllAsync(validateDataTask); var dataFilesCount = dataFiles.Count(); var current = 0; foreach (var dataFile in dataFiles) { if (!fileController.Exists(dataFile)) { continue; } await statusController.UpdateProgressAsync(validateDataTask, ++current, dataFilesCount, dataFile); var precomputedHash = await storedHashController.GetHashAsync(dataFile, validateDataTask); if (!await fileValidateDelegate.ValidateFileAsync(dataFile, precomputedHash, validateDataTask)) { await statusController.WarnAsync(validateDataTask, $"Data file {dataFile} hash doesn't match precomputed value"); } } await statusController.CompleteAsync(validateDataTask); }
public async Task <IDictionary <long, IList <string> > > GetDownloadSourcesAsync(IStatus status) { var processUpdatesTask = await statusController.CreateAsync(status, "Process screenshots updates"); var screenshotsSources = new Dictionary <long, IList <string> >(); var current = 0; var total = await screenshotsDataController.CountAsync(processUpdatesTask); var processProductsScreenshotsTask = await statusController.CreateAsync(processUpdatesTask, "Process product screenshots"); foreach (var id in await screenshotsDataController.ItemizeAllAsync(processProductsScreenshotsTask)) { var productScreenshots = await screenshotsDataController.GetByIdAsync(id, processProductsScreenshotsTask); if (productScreenshots == null) { await statusController.WarnAsync(processProductsScreenshotsTask, $"Product {id} doesn't have screenshots"); continue; } await statusController.UpdateProgressAsync( processUpdatesTask, ++current, total, productScreenshots.Title); var currentProductScreenshotSources = new List <string>(); foreach (var uri in productScreenshots.Uris) { var sourceUri = formatScreenshotsUriDelegate.Format(uri); var destinationUri = Path.Combine( screenshotsDirectoryDelegate.GetDirectory(string.Empty), Path.GetFileName(sourceUri)); if (fileController.Exists(destinationUri)) { continue; } currentProductScreenshotSources.Add(sourceUri); } if (currentProductScreenshotSources.Any()) { screenshotsSources.Add(id, currentProductScreenshotSources); } } await statusController.CompleteAsync(processProductsScreenshotsTask); await statusController.CompleteAsync(processUpdatesTask); return(screenshotsSources); }
public async Task DownloadFromResponseAsync(HttpResponseMessage response, string destination, IStatus status) { response.EnsureSuccessStatusCode(); var filename = response.RequestMessage.RequestUri.Segments.Last(); var fullPath = Path.Combine(destination, filename); int bufferSize = 1024 * 1024; // 1M byte[] buffer = new byte[bufferSize]; int bytesRead = 0; long totalBytesRead = 0; // don't redownload file with the same name and size if (fileController.Exists(fullPath) && fileController.GetSize(fullPath) == response.Content.Headers.ContentLength) { await statusController.InformAsync( status, $"File {fullPath} already exists and matches response size, will not be redownloading"); return; } using (var writeableStream = streamController.OpenWritable(fullPath)) using (var responseStream = await response.Content.ReadAsStreamAsync()) { while ((bytesRead = await responseStream.ReadAsync(buffer, 0, bufferSize)) > 0) { totalBytesRead += bytesRead; var contentLength = response.Content.Headers.ContentLength != null ? (long) response.Content.Headers.ContentLength : totalBytesRead; await writeableStream.WriteAsync(buffer, 0, bytesRead); await statusController.UpdateProgressAsync( status, totalBytesRead, contentLength, filename, DataUnits.Bytes); } } }
public bool VerifyProductFileExists(string productFileUri) { return(fileController.Exists(productFileUri)); }
public override async Task ProcessActivityAsync(IStatus status) { var updateDownloadsTask = await statusController.CreateAsync( status, $"Update {context} downloads"); var getSourcesTask = await statusController.CreateAsync( updateDownloadsTask, $"Get {context} download sources"); var downloadSources = await getDownloadSourcesAsyncDelegate.GetDownloadSourcesAsync(getSourcesTask); await statusController.CompleteAsync(getSourcesTask); var counter = 0; foreach (var downloadSource in downloadSources) { // don't perform expensive updates if there are no actual sources if (downloadSource.Value != null && downloadSource.Value.Count == 0) { continue; } var id = downloadSource.Key; ProductCore product = await productsDataController.GetByIdAsync(id, updateDownloadsTask); if (product == null) { product = await accountProductsDataController.GetByIdAsync(id, updateDownloadsTask); if (product == null) { await statusController.WarnAsync( updateDownloadsTask, $"Downloads are scheduled for the product/account product {id} that doesn't exist"); continue; } } await statusController.UpdateProgressAsync( updateDownloadsTask, ++counter, downloadSources.Count, product.Title); var productDownloads = await productDownloadsDataController.GetByIdAsync(product.Id, updateDownloadsTask); if (productDownloads == null) { productDownloads = new ProductDownloads { Id = product.Id, Title = product.Title, Downloads = new List <ProductDownloadEntry>() }; } // purge existing downloads for this download type as we'll always be scheduling all files we need to download // and don't want to carry over any previously scheduled files that might not be relevant anymore // (e.g. files that were scheduled, but never downloaded and then removed from data files) var existingDownloadsOfType = productDownloads.Downloads.FindAll( d => d.Context == context).ToArray(); foreach (var download in existingDownloadsOfType) { productDownloads.Downloads.Remove(download); } var scheduleDownloadsTask = await statusController.CreateAsync( updateDownloadsTask, "Schedule new downloads"); foreach (var source in downloadSource.Value) { var destinationDirectory = getDirectoryDelegate?.GetDirectory(source); var scheduledDownloadEntry = new ProductDownloadEntry { Context = context, SourceUri = source, Destination = destinationDirectory }; var destinationUri = Path.Combine( destinationDirectory, Path.GetFileName(source)); // we won't schedule downloads for the already existing files // we won't be able to resolve filename for productFiles, but that should cut off // number of images we constantly try to redownload if (fileController.Exists(destinationUri)) { continue; } productDownloads.Downloads.Add(scheduledDownloadEntry); } await productDownloadsDataController.UpdateAsync(productDownloads, scheduleDownloadsTask); await statusController.CompleteAsync(scheduleDownloadsTask); } await statusController.CompleteAsync(updateDownloadsTask); }