public async Task <string> PullAsync(string uri)
{
var started = DateTime.Now;
var data = string.Empty;
if (fileController.Exists(uri))
{
using (var stream = streamController.OpenReadable(uri))
using (StreamReader reader = new StreamReader(stream))
data = await reader.ReadToEndAsync();
}
if (traceDelegate == null)
{
return(data);
}
var completed = DateTime.Now;
var duration = (completed - started).TotalMilliseconds;
traceDelegate?.Trace(
"Pull",
started.ToFileTimeUtc().ToString(),
completed.ToFileTimeUtc().ToString(),
duration.ToString(),
uri);
return(data);
}
public async Task <T> DeserializePullAsync <T>(string uri, IStatus status)
{
var started = DateTime.Now;
var deserializePullTask = await statusController.CreateAsync(status, "Reading serialized data", false);
T data = default(T);
if (fileController.Exists(uri))
{
using (var readableStream = streamController.OpenReadable(uri))
data = Serializer.Deserialize <T>(readableStream);
}
await statusController.CompleteAsync(deserializePullTask, false);
var completed = DateTime.Now;
var duration = (completed - started).TotalMilliseconds;
traceDelegate?.Trace(
"DePull",
started.ToFileTimeUtc().ToString(),
completed.ToFileTimeUtc().ToString(),
duration.ToString(),
uri);
return(data);
}
public async Task DownloadProductFileAsync(long id, string title, string sourceUri, string destination, IStatus status)
{
if (string.IsNullOrEmpty(sourceUri))
{
return;
}
var sourceUriSansSession = formatUriRemoveSessionDelegate.Format(sourceUri);
var destinationUri = formatValidationFileDelegate.Format(sourceUriSansSession);
// return early if validation is not expected for this file
if (!confirmValidationExpectedDelegate.Confirm(sourceUriSansSession))
{
return;
}
if (fileController.Exists(destinationUri))
{
await statusController.InformAsync(status, "Validation file already exists, will not be redownloading");
return;
}
var validationSourceUri = formatValidationUriDelegate.Format(sourceUri);
var downloadValidationFileTask = await statusController.CreateAsync(status, "Download validation file");
await downloadFromUriAsyncDelegate.DownloadFromUriAsync(
validationSourceUri,
validationDirectoryDelegate.GetDirectory(string.Empty),
downloadValidationFileTask);
await statusController.CompleteAsync(downloadValidationFileTask);
}
public override async Task ProcessActivityAsync(IStatus status)
{
var validateDataTask = await statusController.CreateAsync(status, "Validate data");
var dataFiles = await storedHashController.ItemizeAllAsync(validateDataTask);
var dataFilesCount = dataFiles.Count();
var current = 0;
foreach (var dataFile in dataFiles)
{
if (!fileController.Exists(dataFile))
{
continue;
}
await statusController.UpdateProgressAsync(validateDataTask,
++current,
dataFilesCount,
dataFile);
var precomputedHash = await storedHashController.GetHashAsync(dataFile, validateDataTask);
if (!await fileValidateDelegate.ValidateFileAsync(dataFile, precomputedHash, validateDataTask))
{
await statusController.WarnAsync(validateDataTask, $"Data file {dataFile} hash doesn't match precomputed value");
}
}
await statusController.CompleteAsync(validateDataTask);
}
public async Task <IDictionary <long, IList <string> > > GetDownloadSourcesAsync(IStatus status)
{
var processUpdatesTask = await statusController.CreateAsync(status, "Process screenshots updates");
var screenshotsSources = new Dictionary <long, IList <string> >();
var current = 0;
var total = await screenshotsDataController.CountAsync(processUpdatesTask);
var processProductsScreenshotsTask = await statusController.CreateAsync(processUpdatesTask, "Process product screenshots");
foreach (var id in await screenshotsDataController.ItemizeAllAsync(processProductsScreenshotsTask))
{
var productScreenshots = await screenshotsDataController.GetByIdAsync(id, processProductsScreenshotsTask);
if (productScreenshots == null)
{
await statusController.WarnAsync(processProductsScreenshotsTask, $"Product {id} doesn't have screenshots");
continue;
}
await statusController.UpdateProgressAsync(
processUpdatesTask,
++current,
total,
productScreenshots.Title);
var currentProductScreenshotSources = new List <string>();
foreach (var uri in productScreenshots.Uris)
{
var sourceUri = formatScreenshotsUriDelegate.Format(uri);
var destinationUri = Path.Combine(
screenshotsDirectoryDelegate.GetDirectory(string.Empty),
Path.GetFileName(sourceUri));
if (fileController.Exists(destinationUri))
{
continue;
}
currentProductScreenshotSources.Add(sourceUri);
}
if (currentProductScreenshotSources.Any())
{
screenshotsSources.Add(id, currentProductScreenshotSources);
}
}
await statusController.CompleteAsync(processProductsScreenshotsTask);
await statusController.CompleteAsync(processUpdatesTask);
return(screenshotsSources);
}
public async Task DownloadFromResponseAsync(HttpResponseMessage response, string destination, IStatus status)
{
response.EnsureSuccessStatusCode();
var filename = response.RequestMessage.RequestUri.Segments.Last();
var fullPath = Path.Combine(destination, filename);
int bufferSize = 1024 * 1024; // 1M
byte[] buffer = new byte[bufferSize];
int bytesRead = 0;
long totalBytesRead = 0;
// don't redownload file with the same name and size
if (fileController.Exists(fullPath) &&
fileController.GetSize(fullPath) == response.Content.Headers.ContentLength)
{
await statusController.InformAsync(
status,
$"File {fullPath} already exists and matches response size, will not be redownloading");
return;
}
using (var writeableStream = streamController.OpenWritable(fullPath))
using (var responseStream = await response.Content.ReadAsStreamAsync())
{
while ((bytesRead = await responseStream.ReadAsync(buffer, 0, bufferSize)) > 0)
{
totalBytesRead += bytesRead;
var contentLength = response.Content.Headers.ContentLength != null ?
(long) response.Content.Headers.ContentLength :
totalBytesRead;
await writeableStream.WriteAsync(buffer, 0, bytesRead);
await statusController.UpdateProgressAsync(
status,
totalBytesRead,
contentLength,
filename,
DataUnits.Bytes);
}
}
}
public bool VerifyProductFileExists(string productFileUri)
{
return(fileController.Exists(productFileUri));
}
public override async Task ProcessActivityAsync(IStatus status)
{
var updateDownloadsTask = await statusController.CreateAsync(
status,
$"Update {context} downloads");
var getSourcesTask = await statusController.CreateAsync(
updateDownloadsTask,
$"Get {context} download sources");
var downloadSources = await getDownloadSourcesAsyncDelegate.GetDownloadSourcesAsync(getSourcesTask);
await statusController.CompleteAsync(getSourcesTask);
var counter = 0;
foreach (var downloadSource in downloadSources)
{
// don't perform expensive updates if there are no actual sources
if (downloadSource.Value != null &&
downloadSource.Value.Count == 0)
{
continue;
}
var id = downloadSource.Key;
ProductCore product = await productsDataController.GetByIdAsync(id, updateDownloadsTask);
if (product == null)
{
product = await accountProductsDataController.GetByIdAsync(id, updateDownloadsTask);
if (product == null)
{
await statusController.WarnAsync(
updateDownloadsTask,
$"Downloads are scheduled for the product/account product {id} that doesn't exist");
continue;
}
}
await statusController.UpdateProgressAsync(
updateDownloadsTask,
++counter,
downloadSources.Count,
product.Title);
var productDownloads = await productDownloadsDataController.GetByIdAsync(product.Id, updateDownloadsTask);
if (productDownloads == null)
{
productDownloads = new ProductDownloads
{
Id = product.Id,
Title = product.Title,
Downloads = new List <ProductDownloadEntry>()
};
}
// purge existing downloads for this download type as we'll always be scheduling all files we need to download
// and don't want to carry over any previously scheduled files that might not be relevant anymore
// (e.g. files that were scheduled, but never downloaded and then removed from data files)
var existingDownloadsOfType = productDownloads.Downloads.FindAll(
d => d.Context == context).ToArray();
foreach (var download in existingDownloadsOfType)
{
productDownloads.Downloads.Remove(download);
}
var scheduleDownloadsTask = await statusController.CreateAsync(
updateDownloadsTask,
"Schedule new downloads");
foreach (var source in downloadSource.Value)
{
var destinationDirectory = getDirectoryDelegate?.GetDirectory(source);
var scheduledDownloadEntry = new ProductDownloadEntry
{
Context = context,
SourceUri = source,
Destination = destinationDirectory
};
var destinationUri = Path.Combine(
destinationDirectory,
Path.GetFileName(source));
// we won't schedule downloads for the already existing files
// we won't be able to resolve filename for productFiles, but that should cut off
// number of images we constantly try to redownload
if (fileController.Exists(destinationUri))
{
continue;
}
productDownloads.Downloads.Add(scheduledDownloadEntry);
}
await productDownloadsDataController.UpdateAsync(productDownloads, scheduleDownloadsTask);
await statusController.CompleteAsync(scheduleDownloadsTask);
}
await statusController.CompleteAsync(updateDownloadsTask);
}