private async Task UploadFile(BackupFileUploadJob job)
{
var g = Guid.NewGuid().ToString().Substring(0, 8);
try
{
_logger.LogDebug($"{{{g}}} Uploading {job.UploadPath} on thread {Thread.CurrentThread.ManagedThreadId}");
var account = GetStorageAccount(job);
var client = account.CreateCloudBlobClient();
var container = client.GetContainerReference("blobblaze");
await container.CreateIfNotExistsAsync();
var blob = container.GetBlockBlobReference(job.UploadPath);
_logger.LogDebug($"{{{g}}} Using Cloud Storage Account {account.BlobStorageUri.PrimaryUri}");
_logger.LogDebug($"{{{g}}} Uploading to {container.Name}/{job.UploadPath}");
_logger.LogInformation(
$"{{{g}}} Beginning Upload of {job.UploadPath}. File Size is {ByteSize.FromBytes(job.LocalFile.Length).ToString()}");
var startTime = DateTime.UtcNow;
using (var fs = File.OpenRead(job.LocalFile.FullName))
{
await blob.UploadFromStreamAsync(fs, AccessCondition.GenerateEmptyCondition(), null, null, _cancellationTokenSource.Token);
}
job.Status = BackupFileUploadJobStatus.Successful;
var secondsTime = (DateTime.UtcNow - startTime).TotalSeconds;
_logger.LogInformation(
$"{{{g}}} Finished uploading {job.UploadPath} in {secondsTime:0.##}s. Average Speed was {ByteSize.FromBytes(job.LocalFile.Length / secondsTime).ToString()}/s.");
await TrackFile(job);
job.ParentJob.IncrementComplete();
await SetArchiveStatus(job, container);
// set to archive
}
catch (Exception e)
{
_logger.LogError($"{{{g}}} Exception occurred while uploading file {e.GetType().Name}: {e.Message} {e.StackTrace}");
await LogError(job, e);
if (job.RetryCount < MaxNumRetries)
{
_logger.LogInformation($"{{{g}}} Retrying job... {job.RetryCount} of {MaxNumRetries}");
_jobQueue.Add(job);
job.RetryCount++;
}
else
{
job.ParentJob.IncrementErrored();
}
}
finally
{
_uploadTaskSemaphore.Release();
}
}
private async Task TrackFile(BackupFileUploadJob job)
{
using (var context = _factory.CreateContext())
{
var alreadyTracked = await context.TrackedFiles.Include(f => f.BackupFolder)
.SingleOrDefaultAsync(f => f.FileName == job.LocalFile.FullName);
if (alreadyTracked != null)
{
context.Attach(alreadyTracked);
var version = new TrackedFileVersion(alreadyTracked);
await context.TrackedFileVersions.AddAsync(version, _cancellationTokenSource.Token);
alreadyTracked.UpdateFromFileInfo(job.LocalFile);
}
else // new file
{
var trackedFile = new TrackedFile(job);
await context.TrackedFiles.AddAsync(trackedFile, _cancellationTokenSource.Token);
}
await context.SaveChangesAsync(_cancellationTokenSource.Token);
}
}
private void QueueFilesAsPending(IReadOnlyCollection <string> files, BackupFolderJob job)
{
foreach (var file in files)
{
var fileJob = new BackupFileUploadJob(job, new System.IO.FileInfo(file));
_uploadProcessor.AddJob(fileJob);
}
}
private async Task OnUploadComplete(BackupFileUploadJob job)
{
using (var context = _factory.CreateContext())
{
context.BackupJobs.Attach(job.ParentJob);
if (job.ParentJob.Status != BackupFolderJobStatus.InProgress)
{
context.Entry(job.ParentJob).State = EntityState.Modified;
_logger.LogInformation($"Finished Processing all files for folder {job.ParentJob.Folder.Name} at path {job.ParentJob.Folder.Path}");
}
await context.SaveChangesAsync();
}
}
private async Task LogError(BackupFileUploadJob job, Exception e)
{
try
{
using (var context = _factory.CreateContext())
{
var uploadError = new UploadError(job, e);
await context.UploadErrors.AddAsync(uploadError);
await context.SaveChangesAsync();
}
}
catch (Exception)
{
// this can't throw
}
}
private async Task SetArchiveStatus(BackupFileUploadJob job, CloudBlobContainer container)
{
// Emulator does not support access tiers
if (GetStorageAccount(job) == CloudStorageAccount.DevelopmentStorageAccount)
{
return;
}
try
{
var blockReference = container.GetBlockBlobReference(job.UploadPath);
await blockReference.SetStandardBlobTierAsync(StandardBlobTier.Archive);
}
catch (Exception e)
{
_logger.LogError("Failed to Set Blob Access Level for {file}, {message}", job.UploadPath, e.Message);
}
}
private CloudStorageAccount GetStorageAccount(BackupFileUploadJob job)
{
CloudStorageAccount account;
if (job.ParentJob.Folder.Remote.ConnectionString.IsDevelopment)
{
account = CloudStorageAccount.DevelopmentStorageAccount;
}
else
{
if (!CloudStorageAccount.TryParse(job.ParentJob.Folder.Remote.ConnectionString.ToString(), out account))
{
throw new InvalidOperationException("Connection String was Invalid");
}
}
return(account);
}
public void AddJob(BackupFileUploadJob job)
{
_jobQueue.Add(job);
}
