private async Task UploadFile(BackupFileUploadJob job) { var g = Guid.NewGuid().ToString().Substring(0, 8); try { _logger.LogDebug($"{{{g}}} Uploading {job.UploadPath} on thread {Thread.CurrentThread.ManagedThreadId}"); var account = GetStorageAccount(job); var client = account.CreateCloudBlobClient(); var container = client.GetContainerReference("blobblaze"); await container.CreateIfNotExistsAsync(); var blob = container.GetBlockBlobReference(job.UploadPath); _logger.LogDebug($"{{{g}}} Using Cloud Storage Account {account.BlobStorageUri.PrimaryUri}"); _logger.LogDebug($"{{{g}}} Uploading to {container.Name}/{job.UploadPath}"); _logger.LogInformation( $"{{{g}}} Beginning Upload of {job.UploadPath}. File Size is {ByteSize.FromBytes(job.LocalFile.Length).ToString()}"); var startTime = DateTime.UtcNow; using (var fs = File.OpenRead(job.LocalFile.FullName)) { await blob.UploadFromStreamAsync(fs, AccessCondition.GenerateEmptyCondition(), null, null, _cancellationTokenSource.Token); } job.Status = BackupFileUploadJobStatus.Successful; var secondsTime = (DateTime.UtcNow - startTime).TotalSeconds; _logger.LogInformation( $"{{{g}}} Finished uploading {job.UploadPath} in {secondsTime:0.##}s. Average Speed was {ByteSize.FromBytes(job.LocalFile.Length / secondsTime).ToString()}/s."); await TrackFile(job); job.ParentJob.IncrementComplete(); await SetArchiveStatus(job, container); // set to archive } catch (Exception e) { _logger.LogError($"{{{g}}} Exception occurred while uploading file {e.GetType().Name}: {e.Message} {e.StackTrace}"); await LogError(job, e); if (job.RetryCount < MaxNumRetries) { _logger.LogInformation($"{{{g}}} Retrying job... {job.RetryCount} of {MaxNumRetries}"); _jobQueue.Add(job); job.RetryCount++; } else { job.ParentJob.IncrementErrored(); } } finally { _uploadTaskSemaphore.Release(); } }
private async Task TrackFile(BackupFileUploadJob job) { using (var context = _factory.CreateContext()) { var alreadyTracked = await context.TrackedFiles.Include(f => f.BackupFolder) .SingleOrDefaultAsync(f => f.FileName == job.LocalFile.FullName); if (alreadyTracked != null) { context.Attach(alreadyTracked); var version = new TrackedFileVersion(alreadyTracked); await context.TrackedFileVersions.AddAsync(version, _cancellationTokenSource.Token); alreadyTracked.UpdateFromFileInfo(job.LocalFile); } else // new file { var trackedFile = new TrackedFile(job); await context.TrackedFiles.AddAsync(trackedFile, _cancellationTokenSource.Token); } await context.SaveChangesAsync(_cancellationTokenSource.Token); } }
private void QueueFilesAsPending(IReadOnlyCollection <string> files, BackupFolderJob job) { foreach (var file in files) { var fileJob = new BackupFileUploadJob(job, new System.IO.FileInfo(file)); _uploadProcessor.AddJob(fileJob); } }
private async Task OnUploadComplete(BackupFileUploadJob job) { using (var context = _factory.CreateContext()) { context.BackupJobs.Attach(job.ParentJob); if (job.ParentJob.Status != BackupFolderJobStatus.InProgress) { context.Entry(job.ParentJob).State = EntityState.Modified; _logger.LogInformation($"Finished Processing all files for folder {job.ParentJob.Folder.Name} at path {job.ParentJob.Folder.Path}"); } await context.SaveChangesAsync(); } }
private async Task LogError(BackupFileUploadJob job, Exception e) { try { using (var context = _factory.CreateContext()) { var uploadError = new UploadError(job, e); await context.UploadErrors.AddAsync(uploadError); await context.SaveChangesAsync(); } } catch (Exception) { // this can't throw } }
private async Task SetArchiveStatus(BackupFileUploadJob job, CloudBlobContainer container) { // Emulator does not support access tiers if (GetStorageAccount(job) == CloudStorageAccount.DevelopmentStorageAccount) { return; } try { var blockReference = container.GetBlockBlobReference(job.UploadPath); await blockReference.SetStandardBlobTierAsync(StandardBlobTier.Archive); } catch (Exception e) { _logger.LogError("Failed to Set Blob Access Level for {file}, {message}", job.UploadPath, e.Message); } }
private CloudStorageAccount GetStorageAccount(BackupFileUploadJob job) { CloudStorageAccount account; if (job.ParentJob.Folder.Remote.ConnectionString.IsDevelopment) { account = CloudStorageAccount.DevelopmentStorageAccount; } else { if (!CloudStorageAccount.TryParse(job.ParentJob.Folder.Remote.ConnectionString.ToString(), out account)) { throw new InvalidOperationException("Connection String was Invalid"); } } return(account); }
public void AddJob(BackupFileUploadJob job) { _jobQueue.Add(job); }