/// <summary>
/// Parse Blob Event
/// </summary>
/// <param name="eventData"></param>
/// <param name="eventId"></param>
/// <param name="eventDateTime"></param>
/// <returns></returns>
public static IBlobEvent ParseBlobEvent(string eventData, out string eventId, out DateTime eventDateTime)
{
JObject jsonEventData = JObject.Parse(eventData);
string eventType = jsonEventData["eventType"].ToString();
eventId = string.Empty; eventDateTime = DateTime.MinValue;
if (string.Equals(eventType, Constants.Constants.BlobEventType.BLOBCREATED, StringComparison.InvariantCultureIgnoreCase))
{
BlobEvent <CreatedEventData> createdEvent = jsonEventData.ToObject <BlobEvent <CreatedEventData> >();
eventId = createdEvent.id;
eventDateTime = createdEvent.eventTime;
return(createdEvent);
}
else if (string.Equals(eventType, Constants.Constants.BlobEventType.BLOBDELETED, StringComparison.InvariantCultureIgnoreCase))
{
BlobEvent <DeletedEventData> deletedEvent = jsonEventData.ToObject <BlobEvent <DeletedEventData> >();
eventId = deletedEvent.id;
eventDateTime = deletedEvent.eventTime;
return(deletedEvent);
}
return(null);
}
public void DrawTest()
{
// arrange
string specification = @"
⬇10
E10R80E10
78×E10RE78RE10
E10R80E10
⬇10
";
var p = PixelOperations <Rgb24> .Instance.GetPixelBlender(new GraphicsOptions());
var green = Color.FromRgba(0, 255, 0, 128);
this.ExpectedImage = new TestImage(100, 100, Color.Black)
.FillPattern(specification)
// the point 5.1 seconds and 520 Hz should match 51, 48
.GoTo(51, 48)
.Fill(1, 1, p.Blend(Color.Black, green, 0.5f))
.GoTo(52, 49)
.Fill(2, 1, p.Blend(Color.Black, green, 0.5f))
.GoTo(12, 87)
.Fill(1, 1, p.Blend(Color.Black, green, 0.5f))
.Finish();
// BUG: with DrawPointsAsFill: overlaps are painted twice
this.ExpectedImage[52, 49] = new Rgb24(0, 192, 0);
var @event = new BlobEvent()
{
EventEndSeconds = 9,
EventStartSeconds = 1,
HighFrequencyHertz = 900,
LowFrequencyHertz = 100,
};
@event.Points.Add(new SpectralPoint((5.1, 5.2), (510, 520), 0.9));
@event.Points.Add(new SpectralPoint((5.2, 5.3), (500, 510), 0.9));
// double wide, overlaps with previous
@event.Points.Add(new SpectralPoint((5.2, 5.4), (500, 510), 0.9));
@event.Points.Add(new SpectralPoint((1.2, 1.3), (120, 130), 0.9));
var options = new EventRenderingOptions(new UnitConverters(0, 10, 1000, 100, 100))
{
// disable the default blend to make testing easier
FillOptions = new GraphicsOptions(),
Fill = Brushes.Solid(green),
};
// act
this.ActualImage.Mutate(x => @event.Draw(x, options));
// assert
this.AssertImagesEqual();
}
private static void CompleteBlobEvent(BlobEvent @event, RequestData request)
{
if (request != null)
{
@event.Url = request.Url;
@event.UserAgent = request.UserAgent;
@event.RemoteMachine = request.RemoteMachine;
@event.SessionId = request.SessionId;
}
}
public static EventCommon[] CallAed(BaseSonogram sonogram, AedConfiguration aedConfiguration, TimeSpan segmentStartOffset, TimeSpan segmentDuration)
{
Log.Info("AED start");
var aedOptions = new AedOptions(sonogram.Configuration.NyquistFreq)
{
IntensityThreshold = aedConfiguration.IntensityThreshold,
SmallAreaThreshold = aedConfiguration.SmallAreaThreshold,
DoNoiseRemoval = aedConfiguration.NoiseReductionType == NoiseReductionType.None,
};
if (aedConfiguration.BandpassMinimum.HasValue && aedConfiguration.BandpassMaximum.HasValue)
{
var bandPassFilter =
Tuple.Create(
(double)aedConfiguration.BandpassMinimum.Value,
(double)aedConfiguration.BandpassMaximum.Value);
aedOptions.BandPassFilter = new FSharpOption <Tuple <double, double> >(bandPassFilter);
}
IEnumerable <Oblong> oblongs = AcousticEventDetection.detectEvents(aedOptions, sonogram.Data);
Log.Info("AED finished");
var unitConverters = new UnitConverters(
segmentStartOffset.TotalSeconds,
sonogram.SampleRate,
sonogram.Configuration.WindowSize,
sonogram.Configuration.WindowOverlap);
var events = oblongs.Select(
o =>
{
var blob = new BlobEvent()
{
SegmentDurationSeconds = segmentDuration.TotalSeconds,
Name = "AED Event",
};
unitConverters.SetBounds(blob, o);
if (aedConfiguration.IncludeHitElementsInOutput)
{
o.HitElements
.Select(p => unitConverters.ConvertPointToSpectralPoint(p, 1.0))
.ForEach(sp => blob.Points.Add(sp));
}
return(blob);
});
return(events.ToArray());
}
public void CreateEventUserDownloadBlob(User user, Blob blob, RequestData request)
{
BlobEvent @event = new BlobEvent()
{
BlobEventId = Guid.NewGuid(),
EventDateTime = DateTime.UtcNow,
EventType = (int)EventType.Download,
User = user,
Blob = blob
};
CompleteBlobEvent(@event, request);
this.context.BlobEvents.AddObject(@event);
this.context.SaveChanges();
}
public void DerivedPropertiesTest()
{
var @event = new BlobEvent()
{
EventEndSeconds = 9,
EventStartSeconds = 1,
HighFrequencyHertz = 900,
LowFrequencyHertz = 100,
};
@event.Points.Add(new SpectralPoint((5.1, 5.2), (510, 520), 0.9));
Assert.AreEqual(8, @event.EventDurationSeconds);
Assert.AreEqual(800, @event.BandWidthHertz);
Assert.AreEqual(1, @event.Points.Count);
}
/// <summary>
/// Deletes the blob from the target / restore storage account
/// </summary>
/// <param name="eventData"></param>
/// <returns></returns>
public async Task <bool> DeleteBlobFromRestore(IBlobEvent eventData)
{
if (eventData is BlobEvent <DeletedEventData> )
{
string destinationStorageAccountConnectionString =
Environment.GetEnvironmentVariable("SRC_STORAGE_ACCOUNT_CONN");
// Retrieve the storage account from the connection string.
CloudStorageAccount destinationStorageAccount = CloudStorageAccount.Parse(destinationStorageAccountConnectionString);
CloudBlobClient destinationBlobClient = destinationStorageAccount.CreateCloudBlobClient();
BlobEvent <DeletedEventData> deletedEventData = (BlobEvent <DeletedEventData>)eventData;
//creating sourceBlockBlob to get the container name and blob name
CloudBlockBlob sourceBlockBlob = new CloudBlockBlob(new Uri(deletedEventData.data.url));
CloudBlobContainer destinationContainer = destinationBlobClient.GetContainerReference(sourceBlockBlob.Container.Name);
bool destinationContainerExists = await destinationContainer.ExistsAsync();
if (destinationContainerExists)
{
CloudBlockBlob blockBlobToDelete = destinationContainer.GetBlockBlobReference(sourceBlockBlob.Name);
bool blobDeleted = await blockBlobToDelete.DeleteIfExistsAsync();
if (blobDeleted)
{
_logger.LogInformation($"Successfully deleted blob: {sourceBlockBlob.Name} in destination (restore) storage account.");
}
else
{
_logger.LogInformation($"Failed to delete blob: {sourceBlockBlob.Name} in destination (restore) storage account.");
}
return(blobDeleted);
}
else
{
_logger.LogInformation($"Not able to locate the container: {sourceBlockBlob.Container.Name} in destination (restore) storage account.");
}
}
return(false);
}
/// <summary>
/// CopyBlobFromSourceToBackup:
/// - Copies blob from source SA container to destination/backup SA container
/// </summary>
/// <returns></returns>
public async Task <DestinationBlobInfo> CopyBlobFromSourceToBackup(IBlobEvent eventData)
{
DestinationBlobInfo destinationBlobInfo = null;
string destinationStorageAccountConnectionString =
Environment.GetEnvironmentVariable("RES_STORAGE_ACCOUNT_CONN");
string sourceStorageAccountConnectionString =
Environment.GetEnvironmentVariable("SRC_STORAGE_ACCOUNT_CONN");
if (eventData is BlobEvent <CreatedEventData> )
{
// Retrieve the storage account from the connection string.
CloudStorageAccount sourceStorageAccount = CloudStorageAccount.Parse(sourceStorageAccountConnectionString);
CloudBlobClient sourceBlobClient = sourceStorageAccount.CreateCloudBlobClient();
// Retrieve the storage account from the connection string.
CloudStorageAccount destinationStorageAccount = CloudStorageAccount.Parse(destinationStorageAccountConnectionString);
CloudBlobClient destinationBlobClient = destinationStorageAccount.CreateCloudBlobClient();
BlobEvent <CreatedEventData> createdEventData = (BlobEvent <CreatedEventData>)eventData;
string url = createdEventData.data.url;
CloudBlockBlob sourceBlockBlob = new CloudBlockBlob(new Uri(url), sourceBlobClient);
bool sourceBlobExists = await sourceBlockBlob.ExistsAsync();
if (sourceBlobExists)
{
long blobSize = sourceBlockBlob.Properties.Length;
EventDateDetails dateDetails = new EventDateDetails(createdEventData.eventTime);
string destinationContaninerName = dateDetails.year.ToString();
string destinationBlobName = $"wk{dateDetails.WeekNumber}/dy{(int)dateDetails.DayOfWeek}/{sourceBlockBlob.Container.Name}/{sourceBlockBlob.Name}";
destinationBlobName += ".";
destinationBlobName += DateTimeUtil.GetString;
CloudBlobContainer destinationContainer = destinationBlobClient.GetContainerReference(destinationContaninerName);
bool result = await destinationContainer.CreateIfNotExistsAsync();
CloudBlockBlob destinationBlob = destinationContainer.GetBlockBlobReference(destinationBlobName);
_logger.LogInformation($"About to sync copy from Container: {sourceBlockBlob.Container.Name}, Blob: {sourceBlockBlob.Name}. Blob size: {sourceBlockBlob.Properties.Length} bytes");
// copyResult = "SYNCCOPY";
string copyResult = await destinationBlob.StartCopyAsync(sourceBlockBlob);
destinationBlobInfo = new DestinationBlobInfo();
destinationBlobInfo.ContainerName = destinationContainer.Name;
destinationBlobInfo.BlobName = destinationBlobName;
destinationBlobInfo.CopyReferenceId = copyResult;
destinationBlobInfo.OrgContainerName = sourceBlockBlob.Container.Name;
destinationBlobInfo.OrgBlobName = sourceBlockBlob.Name;
_logger.LogInformation($"Copy Scheduled. Source Blob Name: {sourceBlockBlob.Name}, Destination Blob Name: {destinationBlobInfo.BlobName}, Copy Id: {copyResult}.");
return(destinationBlobInfo);
}
else
{
_logger.LogInformation($"Not able to locate the block blob in source storage account---Block blob Name: {sourceBlockBlob.Name}");
}
}
else
{
_logger.LogInformation($"Input event data is not of Created Event Type.");
}
return(destinationBlobInfo);
}
/// <summary>
/// CopyBlobFromSourceToBackup:
/// - Copies blob from source SA container to destination/backup SA container
/// </summary>
/// <returns></returns>
public async Task <DestinationBlobInfo> CopyBlobFromSourceToBackup(IBlobEvent eventData)
{
DestinationBlobInfo destinationBlobInfo = null;
string destinationStorageAccountConnectionString =
Environment.GetEnvironmentVariable("RES_STORAGE_ACCOUNT_CONN");
string sourceStorageAccountConnectionString =
Environment.GetEnvironmentVariable("SRC_STORAGE_ACCOUNT_CONN");
int blobTier =
string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TargetBlobTier")) ?
2 :
int.Parse(Environment.GetEnvironmentVariable("TargetBlobTier"));
StandardBlobTier targetBlobTier; // Default tier is set to Cool
switch (blobTier)
{
case 1:
targetBlobTier = StandardBlobTier.Hot;
break;
case 2:
targetBlobTier = StandardBlobTier.Cool;
break;
case 3:
targetBlobTier = StandardBlobTier.Archive;
break;
default:
targetBlobTier = StandardBlobTier.Cool;
break;
}
;
if (eventData is BlobEvent <CreatedEventData> )
{
// Retrieve the storage account from the connection string.
CloudStorageAccount sourceStorageAccount = CloudStorageAccount.Parse(sourceStorageAccountConnectionString);
CloudBlobClient sourceBlobClient = sourceStorageAccount.CreateCloudBlobClient();
// Retrieve the storage account from the connection string.
CloudStorageAccount destinationStorageAccount = CloudStorageAccount.Parse(destinationStorageAccountConnectionString);
CloudBlobClient destinationBlobClient = destinationStorageAccount.CreateCloudBlobClient();
BlobEvent <CreatedEventData> createdEventData = (BlobEvent <CreatedEventData>)eventData;
string url = createdEventData.data.url;
CloudBlockBlob sourceBlockBlob = new CloudBlockBlob(new Uri(url), sourceBlobClient);
bool sourceBlobExists = await sourceBlockBlob.ExistsAsync();
if (sourceBlobExists)
{
long blobSize = sourceBlockBlob.Properties.Length;
EventDateDetails dateDetails = new EventDateDetails(createdEventData.eventTime);
string destinationContaninerName = dateDetails.year.ToString();
string destinationBlobName = $"wk{dateDetails.WeekNumber}/dy{(int)dateDetails.DayOfWeek}/{sourceBlockBlob.Container.Name}/{sourceBlockBlob.Name}";
destinationBlobName += ".";
destinationBlobName += DateTimeUtil.GetString;
CloudBlobContainer destinationContainer = destinationBlobClient.GetContainerReference(destinationContaninerName);
bool result = await destinationContainer.CreateIfNotExistsAsync();
CloudBlockBlob destinationBlob = destinationContainer.GetBlockBlobReference(destinationBlobName);
_logger.LogInformation($"About to sync copy from Container: {sourceBlockBlob.Container.Name}, Blob: {sourceBlockBlob.Name}. Blob size: {sourceBlockBlob.Properties.Length} bytes");
// copyResult = "SYNCCOPY";
string copyResult =
await destinationBlob.StartCopyAsync(
sourceBlockBlob,
targetBlobTier,
null, // Rehydrate priority
null, // Source access condition
null, // Destination access condition
null, // Blob request options
null, // Operation context
(new CancellationTokenSource()).Token);
destinationBlobInfo = new DestinationBlobInfo();
destinationBlobInfo.ContainerName = destinationContainer.Name;
destinationBlobInfo.BlobName = destinationBlobName;
destinationBlobInfo.CopyReferenceId = copyResult;
destinationBlobInfo.OrgContainerName = sourceBlockBlob.Container.Name;
destinationBlobInfo.OrgBlobName = sourceBlockBlob.Name;
_logger.LogInformation($"Copy Scheduled. Source Blob Name: {sourceBlockBlob.Name}, Destination Blob Name: {destinationBlobInfo.BlobName}, Copy Id: {copyResult}.");
return(destinationBlobInfo);
}
else
{
_logger.LogInformation($"Not able to locate the block blob in source storage account---Block blob Name: {sourceBlockBlob.Name}");
}
}
else
{
_logger.LogInformation($"Input event data is not of Created Event Type.");
}
return(destinationBlobInfo);
}
/// <summary>
/// Run method
/// 1: Reads the messgaes from the table storage in ascending order.
/// 2: Peform the delete or create operation on the destination blob.
/// </summary>
/// <returns></returns>
public async Task Run(DateTime startDate, DateTime endDate)
{
_logger.LogDebug("Inside RestoreBackupWorker.Run");
//Since there can be many records around 84K for a day, let's read the records day by day and perform the restore operation
List <Tuple <int, int, DateTime> > dates = GetDatesForDateRange(startDate, endDate);
_logger.LogInformation($"Number of dates determined.---{dates.Count()}");
int totalSuccessCount = 0;
int totalFailureCount = 0;
foreach (Tuple <int, int, DateTime> dateData in dates)
{
_logger.LogInformation($"Starting restore for Year {dateData.Item1} Week {dateData.Item2} and Date {dateData.Item3.ToString("MM/dd/yyyy")}");
var blobEvents = await _storageRepository.GetBLOBEvents(dateData.Item1, dateData.Item2, dateData.Item3, dateData.Item3.AddDays(1));
_logger.LogInformation($"Found {blobEvents.Count} for {dateData.Item1} and Date {dateData.Item3.ToString("MM/dd/yyyy")}");
if (blobEvents != null && blobEvents.Count > 0)
{
foreach (EventData eventData in blobEvents)
{
try
{
if (eventData.RecievedEventData is BlobEvent <CreatedEventData> )
{
BlobEvent <CreatedEventData> createdBlob = (BlobEvent <CreatedEventData>)eventData.RecievedEventData;
if (eventData.DestinationBlobInfo != null)
{
_logger.LogInformation($"Going to perform copy as it is a created event {createdBlob.data.url}");
await _blobRepository.CopyBlobFromBackupToRestore(eventData.DestinationBlobInfo);
}
else
{
_logger.LogInformation($"Copy of the blob will be ignored as at the time of backup the blob was not present at source. One of the cause can be , a delete has been performed already on this blob. {createdBlob.data.url}");
}
}
else if (eventData.RecievedEventData is BlobEvent <DeletedEventData> )
{
BlobEvent <DeletedEventData> deletedBlob = (BlobEvent <DeletedEventData>)eventData.RecievedEventData;
_logger.LogInformation($"Going to delete as it is a deleted event {deletedBlob.data.url}");
await _blobRepository.DeleteBlobFromRestore(eventData.RecievedEventData);
}
else
{
_logger.LogInformation($"Currently Created and Deleted events are supported. Event Data: {eventData.RecievedEventDataJSON}");
}
totalSuccessCount++;
}
catch (Exception ex)
{
totalFailureCount++;
_logger.LogError($"Exception while restoring event {eventData.RecievedEventDataJSON}. Exception {ex.ToString()}");
}
}
}
}
_logger.LogInformation($" Restore Success records count {totalSuccessCount}. Restore Failure records count {totalFailureCount}.");
_logger.LogDebug("Completed RestoreBackupWorker.Run");
}
/// <summary>
/// Run method
/// 1: Reads messages from the table storage in ascending order.
/// 2: Executes the delete or create operation on the destination blob.
/// </summary>
/// <returns></returns>
public async Task Run(RestoreReqResponse reqResponse)
{
_logger.LogDebug("Begin: RestoreBackupWorker.Run");
//Since there can be many records around 84K for a day, let's read the records day by day and perform the restore operation
List <Tuple <int, int, DateTime> > dates = GetDatesForDateRange(reqResponse.StDate, reqResponse.EnDate);
_logger.LogInformation($"Number of dates determined.---{dates.Count()}");
int totalSuccessCount = 0;
int totalFailureCount = 0;
foreach (Tuple <int, int, DateTime> dateData in dates)
{
_logger.LogInformation($"Starting restore for Year {dateData.Item1} Week {dateData.Item2} and Date {dateData.Item3.ToString("MM/dd/yyyy")}");
var blobEvents = await _logRepository.GetBLOBEvents(dateData.Item1, dateData.Item2, dateData.Item3, dateData.Item3.AddDays(1));
_logger.LogInformation($"Found {blobEvents.Count} for {dateData.Item1} and Date {dateData.Item3.ToString("MM/dd/yyyy")}");
if (blobEvents != null && blobEvents.Count > 0)
{
foreach (EventData eventData in blobEvents)
{
try
{
if (eventData.ReceivedEventData is BlobEvent <CreatedEventData> )
{
BlobEvent <CreatedEventData> createdBlob = (BlobEvent <CreatedEventData>)eventData.ReceivedEventData;
if (eventData.DestinationBlobInfo != null)
{
if ((!String.IsNullOrEmpty(reqResponse.ContainerName)) && (!String.Equals(eventData.DestinationBlobInfo.OrgContainerName, reqResponse.ContainerName)))
{
continue;
}
if ((reqResponse.BlobNames != null) && (!reqResponse.BlobNames.Contains(eventData.DestinationBlobInfo.OrgBlobName)))
{
continue;
}
_logger.LogInformation($"Going to perform copy as it is a created event {createdBlob.data.url}");
await _blobRepository.CopyBlobFromBackupToRestore(eventData.DestinationBlobInfo);
}
else
{
_logger.LogInformation($"Copy of the blob will be ignored as at the time of backup the blob was not present at source. One of the cause can be , a delete has been performed already on this blob. {createdBlob.data.url}");
continue;
}
}
else if (eventData.ReceivedEventData is BlobEvent <DeletedEventData> )
{
BlobEvent <DeletedEventData> deletedBlob = (BlobEvent <DeletedEventData>)eventData.ReceivedEventData;
if (reqResponse.SkipDeletes.ToUpper(new CultureInfo("en-US", false)).Equals(Constants.Constants.RESTORE_SKIP_DELETES_YES))
{
continue;
}
if ((!String.IsNullOrEmpty(reqResponse.ContainerName)) && (!deletedBlob.data.url.Contains(reqResponse.ContainerName)))
{
continue;
}
if ((reqResponse.BlobNames != null) && (!reqResponse.BlobNames.Exists(x => deletedBlob.data.url.Contains(x))))
{
continue;
}
_logger.LogInformation($"Going to perform delete as it is a deleted event {deletedBlob.data.url}");
await _blobRepository.DeleteBlobFromRestore(eventData.ReceivedEventData);
}
else
{
_logger.LogInformation($"Currently only Created and Deleted events are supported. Event Data: {eventData.ReceivedEventDataJSON}");
continue;
}
totalSuccessCount++;
}
catch (Exception ex)
{
totalFailureCount++;
_logger.LogError($"Exception while restoring event {eventData.ReceivedEventDataJSON}. Exception {ex.ToString()}");
}
if (reqResponse.ReqType.Equals(Constants.Constants.RESTORE_REQUEST_TYPE_ASYNC) && (totalSuccessCount % _updateFrequencyCount == 0))
{
reqResponse.TotalSuccessCount = totalSuccessCount;
reqResponse.TotalFailureCount = totalFailureCount;
await _restoreTblRepository.UpdateRestoreRequest(reqResponse);
}
;
}
;
// Update the processed record count for async restore request
}
}
; // End of outer For loop
_logger.LogInformation($"Restore Success records count: {totalSuccessCount}. Restore Failure records count: {totalFailureCount}.");
reqResponse.TotalSuccessCount = totalSuccessCount;
reqResponse.TotalFailureCount = totalFailureCount;
_logger.LogDebug("End: RestoreBackupWorker.Run");
}
/// <summary>
/// CopyBlobFromSourceToBackup
/// </summary>
/// <returns></returns>
public async Task <DestinationBlobInfo> CopyBlobFromSourceToBackup(IBlobEvent eventData)
{
DestinationBlobInfo destinationBlobInfo = null;
string destinationStorageAccountConnectionString = _config.GetConnectionString("BackupBlobStorage");
string sourceStorageAccountConnectionString = _config.GetConnectionString("SourceBlobStorage");
bool isServerCopy = bool.Parse(_config.GetSection("AppSettings")["IsServerCopy"]);
if (eventData is BlobEvent <CreatedEventData> )
{
// Retrieve the storage account from the connection string.
CloudStorageAccount sourceStorageAccount = CloudStorageAccount.Parse(sourceStorageAccountConnectionString);
CloudBlobClient sourceBlobClient = sourceStorageAccount.CreateCloudBlobClient();
// Retrieve the storage account from the connection string.
CloudStorageAccount destinationStorageAccount = CloudStorageAccount.Parse(destinationStorageAccountConnectionString);
CloudBlobClient destinationBlobClient = destinationStorageAccount.CreateCloudBlobClient();
BlobEvent <CreatedEventData> createdEventData = (BlobEvent <CreatedEventData>)eventData;
string url = createdEventData.data.url;
CloudBlockBlob sourceBlockBlob = new CloudBlockBlob(new Uri(url), sourceBlobClient);
bool sourceBlobExists = await sourceBlockBlob.ExistsAsync();
if (sourceBlobExists)
{
long blobSize = sourceBlockBlob.Properties.Length;
EventDateDetails dateDetails = new EventDateDetails(createdEventData.eventTime);
string destinationContaninerName = dateDetails.year.ToString();
string destinationBlobName = $"wk{dateDetails.WeekNumber}/dy{(int)dateDetails.DayOfWeek}/{sourceBlockBlob.Container.Name}/{sourceBlockBlob.Name}";
CloudBlobContainer destinationContainer = destinationBlobClient.GetContainerReference(destinationContaninerName);
bool result = await destinationContainer.CreateIfNotExistsAsync();
CloudBlockBlob destinationBlob = destinationContainer.GetBlockBlobReference(destinationBlobName);
string copyResult = string.Empty;
if (isServerCopy)
{
string blobToken = GenerateSASBlobToken(sourceBlockBlob);
_logger.LogInformation($"About to server copy {sourceBlockBlob.Name}. Blob size {sourceBlockBlob.Properties.Length} bytes");
copyResult = await destinationBlob.StartCopyAsync(new Uri(sourceBlockBlob.Uri.AbsoluteUri + blobToken));
}
else
{
_logger.LogInformation($"About to sync copy {sourceBlockBlob.Name}. Blob size {sourceBlockBlob.Properties.Length} bytes");
copyResult = "SYNCCOPY";
await TransferManager.CopyAsync(sourceBlockBlob, destinationBlob, false);
}
destinationBlobInfo = new DestinationBlobInfo();
destinationBlobInfo.ContainerName = destinationContainer.Name;
destinationBlobInfo.BlobName = destinationBlobName;
destinationBlobInfo.CopyReferenceId = copyResult;
destinationBlobInfo.OrgContainerName = sourceBlockBlob.Container.Name;
destinationBlobInfo.OrgBlobName = sourceBlockBlob.Name;
return(destinationBlobInfo);
}
else
{
_logger.LogInformation($"Not able to locate the block blob in source storage account---Block blob Name {sourceBlockBlob.Name}");
}
}
else
{
_logger.LogInformation($"Input event data is not of Created Event Type.");
}
return(destinationBlobInfo);
}
/// <summary>
/// Run method
/// 1: Reads the messages from the table storage in ascending order.
/// 2: Peform the delete or create operation on the destination blob.
/// </summary>
/// <returns></returns>
// public async Task Run(DateTime startDate, DateTime endDate, String contName) ID05202020.o
public async Task Run(RestoreReqResponse reqResponse) // ID05202020.n
{
_logger.LogDebug("Inside RestoreBackupWorker.Run");
//Since there can be many records around 84K for a day, let's read the records day by day and perform the restore operation
// List<Tuple<int,int, DateTime>> dates = GetDatesForDateRange(startDate, endDate); ID05202020.o
List <Tuple <int, int, DateTime> > dates = GetDatesForDateRange(reqResponse.StDate, reqResponse.EnDate); // ID05202020.n
_logger.LogInformation($"Number of dates determined.---{dates.Count()}");
int totalSuccessCount = 0;
int totalFailureCount = 0;
foreach (Tuple <int, int, DateTime> dateData in dates)
{
_logger.LogInformation($"Starting restore for Year {dateData.Item1} Week {dateData.Item2} and Date {dateData.Item3.ToString("MM/dd/yyyy")}");
var blobEvents = await _storageRepository.GetBLOBEvents(dateData.Item1, dateData.Item2, dateData.Item3, dateData.Item3.AddDays(1));
_logger.LogInformation($"Found {blobEvents.Count} for {dateData.Item1} and Date {dateData.Item3.ToString("MM/dd/yyyy")}");
if (blobEvents != null && blobEvents.Count > 0)
{
foreach (EventData eventData in blobEvents)
{
try
{
if (eventData.RecievedEventData is BlobEvent <CreatedEventData> )
{
BlobEvent <CreatedEventData> createdBlob = (BlobEvent <CreatedEventData>)eventData.RecievedEventData;
if (eventData.DestinationBlobInfo != null)
{
// ID05192020.sn
if ((!String.IsNullOrEmpty(reqResponse.ContainerName)) && (!String.Equals(eventData.DestinationBlobInfo.OrgContainerName, reqResponse.ContainerName))) // ID05202020.n
{
continue;
}
if ((!String.IsNullOrEmpty(reqResponse.BlobName)) && (!String.Equals(eventData.DestinationBlobInfo.OrgBlobName, reqResponse.BlobName))) // ID05202020.n
{
continue;
}
_logger.LogDebug($"I/U Orig. Container:{eventData.DestinationBlobInfo.OrgContainerName},Req. Container:{reqResponse.ContainerName}");
_logger.LogDebug($"I/U Orig. Blob:{eventData.DestinationBlobInfo.OrgBlobName},Req. Blob:{reqResponse.BlobName}");
_logger.LogInformation($"Going to perform copy as it is a created event {createdBlob.data.url}");
// ID05192020.en
await _blobRepository.CopyBlobFromBackupToRestore(eventData.DestinationBlobInfo);
}
else
{
_logger.LogInformation($"Copy of the blob will be ignored as at the time of backup the blob was not present at source. One of the cause can be , a delete has been performed already on this blob. {createdBlob.data.url}");
continue; // ID05202020.n
}
}
else if (eventData.RecievedEventData is BlobEvent <DeletedEventData> )
{
BlobEvent <DeletedEventData> deletedBlob = (BlobEvent <DeletedEventData>)eventData.RecievedEventData;
// ID05192020.sn
if (reqResponse.SkipDeletes.ToUpper(new CultureInfo("en-US", false)).Equals(Yes)) // ID05202020.n
{
continue;
}
if ((!String.IsNullOrEmpty(reqResponse.ContainerName)) && (!deletedBlob.data.url.Contains(reqResponse.ContainerName))) // ID05202020.n
{
continue;
}
if ((!String.IsNullOrEmpty(reqResponse.BlobName)) && (!deletedBlob.data.url.Contains(reqResponse.BlobName))) // ID05202020.n
{
continue;
}
_logger.LogDebug($"D Req. Container:{reqResponse.ContainerName}");
_logger.LogDebug($"D Req. Blob:{reqResponse.BlobName}");
_logger.LogInformation($"Going to delete as it is a deleted event {deletedBlob.data.url}");
// ID05192020.en
await _blobRepository.DeleteBlobFromRestore(eventData.RecievedEventData);
}
else
{
_logger.LogInformation($"Currently Created and Deleted events are supported. Event Data: {eventData.RecievedEventDataJSON}");
continue; // ID05202020.n
}
totalSuccessCount++;
}
catch (Exception ex)
{
totalFailureCount++;
_logger.LogError($"Exception while restoring event {eventData.RecievedEventDataJSON}. Exception {ex.ToString()}");
}
}
}
}
_logger.LogInformation($" Restore Success records count {totalSuccessCount}. Restore Failure records count {totalFailureCount}.");
reqResponse.TotalSuccessCount = totalSuccessCount;
reqResponse.TotalFailureCount = totalFailureCount;
_logger.LogDebug("Completed RestoreBackupWorker.Run");
}
