private async Task <Dictionary <string, DateTimeOffset> > LoadPackagesAtDestination()
{
var results = new Dictionary <string, DateTimeOffset>();
BlobContinuationToken token = new BlobContinuationToken();
BlobResultSegment segment;
var options = new BlobRequestOptions();
var context = new OperationContext();
do
{
segment = await DestinationContainer.ListBlobsSegmentedAsync(
prefix : null,
useFlatBlobListing : true,
blobListingDetails : BlobListingDetails.Metadata,
maxResults : null,
currentToken : token,
options : options,
operationContext : context);
results.AddRange(
segment
.Results
.OfType <CloudBlockBlob>()
.Select(b => new KeyValuePair <string, DateTimeOffset>(b.Name, b.Properties.LastModified.HasValue ? b.Properties.LastModified.Value : DateTimeOffset.Now)));
Log.GatheredDestinationPackagesListSegment(Destination.Credentials.AccountName, DestinationContainer.Name, results.Count);
token = segment.ContinuationToken;
} while (token != null);
return(results);
}
protected async Task CopyOrOverwritePackage(string sourceContainerSharedAccessUri, string sourceBlobName, string destinationBlobName, string packageHash)
{
// Identify the source and destination blobs
var sourceBlob = SourceContainer.GetBlockBlobReference(sourceBlobName);
var destBlob = DestinationContainer.GetBlockBlobReference(destinationBlobName);
// If the destination blob already exists, it will be overwritten
// To prevent several requests, do not check if it exists
if (!await sourceBlob.ExistsAsync())
{
Log.SourceBlobMissing(sourceBlob.Name);
}
else
{
var sourceUri = new Uri(sourceBlob.Uri, sourceContainerSharedAccessUri);
// Start the copy or overwrite
Log.StartingCopy(sourceUri.AbsoluteUri, destBlob.Uri.AbsoluteUri);
if (!WhatIf)
{
await destBlob.StartCopyFromBlobAsync(sourceUri);
}
Log.StartedCopy(sourceUri.AbsoluteUri, destBlob.Uri.AbsoluteUri);
}
}
protected override void SourceModelListPageAction(List <TDomainModel> modelList)
{
var dbContext = DestinationContainer.Resolve <ISqlServerDbContext>();
using var connection = dbContext.Specific.Database.GetDbConnection();
dbContext.Specific.Database.OpenConnection();
var tableName = dbContext.Specific.Model.GetFullTableName <TIdentityEntity>();
using (var command = connection.CreateCommand())
{
command.CommandText = $"SET IDENTITY_INSERT {tableName} ON;";
_ = command.ExecuteNonQuery();
}
try
{
base.SourceModelListPageAction(modelList);
}
finally
{
using (var command = connection.CreateCommand())
{
command.CommandText = $"SET IDENTITY_INSERT {tableName} OFF;";
_ = command.ExecuteNonQuery();
}
}
}
private async Task CopyPackageToDestination(SharedAccessBlobPolicy policy, string packageBlobName)
{
// Identify the source and destination blobs
var sourceBlob = SourceContainer.GetBlockBlobReference(packageBlobName);
var destBlob = DestinationContainer.GetBlockBlobReference(packageBlobName);
// If the destination blob already exists, it will be overwritten. NO harm done
// While it should not happen, to prevent several requests, do not check if it exists
if (!await sourceBlob.ExistsAsync())
{
Log.SourceBlobMissing(sourceBlob.Name);
}
else
{
// Need to use Shared Access Signature since blobs to be copied are in a private container
// Create a Shared Access Signature (SAS) Uri, for every blob to be copied
// Set start time as Now and expiry time as Now + 12 hours
policy.SharedAccessStartTime = DateTimeOffset.Now;
policy.SharedAccessExpiryTime = DateTimeOffset.Now + TimeSpan.FromHours(12);
var sourceContainerSharedAccessUri = SourceContainer.GetSharedAccessSignature(policy);
Log.SharedAccessSignatureURI(sourceContainerSharedAccessUri);
var sourceUri = new Uri(sourceBlob.Uri, sourceContainerSharedAccessUri);
// Start the copy or overwrite
Log.StartingCopy(sourceUri.AbsoluteUri, destBlob.Uri.AbsoluteUri);
if (!WhatIf)
{
await destBlob.StartCopyFromBlobAsync(sourceUri);
}
Log.StartedCopy(sourceUri.AbsoluteUri, destBlob.Uri.AbsoluteUri);
}
}
public override int Execute()
{
var res = base.Execute();
var dbContext = DestinationContainer.Resolve <ISqlServerDbContext>();
_sequenceRebuilder.Execute(dbContext, x => x.Id);
return(res);
}
protected override void SourceModelListPageAction(List <TDomainModel> modelList)
{
base.SourceModelListPageAction(modelList);
var watch = StopwatchHelper.StopwatchAction(() =>
DestinationContainer.Resolve <ISqlServerDbContext>().SaveChanges());
Logger.Debug(
$"{modelList.Count.ToString()} {typeof(TDomainModel).Name} documents have been stored by SqlServer in {watch.Elapsed.ToString()}.");
}
private async Task WriteToBlob(string report, string name)
{
var blob = DestinationContainer.GetBlockBlobReference(name);
Trace.TraceInformation(string.Format("Writing report to {0}", blob.Uri.AbsoluteUri));
blob.Properties.ContentType = "json";
await blob.UploadTextAsync(report);
Trace.TraceInformation(string.Format("Wrote report to {0}", blob.Uri.AbsoluteUri));
}
protected async Task DeletePackage(string blobName)
{
var blob = DestinationContainer.GetBlockBlobReference(blobName);
Log.StartingDelete(blob.Uri.AbsoluteUri);
if (!WhatIf)
{
await blob.DeleteIfExistsAsync();
}
Log.StartedDelete(blob.Uri.AbsoluteUri);
}
protected async Task WriteReport(string report, string name, Formatting formatting)
{
if (!string.IsNullOrEmpty(OutputDirectory))
{
await WriteToFile(report, name);
}
else
{
await DestinationContainer.CreateIfNotExistsAsync();
await WriteToBlob(report, name);
}
}
public override int Execute()
{
var res = base.Execute();
var dbContext = DestinationContainer.Resolve <ISqlServerDbContext>();
_ = dbContext.Add(new MigrationHistory
{
CollectionName = typeof(TDomainModel).Name,
Date = DateTime.UtcNow
});
dbContext.SaveChanges();
return(res);
}
public override bool Validate()
{
var dbContext = DestinationContainer.Resolve <ISqlServerDbContext>();
var isMigrated = dbContext.Query <MigrationHistory>()
.Any(x => x.CollectionName == typeof(TDomainModel).Name);
if (isMigrated)
{
Logger.Warning($"{typeof(TDomainModel).Name} documents have already been migrated.");
return(false);
}
var domainModelContext = DestinationContainer.Resolve <IDomainModelContext>();
var count = domainModelContext.Count <TDomainModel>();
if (count > 0)
{
Logger.Warning($"SqlServer database already contains {typeof(TDomainModel).Name} documents, migration will be skipped.");
return(false);
}
return(true);
}
protected internal override async Task Execute()
{
PackageDatabase = PackageDatabase ?? Config.Sql.GetConnectionString(KnownSqlConnection.Legacy);
Source = Source ?? Config.Storage.Backup;
Destination = Destination ?? Config.Storage.Primary;
PackageDatabase.TrimNetworkProtocol();
SourceContainer = Source.CreateCloudBlobClient().GetContainerReference(
String.IsNullOrEmpty(SourceContainerName) ? BlobContainerNames.Backups : SourceContainerName);
DestinationContainer = Destination.CreateCloudBlobClient().GetContainerReference(
String.IsNullOrEmpty(DestinationContainerName) ? BlobContainerNames.LegacyPackages : DestinationContainerName);
Log.PreparingToSync(Source.Credentials.AccountName, SourceContainer.Name, Destination.Credentials.AccountName, DestinationContainer.Name, PackageDatabase.DataSource, PackageDatabase.InitialCatalog);
// Gather packages
Log.GatheringListOfPackages(PackageDatabase.DataSource, PackageDatabase.InitialCatalog);
IList <PackageRef> packagesInDB;
using (var connection = await PackageDatabase.ConnectTo())
{
packagesInDB = (await connection.QueryAsync <PackageRef>(@"
SELECT pr.Id, p.NormalizedVersion AS Version, p.Hash, p.LastEdited
FROM Packages p
INNER JOIN PackageRegistrations pr ON p.PackageRegistrationKey = pr.[Key]"))
.ToList();
}
Log.GatheredListOfPackages(packagesInDB.Count, PackageDatabase.DataSource, PackageDatabase.InitialCatalog);
if (!WhatIf)
{
await DestinationContainer.CreateIfNotExistsAsync();
}
// Collect a list of packages in destination with metadata
Log.GatheringDestinationPackages(Destination.BlobEndpoint.ToString(), DestinationContainer.Name);
var destinationPackages = await LoadPackagesAtDestination();
Log.GatheredDestinationPackagesList(Destination.BlobEndpoint.ToString(), DestinationContainer.Name, destinationPackages.Count);
Log.CalculatingCopyOrOverwritePackages(packagesInDB.Count, destinationPackages.Count);
var packagesToCopyOrOverwrite = PackagesToCopyOrOverwrite(packagesInDB, destinationPackages);
Log.CalculatedCopyOrOverwritePackages(packagesInDB.Count, packagesToCopyOrOverwrite.Count);
Log.StartingSync(packagesToCopyOrOverwrite.Count);
if (packagesToCopyOrOverwrite.Count > 0)
{
var policy = new SharedAccessBlobPolicy();
policy.SharedAccessStartTime = DateTimeOffset.Now;
policy.SharedAccessExpiryTime = DateTimeOffset.Now + TimeSpan.FromHours(2) + TimeSpan.FromMinutes(2 * packagesToCopyOrOverwrite.Count);
policy.Permissions = SharedAccessBlobPermissions.Read;
var sourceContainerSharedAccessUri = SourceContainer.GetSharedAccessSignature(policy);
Log.SharedAccessSignatureURI(sourceContainerSharedAccessUri);
foreach (var packageRef in packagesToCopyOrOverwrite)
{
await CopyOrOverwritePackage(sourceContainerSharedAccessUri, StorageHelpers.GetPackageBackupBlobName(packageRef),
StorageHelpers.GetPackageBlobName(packageRef), packageRef.Hash);
if ((Invocation.NextVisibleAt - DateTimeOffset.UtcNow) < TimeSpan.FromMinutes(1))
{
// Running out of time! Extend the job
Log.ExtendingJobLeaseWhileSyncingProgresses();
await Extend(TimeSpan.FromMinutes(5));
Log.ExtendedJobLease();
}
else
{
Log.JobLeaseOk();
}
}
}
Log.CalculatingDeletePackages(destinationPackages.Count, packagesInDB.Count);
var packagesToDelete = PackagesToDelete(packagesInDB, destinationPackages);
Log.CalculatedDeletePackages(packagesToDelete.Count, destinationPackages.Count);
if (packagesToDelete.Count > 0)
{
foreach (var packageBlobName in packagesToDelete)
{
await DeletePackage(packageBlobName);
if ((Invocation.NextVisibleAt - DateTimeOffset.UtcNow) < TimeSpan.FromMinutes(1))
{
// Running out of time! Extend the job
Log.ExtendingJobLeaseWhileSyncingProgresses();
await Extend(TimeSpan.FromMinutes(5));
Log.ExtendedJobLease();
}
else
{
Log.JobLeaseOk();
}
}
}
Log.StartedSync();
}
protected internal override async Task Execute()
{
Source = Source ?? Config.Storage.Legacy;
Destination = Destination ?? Config.Storage.Backup;
SourceContainer = Source.CreateCloudBlobClient().GetContainerReference(
String.IsNullOrEmpty(SourceContainerName) ? BlobContainerNames.Backups : SourceContainerName);
DestinationContainer = Destination.CreateCloudBlobClient().GetContainerReference(
String.IsNullOrEmpty(DestinationContainerName) ? BlobContainerNames.Backups : DestinationContainerName);
Log.PreparingToTransfer(Source.Credentials.AccountName, SourceContainer.Name, Destination.Credentials.AccountName, DestinationContainer.Name);
// Gather packages
Log.GatheringListOfPackages(Source.BlobEndpoint.ToString(), SourceContainer.Name);
var sourcePackages = await LoadBlobList(Log, Source, SourceContainer);
Log.GatheredListOfPackages(sourcePackages.Count, Source.BlobEndpoint.ToString(), SourceContainer.Name);
if (!WhatIf)
{
await DestinationContainer.CreateIfNotExistsAsync();
}
// Collect a list of packages in destination with metadata
Log.GatheringListOfPackages(Destination.BlobEndpoint.ToString(), DestinationContainer.Name);
var destinationPackages = await LoadBlobList(Log, Destination, DestinationContainer);
Log.GatheredListOfPackages(destinationPackages.Count, Destination.BlobEndpoint.ToString(), DestinationContainer.Name);
Log.CalculatingCopyPackages(sourcePackages.Count, destinationPackages.Count);
var packageBlobsToCopy = sourcePackages.Where(p => !destinationPackages.Contains(p)).ToList();
Log.CalculatedCopyPackages(sourcePackages.Count, packageBlobsToCopy.Count);
Log.StartingTransfer(packageBlobsToCopy.Count);
if (packageBlobsToCopy.Count > 0)
{
var policy = new SharedAccessBlobPolicy();
policy.Permissions = SharedAccessBlobPermissions.Read;
foreach (var packageBlobName in packageBlobsToCopy)
{
await CopyPackageToDestination(policy, packageBlobName);
if ((Invocation.NextVisibleAt - DateTimeOffset.UtcNow) < TimeSpan.FromMinutes(1))
{
// Running out of time! Extend the job
Log.ExtendingJobLeaseWhileTransferProgresses();
await Extend(TimeSpan.FromMinutes(5));
Log.ExtendedJobLease();
}
else
{
Log.JobLeaseOk();
}
}
}
Log.StartedTransfer();
}
protected virtual int ProcessSourceModelByPages()
{
var total = 0;
if (PageSize == 0)
{
Logger.Debug($"All {typeof(TDomainModel).Name} documents are requested.");
var allModels = StopwatchHelper.StopwatchFunc(() => GetSourceDocuments().ToList(), out var watch);
total = allModels.Count;
Logger.Debug($"{total} {typeof(TDomainModel).Name} documents have been received in {watch.Elapsed}.");
watch = StopwatchHelper.StopwatchAction(() => SourceModelListPageAction(allModels));
Logger.Debug($"{total} {typeof(TDomainModel).Name} documents have been processed in {watch.Elapsed}.");
}
else
{
Logger.Debug($"{typeof(TDomainModel).Name} documents requested by pages with {PageSize} docs for each page.");
var idx = 0;
var page = 1;
var modelList = new List <TDomainModel>((int)PageSize);
foreach (var doc in GetSourceDocuments())
{
if (idx >= PageSize)
{
CallPageAction();
modelList.Clear();
idx = 0;
page++;
}
AdjustModel(doc);
modelList.Add(doc);
idx++;
total++;
}
if (idx > 0)
{
CallPageAction();
}
#region LocalFunc
void CallPageAction()
{
var destContainer = DestinationContainer;
try
{
using (DestinationContainer = DestinationContainer.BeginLifetimeScope())
{
var watch = StopwatchHelper.StopwatchAction(() => SourceModelListPageAction(modelList));
Logger.Debug(
$"Page #{page} of {idx} {typeof(TDomainModel).Name} documents has been processed in {watch.Elapsed}.");
}
}
finally
{
DestinationContainer = destContainer;
}
}
#endregion
}
return(total);
}
protected virtual void AddDestinationModelRange(IEnumerable <TDomainModel> destinationModels) => DestinationContainer.Resolve <IDomainModelContext>().AddRange(destinationModels.Cast <object>());
protected virtual void AddDestinationModel(TDomainModel destinationModel) => DestinationContainer.Resolve <IDomainModelContext>().Add(destinationModel);
