private async Task CopyPackageToDestination(SharedAccessBlobPolicy policy, string packageBlobName)
{
// Identify the source and destination blobs
var sourceBlob = SourceContainer.GetBlockBlobReference(packageBlobName);
var destBlob = DestinationContainer.GetBlockBlobReference(packageBlobName);
// If the destination blob already exists, it will be overwritten. NO harm done
// While it should not happen, to prevent several requests, do not check if it exists
if (!await sourceBlob.ExistsAsync())
{
Log.SourceBlobMissing(sourceBlob.Name);
}
else
{
// Need to use Shared Access Signature since blobs to be copied are in a private container
// Create a Shared Access Signature (SAS) Uri, for every blob to be copied
// Set start time as Now and expiry time as Now + 12 hours
policy.SharedAccessStartTime = DateTimeOffset.Now;
policy.SharedAccessExpiryTime = DateTimeOffset.Now + TimeSpan.FromHours(12);
var sourceContainerSharedAccessUri = SourceContainer.GetSharedAccessSignature(policy);
Log.SharedAccessSignatureURI(sourceContainerSharedAccessUri);
var sourceUri = new Uri(sourceBlob.Uri, sourceContainerSharedAccessUri);
// Start the copy or overwrite
Log.StartingCopy(sourceUri.AbsoluteUri, destBlob.Uri.AbsoluteUri);
if (!WhatIf)
{
await destBlob.StartCopyFromBlobAsync(sourceUri);
}
Log.StartedCopy(sourceUri.AbsoluteUri, destBlob.Uri.AbsoluteUri);
}
}
protected internal override async Task Execute()
{
PackageDatabase = PackageDatabase ?? Config.Sql.GetConnectionString(KnownSqlConnection.Legacy);
Source = Source ?? Config.Storage.Backup;
Destination = Destination ?? Config.Storage.Primary;
PackageDatabase.TrimNetworkProtocol();
SourceContainer = Source.CreateCloudBlobClient().GetContainerReference(
String.IsNullOrEmpty(SourceContainerName) ? BlobContainerNames.Backups : SourceContainerName);
DestinationContainer = Destination.CreateCloudBlobClient().GetContainerReference(
String.IsNullOrEmpty(DestinationContainerName) ? BlobContainerNames.LegacyPackages : DestinationContainerName);
Log.PreparingToSync(Source.Credentials.AccountName, SourceContainer.Name, Destination.Credentials.AccountName, DestinationContainer.Name, PackageDatabase.DataSource, PackageDatabase.InitialCatalog);
// Gather packages
Log.GatheringListOfPackages(PackageDatabase.DataSource, PackageDatabase.InitialCatalog);
IList <PackageRef> packagesInDB;
using (var connection = await PackageDatabase.ConnectTo())
{
packagesInDB = (await connection.QueryAsync <PackageRef>(@"
SELECT pr.Id, p.NormalizedVersion AS Version, p.Hash, p.LastEdited
FROM Packages p
INNER JOIN PackageRegistrations pr ON p.PackageRegistrationKey = pr.[Key]"))
.ToList();
}
Log.GatheredListOfPackages(packagesInDB.Count, PackageDatabase.DataSource, PackageDatabase.InitialCatalog);
if (!WhatIf)
{
await DestinationContainer.CreateIfNotExistsAsync();
}
// Collect a list of packages in destination with metadata
Log.GatheringDestinationPackages(Destination.BlobEndpoint.ToString(), DestinationContainer.Name);
var destinationPackages = await LoadPackagesAtDestination();
Log.GatheredDestinationPackagesList(Destination.BlobEndpoint.ToString(), DestinationContainer.Name, destinationPackages.Count);
Log.CalculatingCopyOrOverwritePackages(packagesInDB.Count, destinationPackages.Count);
var packagesToCopyOrOverwrite = PackagesToCopyOrOverwrite(packagesInDB, destinationPackages);
Log.CalculatedCopyOrOverwritePackages(packagesInDB.Count, packagesToCopyOrOverwrite.Count);
Log.StartingSync(packagesToCopyOrOverwrite.Count);
if (packagesToCopyOrOverwrite.Count > 0)
{
var policy = new SharedAccessBlobPolicy();
policy.SharedAccessStartTime = DateTimeOffset.Now;
policy.SharedAccessExpiryTime = DateTimeOffset.Now + TimeSpan.FromHours(2) + TimeSpan.FromMinutes(2 * packagesToCopyOrOverwrite.Count);
policy.Permissions = SharedAccessBlobPermissions.Read;
var sourceContainerSharedAccessUri = SourceContainer.GetSharedAccessSignature(policy);
Log.SharedAccessSignatureURI(sourceContainerSharedAccessUri);
foreach (var packageRef in packagesToCopyOrOverwrite)
{
await CopyOrOverwritePackage(sourceContainerSharedAccessUri, StorageHelpers.GetPackageBackupBlobName(packageRef),
StorageHelpers.GetPackageBlobName(packageRef), packageRef.Hash);
if ((Invocation.NextVisibleAt - DateTimeOffset.UtcNow) < TimeSpan.FromMinutes(1))
{
// Running out of time! Extend the job
Log.ExtendingJobLeaseWhileSyncingProgresses();
await Extend(TimeSpan.FromMinutes(5));
Log.ExtendedJobLease();
}
else
{
Log.JobLeaseOk();
}
}
}
Log.CalculatingDeletePackages(destinationPackages.Count, packagesInDB.Count);
var packagesToDelete = PackagesToDelete(packagesInDB, destinationPackages);
Log.CalculatedDeletePackages(packagesToDelete.Count, destinationPackages.Count);
if (packagesToDelete.Count > 0)
{
foreach (var packageBlobName in packagesToDelete)
{
await DeletePackage(packageBlobName);
if ((Invocation.NextVisibleAt - DateTimeOffset.UtcNow) < TimeSpan.FromMinutes(1))
{
// Running out of time! Extend the job
Log.ExtendingJobLeaseWhileSyncingProgresses();
await Extend(TimeSpan.FromMinutes(5));
Log.ExtendedJobLease();
}
else
{
Log.JobLeaseOk();
}
}
}
Log.StartedSync();
}
