public override bool Init(IDictionary <string, string> jobArgsDictionary)
{
Source =
new SqlConnectionStringBuilder(
JobConfigurationManager.GetArgument(jobArgsDictionary,
JobArgumentNames.SourceDatabase,
EnvironmentVariableKeys.SqlGallery));
WarehouseConnection =
new SqlConnectionStringBuilder(
JobConfigurationManager.GetArgument(jobArgsDictionary,
JobArgumentNames.DestinationDatabase,
EnvironmentVariableKeys.SqlWarehouse));
Destination = CloudStorageAccount.Parse(
JobConfigurationManager.GetArgument(jobArgsDictionary,
JobArgumentNames.WarehouseStorageAccount, EnvironmentVariableKeys.WarehouseStorage));
DestinationContainerName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.WarehouseContainerName) ?? DefaultPackageStatsContainerName;
DestinationContainer = Destination.CreateCloudBlobClient().GetContainerReference(DestinationContainerName);
_globalReportBuilders = new Dictionary <string, Func <Task> >(StringComparer.OrdinalIgnoreCase)
{
{ NuGetClientVersion, () => CreateReport(NuGetClientVersion, "Scripts.DownloadReport_NuGetClientVersion.sql") },
{ Last6Months, () => CreateReport(Last6Months, "Scripts.DownloadReport_Last6Months.sql") },
{ RecentPopularity, () => CreateReport(RecentPopularity, "Scripts.DownloadReport_RecentPopularity.sql") },
{ RecentPopularityDetail, () => CreateReport(RecentPopularityDetail, "Scripts.DownloadReport_RecentPopularityDetail.sql") },
};
return(true);
}
public override bool Init(IDictionary <string, string> jobArgsDictionary)
{
Source =
new SqlConnectionStringBuilder(
JobConfigurationManager.GetArgument(jobArgsDictionary,
JobArgumentNames.SourceDatabase,
EnvironmentVariableKeys.SqlGallery));
OutputDirectory = JobConfigurationManager.GetArgument(jobArgsDictionary,
JobArgumentNames.OutputDirectory);
if (string.IsNullOrEmpty(OutputDirectory))
{
Destination = CloudStorageAccount.Parse(
JobConfigurationManager.GetArgument(jobArgsDictionary,
JobArgumentNames.PrimaryDestination, EnvironmentVariableKeys.StoragePrimary));
DestinationContainerName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.DestinationContainerName) ?? DefaultContainerName;
DestinationContainer = Destination.CreateCloudBlobClient().GetContainerReference(DestinationContainerName);
}
return(true);
}
public override bool Init(IDictionary <string, string> jobArgsDictionary)
{
WarehouseConnection =
new SqlConnectionStringBuilder(
JobConfigurationManager.GetArgument(jobArgsDictionary,
JobArgumentNames.DestinationDatabase,
EnvironmentVariableKeys.SqlWarehouse));
Destination = CloudStorageAccount.Parse(
JobConfigurationManager.GetArgument(jobArgsDictionary,
JobArgumentNames.PrimaryDestination, EnvironmentVariableKeys.StoragePrimary));
DestinationContainerName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.DestinationContainerName) ?? DefaultContainerName;
DestinationContainer = Destination.CreateCloudBlobClient().GetContainerReference(DestinationContainerName);
string rankingCountString = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.RankingCount);
if (string.IsNullOrEmpty(rankingCountString))
{
RankingCount = DefaultRankingCount;
}
else
{
RankingCount = Convert.ToInt32(rankingCountString);
}
return(true);
}
protected virtual void LoadDefaults()
{
Destination = Destination ?? Config.Storage.Primary;
if (Destination != null)
{
DestinationContainer = Destination.CreateCloudBlobClient().GetContainerReference(
String.IsNullOrEmpty(DestinationContainerName) ? _defaultContainerName : DestinationContainerName);
}
}
private void LoadDefaults()
{
WarehouseConnection = WarehouseConnection ?? Config.Sql.Warehouse;
Destination = Destination ?? Config.Storage.Legacy;
if (Destination != null)
{
DestinationContainer = Destination.CreateCloudBlobClient().GetContainerReference(
String.IsNullOrEmpty(DestinationContainerName) ? BlobContainerNames.LegacyStats : DestinationContainerName);
}
}
public SqlExportArguments(IDictionary <string, string> jobArgsDictionary, string defaultContainerName, string defaultName)
{
var connStrBldr = new SqlConnectionStringBuilder(
JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SourceDatabase, EnvironmentVariableKeys.SqlGallery));
ConnectionString = connStrBldr.ToString();
OutputDirectory = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.OutputDirectory);
if (string.IsNullOrEmpty(OutputDirectory))
{
Destination = CloudStorageAccount.Parse(
JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PrimaryDestination, EnvironmentVariableKeys.StoragePrimary));
DestinationContainerName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.DestinationContainerName) ?? defaultContainerName;
DestinationContainer = Destination.CreateCloudBlobClient().GetContainerReference(DestinationContainerName);
}
Name = defaultName;
}
protected internal override async Task Execute()
{
PackageDatabase = PackageDatabase ?? Config.Sql.GetConnectionString(KnownSqlConnection.Legacy);
Source = Source ?? Config.Storage.Backup;
Destination = Destination ?? Config.Storage.Primary;
PackageDatabase.TrimNetworkProtocol();
SourceContainer = Source.CreateCloudBlobClient().GetContainerReference(
String.IsNullOrEmpty(SourceContainerName) ? BlobContainerNames.Backups : SourceContainerName);
DestinationContainer = Destination.CreateCloudBlobClient().GetContainerReference(
String.IsNullOrEmpty(DestinationContainerName) ? BlobContainerNames.LegacyPackages : DestinationContainerName);
Log.PreparingToSync(Source.Credentials.AccountName, SourceContainer.Name, Destination.Credentials.AccountName, DestinationContainer.Name, PackageDatabase.DataSource, PackageDatabase.InitialCatalog);
// Gather packages
Log.GatheringListOfPackages(PackageDatabase.DataSource, PackageDatabase.InitialCatalog);
IList <PackageRef> packagesInDB;
using (var connection = await PackageDatabase.ConnectTo())
{
packagesInDB = (await connection.QueryAsync <PackageRef>(@"
SELECT pr.Id, p.NormalizedVersion AS Version, p.Hash, p.LastEdited
FROM Packages p
INNER JOIN PackageRegistrations pr ON p.PackageRegistrationKey = pr.[Key]"))
.ToList();
}
Log.GatheredListOfPackages(packagesInDB.Count, PackageDatabase.DataSource, PackageDatabase.InitialCatalog);
if (!WhatIf)
{
await DestinationContainer.CreateIfNotExistsAsync();
}
// Collect a list of packages in destination with metadata
Log.GatheringDestinationPackages(Destination.BlobEndpoint.ToString(), DestinationContainer.Name);
var destinationPackages = await LoadPackagesAtDestination();
Log.GatheredDestinationPackagesList(Destination.BlobEndpoint.ToString(), DestinationContainer.Name, destinationPackages.Count);
Log.CalculatingCopyOrOverwritePackages(packagesInDB.Count, destinationPackages.Count);
var packagesToCopyOrOverwrite = PackagesToCopyOrOverwrite(packagesInDB, destinationPackages);
Log.CalculatedCopyOrOverwritePackages(packagesInDB.Count, packagesToCopyOrOverwrite.Count);
Log.StartingSync(packagesToCopyOrOverwrite.Count);
if (packagesToCopyOrOverwrite.Count > 0)
{
var policy = new SharedAccessBlobPolicy();
policy.SharedAccessStartTime = DateTimeOffset.Now;
policy.SharedAccessExpiryTime = DateTimeOffset.Now + TimeSpan.FromHours(2) + TimeSpan.FromMinutes(2 * packagesToCopyOrOverwrite.Count);
policy.Permissions = SharedAccessBlobPermissions.Read;
var sourceContainerSharedAccessUri = SourceContainer.GetSharedAccessSignature(policy);
Log.SharedAccessSignatureURI(sourceContainerSharedAccessUri);
foreach (var packageRef in packagesToCopyOrOverwrite)
{
await CopyOrOverwritePackage(sourceContainerSharedAccessUri, StorageHelpers.GetPackageBackupBlobName(packageRef),
StorageHelpers.GetPackageBlobName(packageRef), packageRef.Hash);
if ((Invocation.NextVisibleAt - DateTimeOffset.UtcNow) < TimeSpan.FromMinutes(1))
{
// Running out of time! Extend the job
Log.ExtendingJobLeaseWhileSyncingProgresses();
await Extend(TimeSpan.FromMinutes(5));
Log.ExtendedJobLease();
}
else
{
Log.JobLeaseOk();
}
}
}
Log.CalculatingDeletePackages(destinationPackages.Count, packagesInDB.Count);
var packagesToDelete = PackagesToDelete(packagesInDB, destinationPackages);
Log.CalculatedDeletePackages(packagesToDelete.Count, destinationPackages.Count);
if (packagesToDelete.Count > 0)
{
foreach (var packageBlobName in packagesToDelete)
{
await DeletePackage(packageBlobName);
if ((Invocation.NextVisibleAt - DateTimeOffset.UtcNow) < TimeSpan.FromMinutes(1))
{
// Running out of time! Extend the job
Log.ExtendingJobLeaseWhileSyncingProgresses();
await Extend(TimeSpan.FromMinutes(5));
Log.ExtendedJobLease();
}
else
{
Log.JobLeaseOk();
}
}
}
Log.StartedSync();
}
protected internal override async Task Execute()
{
Source = Source ?? Config.Storage.Legacy;
Destination = Destination ?? Config.Storage.Backup;
SourceContainer = Source.CreateCloudBlobClient().GetContainerReference(
String.IsNullOrEmpty(SourceContainerName) ? BlobContainerNames.Backups : SourceContainerName);
DestinationContainer = Destination.CreateCloudBlobClient().GetContainerReference(
String.IsNullOrEmpty(DestinationContainerName) ? BlobContainerNames.Backups : DestinationContainerName);
Log.PreparingToTransfer(Source.Credentials.AccountName, SourceContainer.Name, Destination.Credentials.AccountName, DestinationContainer.Name);
// Gather packages
Log.GatheringListOfPackages(Source.BlobEndpoint.ToString(), SourceContainer.Name);
var sourcePackages = await LoadBlobList(Log, Source, SourceContainer);
Log.GatheredListOfPackages(sourcePackages.Count, Source.BlobEndpoint.ToString(), SourceContainer.Name);
if (!WhatIf)
{
await DestinationContainer.CreateIfNotExistsAsync();
}
// Collect a list of packages in destination with metadata
Log.GatheringListOfPackages(Destination.BlobEndpoint.ToString(), DestinationContainer.Name);
var destinationPackages = await LoadBlobList(Log, Destination, DestinationContainer);
Log.GatheredListOfPackages(destinationPackages.Count, Destination.BlobEndpoint.ToString(), DestinationContainer.Name);
Log.CalculatingCopyPackages(sourcePackages.Count, destinationPackages.Count);
var packageBlobsToCopy = sourcePackages.Where(p => !destinationPackages.Contains(p)).ToList();
Log.CalculatedCopyPackages(sourcePackages.Count, packageBlobsToCopy.Count);
Log.StartingTransfer(packageBlobsToCopy.Count);
if (packageBlobsToCopy.Count > 0)
{
var policy = new SharedAccessBlobPolicy();
policy.Permissions = SharedAccessBlobPermissions.Read;
foreach (var packageBlobName in packageBlobsToCopy)
{
await CopyPackageToDestination(policy, packageBlobName);
if ((Invocation.NextVisibleAt - DateTimeOffset.UtcNow) < TimeSpan.FromMinutes(1))
{
// Running out of time! Extend the job
Log.ExtendingJobLeaseWhileTransferProgresses();
await Extend(TimeSpan.FromMinutes(5));
Log.ExtendedJobLease();
}
else
{
Log.JobLeaseOk();
}
}
}
Log.StartedTransfer();
}
protected internal override async Task Execute()
{
var now = DateTimeOffset.UtcNow;
// Load default data if not provided
PackageDatabase = PackageDatabase ?? Config.Sql.GetConnectionString(KnownSqlConnection.Legacy);
Source = Source ?? Config.Storage.Legacy;
Destination = Destination ?? Config.Storage.Legacy;
SourceContainer = Source.CreateCloudBlobClient().GetContainerReference(
String.IsNullOrEmpty(SourceContainerName) ? BlobContainerNames.LegacyPackages : SourceContainerName);
DestinationContainer = Destination.CreateCloudBlobClient().GetContainerReference(
String.IsNullOrEmpty(DestinationContainerName) ? BlobContainerNames.Backups : DestinationContainerName);
Log.PreparingToBackup(Source.Credentials.AccountName, SourceContainer.Name, Destination.Credentials.AccountName, DestinationContainer.Name, PackageDatabase.DataSource, PackageDatabase.InitialCatalog);
// Gather packages
Log.GatheringListOfPackages(PackageDatabase.DataSource, PackageDatabase.InitialCatalog);
IList <PackageRef> packages;
using (var connection = await PackageDatabase.ConnectTo()) {
packages = (await connection.QueryAsync <PackageRef>(@"
SELECT pr.Id, p.NormalizedVersion AS Version, p.Hash
FROM Packages p
INNER JOIN PackageRegistrations pr ON p.PackageRegistrationKey = pr.[Key]"))
.ToList();
}
Log.GatheredListOfPackages(packages.Count, PackageDatabase.DataSource, PackageDatabase.InitialCatalog);
// Collect a list of backups
Log.GatheringBackupList(Destination.Credentials.AccountName, DestinationContainer.Name);
var backups = await LoadBackupsList();
Log.GatheredBackupList(Destination.Credentials.AccountName, DestinationContainer.Name, backups.Count);
// Calculate needed backups
Log.CalculatingBackupSet(packages.Count, backups.Count);
var backupSet = CalculateBackupSet(packages, backups);
Log.CalculatedBackupSet(packages.Count, backupSet.Count);
if (!WhatIf)
{
await DestinationContainer.CreateIfNotExistsAsync();
}
Log.StartingBackup(packages.Count);
if (RunInParallel)
{
Parallel.ForEach(
backupSet,
new ParallelOptions()
{
MaxDegreeOfParallelism = TaskPerCoreFactor * Environment.ProcessorCount
},
t => BackupPackage(t.Item1, t.Item2).Wait());
}
else
{
foreach (var backupRecord in backupSet)
{
await BackupPackage(backupRecord.Item1, backupRecord.Item2);
if ((Invocation.NextVisibleAt - DateTimeOffset.UtcNow) < TimeSpan.FromMinutes(1))
{
// Running out of time! Extend the job
Log.ExtendingJobLeaseWhileBackupProgresses();
await Extend(TimeSpan.FromMinutes(5));
Log.ExtendedJobLease();
}
else
{
Log.JobLeaseOk();
}
}
}
Log.StartedBackup();
}
