private static List <DataServicePackageWithCreated> GetUnlistedPackages(Uri sourceUri, DateTime lastCreated)
{
var serviceContext = new DataServices.DataServiceContext(sourceUri)
{
MergeOption = DataServices.MergeOption.OverwriteChanges,
IgnoreMissingProperties = true,
};
int skipIndex = 0;
var unlistedPackages = GetUnlistedPackages(serviceContext, lastCreated);
var allUnlistedPackagesList = new List <DataServicePackageWithCreated>();
do
{
var unlistedPackagesList = unlistedPackages.Skip(skipIndex).ToList();
if (unlistedPackagesList.Count == 0)
{
break;
}
allUnlistedPackagesList.AddRange(unlistedPackagesList);
skipIndex = skipIndex + unlistedPackagesList.Count;
} while (true);
return(allUnlistedPackagesList);
}
private static List <int> GetDeletedPackageIndices(Uri sourceUri, List <MinPackage> destinationPackages, int startIndex, int endIndex, DateTime dateTime)
{
var serviceContext = new DataServices.DataServiceContext(sourceUri)
{
MergeOption = DataServices.MergeOption.OverwriteChanges,
IgnoreMissingProperties = true,
};
var packagesQuery = serviceContext.CreateQuery <DataServicePackageWithCreated>("Packages");
var queryOptionValue = String.Format("Created eq DateTime'{0}'", dateTime.ToString("o"));
packagesQuery = packagesQuery.AddQueryOption("$orderby", "Id");
packagesQuery = packagesQuery.AddQueryOption("$orderby", "Version");
packagesQuery = packagesQuery.AddQueryOption("$filter", queryOptionValue);
var skipIndex = 0;
var sourcePackages = new List <DataServicePackageWithCreated>();
do
{
var list = packagesQuery.Skip(skipIndex).ToList();
if (list.Count == 0)
{
break;
}
sourcePackages.AddRange(list);
skipIndex = skipIndex + list.Count;
}while(true);
if (sourcePackages.Count == 0)
{
throw new InvalidOperationException("Don't call this method when it is already known that there are no source packages");
}
var deletedPackageIndices = new List <int>();
for (int i = startIndex; i < endIndex; i++)
{
var destPackage = destinationPackages[i];
var destPackageInSource = sourcePackages.Where(s => String.Equals(s.Id, destPackage.Id) && s.SemanticVersion.Equals(destPackage.SemanticVersion)).SingleOrDefault();
if (destPackageInSource == null)
{
// This destination package is not present in source
// Add index to list of deleted packages
deletedPackageIndices.Add(i);
}
}
return(deletedPackageIndices);
}
/// <summary>
/// Queries for packages created after 'lastCreated'. At most, 40 packages may be returned
/// Mirror that batch of packages to the destination server
/// </summary>
/// <returns>Returns lastMirroredPackage or null</returns>
private DataServicePackageWithCreated QueryAndMirrorBatch(DataServices.DataServiceContext serviceContext, PackageServer destinationServer, DateTime lastCreated,
string apiKey, int timeOut, JObject mirrorJson, ref int retries, ref int count, ref int skipIndex, SqlConnectionStringBuilder cstr, CloudStorageAccount account)
{
var newPackages = GetNewPackagesToMirror(serviceContext, lastCreated);
// Push packages
var tempFolderPath = GetTempFolderPath(serviceContext.BaseUri.DnsSafeHost);
Log.TempFolderPath(tempFolderPath);
var tempLocalRepo = new LocalPackageRepository(tempFolderPath);
var tempPackageManager = new PackageManager(new DataServicePackageRepository(serviceContext.BaseUri), tempFolderPath);
DataServicePackageWithCreated currentPackage = null;
DataServicePackageWithCreated lastMirroredPackage = null;
try
{
do
{
// The following code deletes the temp folder if one exists and creates a new one
GetTempFolderPath(serviceContext.BaseUri.DnsSafeHost);
var newPackagesList = newPackages.Skip(skipIndex).ToList();
Log.PackagesCopyCount(newPackagesList.Count);
if (newPackagesList.Count == 0)
{
break;
}
foreach (DataServicePackageWithCreated package in newPackagesList)
{
try
{
currentPackage = package;
MirrorPackage(package, destinationServer, tempPackageManager, tempLocalRepo, apiKey, timeOut);
var jObject = AddNewPackage(mirrorJson, package);
if (!package.IsListed)
{
// The new package being pushed is not listed. Mark it as unlisted
NuGetV2RepositoryMirrorPackageDeletor.SetListed(cstr, jObject, package.Id, package.SemanticVersion.ToString(), false).Wait();
}
Log.PushedToDestination(++count);
}
catch (SourceException ex)
{
ThrowSourceExceptionIfNeeded(ex, ref retries, package);
}
catch (DestinationException ex)
{
ThrowDestinationExceptionIfNeeded(ex, package, mirrorJson, cstr, account).Wait();
}
lastMirroredPackage = package;
retries = 0;
++skipIndex;
}
} while (true);
}
catch (Exception ex)
{
retries++;
Log.ServerUnreachable(retries, ex.Message);
if (currentPackage != null)
{
Log.MirrorFailed(currentPackage.ToString());
}
}
// Delete the packages stored locally
Directory.Delete(tempFolderPath, recursive: true);
Log.DeletedTempFolder(tempFolderPath);
return(lastMirroredPackage);
}
private DataServices.DataServiceQuery <DataServicePackageWithCreated> GetNewPackagesToMirror(DataServices.DataServiceContext serviceContext, DateTime lastCreated)
{
Log.QueryNewPackages(serviceContext.BaseUri.AbsoluteUri, lastCreated.ToString(DateTimeFormatSpecifier));
var packagesQuery = serviceContext.CreateQuery <DataServicePackageWithCreated>("Packages");
// The following query gets packages created after 'lastCreated' and sorted by 'Created' ascending
var queryOptionValue = String.Format("Created gt DateTime'{0}'", lastCreated.ToString(DateTimeFormatSpecifier));
packagesQuery = packagesQuery.AddQueryOption("$orderby", "Created, Id, Version");
packagesQuery = packagesQuery.AddQueryOption("$filter", queryOptionValue);
Log.QueryFilter(queryOptionValue);
return(packagesQuery);
}
protected async internal override Task Execute()
{
// Validate mandatory parameters
if (String.IsNullOrEmpty(SourceV2Feed))
{
throw new ArgumentException("SourceV2Feed cannot be null or empty");
}
// If the SourceV2Feed is not a URI, the following line will throw URIFormatException
var sourceV2FeedUri = new Uri(SourceV2Feed);
if (String.IsNullOrEmpty(DestinationUri))
{
throw new ArgumentException("DestinationUri cannot be null or empty");
}
if (String.IsNullOrEmpty(ApiKey))
{
throw new ArgumentException("ApiKey cannot be null or empty");
}
// Packages are in Legacy account. PackageDatabase is the InitialCatalog in the legacy account
var account = Config.Storage.Legacy;
var cstr = Config.Sql.Legacy;
if (cstr == null)
{
throw new ArgumentNullException("Legacy sql cannot be null");
}
// Arrange or set defaults for parameters that are not provided
UserAgent = String.Format("{0}v2FeedMirrorer", sourceV2FeedUri.DnsSafeHost);
var timeOutPerPush = TimeSpan.FromMinutes(PushTimeOutInMinutes);
MirrorStorage = MirrorStorage ?? Config.Storage.Legacy;
if (MirrorStorage == null)
{
throw new ArgumentNullException("MirrorStorage", "Mirror storage is not provided or present in the config");
}
MirrorBlobContainerName = String.IsNullOrEmpty(MirrorBlobContainerName) ? DefaultMirrorBlobContainerName : MirrorBlobContainerName;
MirrorBlobContainer = MirrorStorage.CreateCloudBlobClient().GetContainerReference(MirrorBlobContainerName);
MirrorBlobName = String.IsNullOrEmpty(MirrorBlobName) ? DefaultMirrorBlobName : MirrorBlobName;
var sourceUri = new Uri(sourceV2FeedUri, "/api/v2/");
var serviceContext = new DataServices.DataServiceContext(sourceUri)
{
MergeOption = DataServices.MergeOption.OverwriteChanges,
IgnoreMissingProperties = true,
};
var mirrorJson = await GetJObject(MirrorBlobContainer, MirrorBlobName);
if (!IsMirrorJsonValid(mirrorJson))
{
throw new InvalidOperationException("mirrorJson is not valid. Either packageIndex array is not present. Or, the elements are not sorted by SourceCreatedDate");
}
Exception caughtException = null;
if (ExecuteDeletes)
{
try
{
await NuGetV2RepositoryMirrorPackageDeletor.DeleteAndSetListedPackages(sourceUri, mirrorJson, account, cstr);
}
catch (Exception ex)
{
caughtException = ex;
}
await SetJObject(MirrorBlobContainer, MirrorBlobName, mirrorJson);
if (caughtException != null)
{
throw caughtException;
}
return;
}
var oldLastCreated = mirrorJson.Value <DateTime>(LastCreatedKey);
var lastCreated = oldLastCreated;
Log.PreparingToMirror(MirrorBlobName, lastCreated.ToString(DateTimeFormatSpecifier), lastCreated.Kind.ToString(), sourceUri.AbsoluteUri, DestinationUri);
int retries = 0;
int count = 0;
int skipIndex = 0;
//
// POSSIBLE ACTIONS when an error is encountered
// A) Always Skip to next package
// B) Retry 'MaxRetries' times and skip to next package
// C) Retry 'MaxRetries' times and fail
//
// KNOWN ERRORS
// 1) '409 Conflict' from Destination- Package already exists in destination
// i) Action: (A). Always Skip to next package
// ii) Before skipping, Update lastMirroredPackage.Created locally, i.e inside the while loop
// 2) '403 Forbidden' from Source. For reasons unknown, certain listed packages are not available for download. Source returns "Access Denied"
// i) Action: (B). Retry 'MaxRetries' times and Skip to next package
// ii) Log every retry. Before skipping, Update lastMirroredPackage.Created locally, i.e inside the while loop
// 3) '404 Not Found' from Source. Package is available on the feed but not available for download already
// i) Action: (C). Retry 'MaxRetries' times and Fail
// ii) Log every retry. Update lastMirroredPackage.Created in blob storage
// 4) Unknown Error
// i) Action: (C). Retry 'MaxRetries' times and Fail
// ii) Log every retry. Update lastMirroredPackage.Created in blob storage
//
// Test Cases
// 1) Add a new package to the source. COVERED
// Result: The new package should be present on the destination
// 2) Add a package as unlisted to the source. COVERED
// Result: The new package should be present on the destination as unlisted
// 3) Delete a package version from the source. COVERED
// Result: The package should be deleted from the destination
// 4) Delete a package version from the source which is the last version with the package Id. NOT COVERED
// Result: The package should be deleted from the destination. And, PackageRegistration should be deleted too as appropriate
// 5) Delete a package version from the source. And, add a new package with same Id and version as the deleted one. COVERED
// Result: Old package with Id and version must be deleted from the destination. And, the new package with the same Id and Version must be added
// 6) Mark a listed package as unlisted. COVERED
// Result: The package should be unlisted in the destination too
// 7) Mark an unlisted package as listed. COVERED
// Result: The package should be listed in the destination too
//
try
{
do
{
if ((Invocation.NextVisibleAt - DateTimeOffset.UtcNow) < TimeSpan.FromMinutes(120))
{
// Based on default configuration on the repository, there are at most 40 packages returned by a single query (which will get downloaded and pushed)
// With a conservative estimate of 3 minutes per package, expecting a minimum of
// 120 minutes to run 1 iteration of this do-while loop
await Extend(TimeSpan.FromMinutes(120));
}
// In each query, at most, 40 packages may be returned. So, continue performing the queries
// in this do-while, so long as there are results returned
// Query for packages created since oldLastCreated
PackageServer destinationServer = new PackageServer(DestinationUri, UserAgent);
var lastMirroredPackage = QueryAndMirrorBatch(serviceContext, destinationServer, oldLastCreated, ApiKey, timeOutPerPush.Milliseconds, mirrorJson, ref retries, ref count, ref skipIndex, cstr, account);
if (lastMirroredPackage != null)
{
if (!lastMirroredPackage.Created.HasValue)
{
throw new InvalidOperationException("Last mirrored package : " + lastMirroredPackage.ToString() + "has a null Created Time.. WRONG!!!");
}
// Note that the Created DateTime is always stored in UTC, but the DateTimeKind of the value obtained is Unspecified
// So, store it as UTC
lastCreated = new DateTime(lastMirroredPackage.Created.Value.DateTime.Ticks, DateTimeKind.Utc);
Log.EndOfIteration(lastCreated.ToString(DateTimeFormatSpecifier));
}
else if (retries == 0 || retries > MaxRetries)
{
break;
}
} while (true);
}
catch (Exception ex)
{
// Catch the exception here so that new lastCreated is always stored
// We can throw this exception at the end
caughtException = ex;
}
if (!oldLastCreated.Equals(lastCreated))
{
mirrorJson[LastCreatedKey] = lastCreated.ToString(DateTimeFormatSpecifier);
await SetJObject(MirrorBlobContainer, MirrorBlobName, mirrorJson);
Log.NewLastCreatedAtEndOfInvocation(MirrorBlobName, lastCreated.ToString(DateTimeFormatSpecifier), lastCreated.Kind.ToString());
}
if (caughtException != null)
{
throw caughtException;
}
}
private static DataServices.DataServiceQuery <DataServicePackageWithCreated> GetUnlistedPackages(DataServices.DataServiceContext serviceContext, DateTime lastCreated)
{
var packagesQuery = serviceContext.CreateQuery <DataServicePackageWithCreated>("Packages");
// The following query gets unlisted packages created on or before 'lastCreated'
var queryOptionValue = String.Format(UnlistedPackagesQueryOption, lastCreated.ToString("O"));
packagesQuery = packagesQuery.AddQueryOption("$orderby", "Created, Id, Version");
packagesQuery = packagesQuery.AddQueryOption("$filter", queryOptionValue);
return(packagesQuery);
}
