private async Task ThrowDestinationExceptionIfNeeded(DestinationException ex, DataServicePackageWithCreated package, JObject mirrorJson, SqlConnectionStringBuilder cstr, CloudStorageAccount account)
{
var inner = ex.InnerException;
HttpStatusCode?code = (inner != null && inner is InvalidOperationException) ? GetHttpStatusCodeFrom(inner.InnerException) : GetHttpStatusCodeFrom(inner);
if (code == null || code != HttpStatusCode.Conflict)
{
throw ex;
}
switch (code)
{
// '409 Conflict' from Destination- Package already exists in destination. Don't rethrow
case HttpStatusCode.Conflict:
var sourceJObject = GetJObject(mirrorJson, package.Id, package.SemanticVersion);
if (sourceJObject == null)
{
throw new InvalidOperationException("Package" + package.Id + "//" + package.SemanticVersion.ToString() + "is already mirrored, but, not present in mirror.json. WRONG!");
}
var oldSourceCreated = sourceJObject[SourceCreatedKey].Value <DateTime>();
var newSourceCreated = new DateTime(package.Created.Value.DateTime.Ticks, DateTimeKind.Utc);
if (!newSourceCreated.Equals(oldSourceCreated))
{
// This package while already mirrored to the destination, has been deleted from the source and created again to the source
// Hence, the different SourceCreated Date
// Need to delete the package
Log.DeletingOldRevision(package.ToString(), oldSourceCreated.ToString(DateTimeFormatSpecifier), newSourceCreated.ToString(DateTimeFormatSpecifier));
await NuGetV2RepositoryMirrorPackageDeletor.DeletePackage(cstr, account, sourceJObject, package.Id, package.SemanticVersion.ToString());
throw ex;
}
Log.PackageAlreadyExists(package.ToString());
break;
// Any other code or if code is null. Throw
default:
throw ex;
}
}
/// <summary>
/// Queries for packages created after 'lastCreated'. At most, 40 packages may be returned
/// Mirror that batch of packages to the destination server
/// </summary>
/// <returns>Returns lastMirroredPackage or null</returns>
private DataServicePackageWithCreated QueryAndMirrorBatch(DataServices.DataServiceContext serviceContext, PackageServer destinationServer, DateTime lastCreated,
string apiKey, int timeOut, JObject mirrorJson, ref int retries, ref int count, ref int skipIndex, SqlConnectionStringBuilder cstr, CloudStorageAccount account)
{
var newPackages = GetNewPackagesToMirror(serviceContext, lastCreated);
// Push packages
var tempFolderPath = GetTempFolderPath(serviceContext.BaseUri.DnsSafeHost);
Log.TempFolderPath(tempFolderPath);
var tempLocalRepo = new LocalPackageRepository(tempFolderPath);
var tempPackageManager = new PackageManager(new DataServicePackageRepository(serviceContext.BaseUri), tempFolderPath);
DataServicePackageWithCreated currentPackage = null;
DataServicePackageWithCreated lastMirroredPackage = null;
try
{
do
{
// The following code deletes the temp folder if one exists and creates a new one
GetTempFolderPath(serviceContext.BaseUri.DnsSafeHost);
var newPackagesList = newPackages.Skip(skipIndex).ToList();
Log.PackagesCopyCount(newPackagesList.Count);
if (newPackagesList.Count == 0)
{
break;
}
foreach (DataServicePackageWithCreated package in newPackagesList)
{
try
{
currentPackage = package;
MirrorPackage(package, destinationServer, tempPackageManager, tempLocalRepo, apiKey, timeOut);
var jObject = AddNewPackage(mirrorJson, package);
if (!package.IsListed)
{
// The new package being pushed is not listed. Mark it as unlisted
NuGetV2RepositoryMirrorPackageDeletor.SetListed(cstr, jObject, package.Id, package.SemanticVersion.ToString(), false).Wait();
}
Log.PushedToDestination(++count);
}
catch (SourceException ex)
{
ThrowSourceExceptionIfNeeded(ex, ref retries, package);
}
catch (DestinationException ex)
{
ThrowDestinationExceptionIfNeeded(ex, package, mirrorJson, cstr, account).Wait();
}
lastMirroredPackage = package;
retries = 0;
++skipIndex;
}
} while (true);
}
catch (Exception ex)
{
retries++;
Log.ServerUnreachable(retries, ex.Message);
if (currentPackage != null)
{
Log.MirrorFailed(currentPackage.ToString());
}
}
// Delete the packages stored locally
Directory.Delete(tempFolderPath, recursive: true);
Log.DeletedTempFolder(tempFolderPath);
return(lastMirroredPackage);
}
protected async internal override Task Execute()
{
// Validate mandatory parameters
if (String.IsNullOrEmpty(SourceV2Feed))
{
throw new ArgumentException("SourceV2Feed cannot be null or empty");
}
// If the SourceV2Feed is not a URI, the following line will throw URIFormatException
var sourceV2FeedUri = new Uri(SourceV2Feed);
if (String.IsNullOrEmpty(DestinationUri))
{
throw new ArgumentException("DestinationUri cannot be null or empty");
}
if (String.IsNullOrEmpty(ApiKey))
{
throw new ArgumentException("ApiKey cannot be null or empty");
}
// Packages are in Legacy account. PackageDatabase is the InitialCatalog in the legacy account
var account = Config.Storage.Legacy;
var cstr = Config.Sql.Legacy;
if (cstr == null)
{
throw new ArgumentNullException("Legacy sql cannot be null");
}
// Arrange or set defaults for parameters that are not provided
UserAgent = String.Format("{0}v2FeedMirrorer", sourceV2FeedUri.DnsSafeHost);
var timeOutPerPush = TimeSpan.FromMinutes(PushTimeOutInMinutes);
MirrorStorage = MirrorStorage ?? Config.Storage.Legacy;
if (MirrorStorage == null)
{
throw new ArgumentNullException("MirrorStorage", "Mirror storage is not provided or present in the config");
}
MirrorBlobContainerName = String.IsNullOrEmpty(MirrorBlobContainerName) ? DefaultMirrorBlobContainerName : MirrorBlobContainerName;
MirrorBlobContainer = MirrorStorage.CreateCloudBlobClient().GetContainerReference(MirrorBlobContainerName);
MirrorBlobName = String.IsNullOrEmpty(MirrorBlobName) ? DefaultMirrorBlobName : MirrorBlobName;
var sourceUri = new Uri(sourceV2FeedUri, "/api/v2/");
var serviceContext = new DataServices.DataServiceContext(sourceUri)
{
MergeOption = DataServices.MergeOption.OverwriteChanges,
IgnoreMissingProperties = true,
};
var mirrorJson = await GetJObject(MirrorBlobContainer, MirrorBlobName);
if (!IsMirrorJsonValid(mirrorJson))
{
throw new InvalidOperationException("mirrorJson is not valid. Either packageIndex array is not present. Or, the elements are not sorted by SourceCreatedDate");
}
Exception caughtException = null;
if (ExecuteDeletes)
{
try
{
await NuGetV2RepositoryMirrorPackageDeletor.DeleteAndSetListedPackages(sourceUri, mirrorJson, account, cstr);
}
catch (Exception ex)
{
caughtException = ex;
}
await SetJObject(MirrorBlobContainer, MirrorBlobName, mirrorJson);
if (caughtException != null)
{
throw caughtException;
}
return;
}
var oldLastCreated = mirrorJson.Value <DateTime>(LastCreatedKey);
var lastCreated = oldLastCreated;
Log.PreparingToMirror(MirrorBlobName, lastCreated.ToString(DateTimeFormatSpecifier), lastCreated.Kind.ToString(), sourceUri.AbsoluteUri, DestinationUri);
int retries = 0;
int count = 0;
int skipIndex = 0;
//
// POSSIBLE ACTIONS when an error is encountered
// A) Always Skip to next package
// B) Retry 'MaxRetries' times and skip to next package
// C) Retry 'MaxRetries' times and fail
//
// KNOWN ERRORS
// 1) '409 Conflict' from Destination- Package already exists in destination
// i) Action: (A). Always Skip to next package
// ii) Before skipping, Update lastMirroredPackage.Created locally, i.e inside the while loop
// 2) '403 Forbidden' from Source. For reasons unknown, certain listed packages are not available for download. Source returns "Access Denied"
// i) Action: (B). Retry 'MaxRetries' times and Skip to next package
// ii) Log every retry. Before skipping, Update lastMirroredPackage.Created locally, i.e inside the while loop
// 3) '404 Not Found' from Source. Package is available on the feed but not available for download already
// i) Action: (C). Retry 'MaxRetries' times and Fail
// ii) Log every retry. Update lastMirroredPackage.Created in blob storage
// 4) Unknown Error
// i) Action: (C). Retry 'MaxRetries' times and Fail
// ii) Log every retry. Update lastMirroredPackage.Created in blob storage
//
// Test Cases
// 1) Add a new package to the source. COVERED
// Result: The new package should be present on the destination
// 2) Add a package as unlisted to the source. COVERED
// Result: The new package should be present on the destination as unlisted
// 3) Delete a package version from the source. COVERED
// Result: The package should be deleted from the destination
// 4) Delete a package version from the source which is the last version with the package Id. NOT COVERED
// Result: The package should be deleted from the destination. And, PackageRegistration should be deleted too as appropriate
// 5) Delete a package version from the source. And, add a new package with same Id and version as the deleted one. COVERED
// Result: Old package with Id and version must be deleted from the destination. And, the new package with the same Id and Version must be added
// 6) Mark a listed package as unlisted. COVERED
// Result: The package should be unlisted in the destination too
// 7) Mark an unlisted package as listed. COVERED
// Result: The package should be listed in the destination too
//
try
{
do
{
if ((Invocation.NextVisibleAt - DateTimeOffset.UtcNow) < TimeSpan.FromMinutes(120))
{
// Based on default configuration on the repository, there are at most 40 packages returned by a single query (which will get downloaded and pushed)
// With a conservative estimate of 3 minutes per package, expecting a minimum of
// 120 minutes to run 1 iteration of this do-while loop
await Extend(TimeSpan.FromMinutes(120));
}
// In each query, at most, 40 packages may be returned. So, continue performing the queries
// in this do-while, so long as there are results returned
// Query for packages created since oldLastCreated
PackageServer destinationServer = new PackageServer(DestinationUri, UserAgent);
var lastMirroredPackage = QueryAndMirrorBatch(serviceContext, destinationServer, oldLastCreated, ApiKey, timeOutPerPush.Milliseconds, mirrorJson, ref retries, ref count, ref skipIndex, cstr, account);
if (lastMirroredPackage != null)
{
if (!lastMirroredPackage.Created.HasValue)
{
throw new InvalidOperationException("Last mirrored package : " + lastMirroredPackage.ToString() + "has a null Created Time.. WRONG!!!");
}
// Note that the Created DateTime is always stored in UTC, but the DateTimeKind of the value obtained is Unspecified
// So, store it as UTC
lastCreated = new DateTime(lastMirroredPackage.Created.Value.DateTime.Ticks, DateTimeKind.Utc);
Log.EndOfIteration(lastCreated.ToString(DateTimeFormatSpecifier));
}
else if (retries == 0 || retries > MaxRetries)
{
break;
}
} while (true);
}
catch (Exception ex)
{
// Catch the exception here so that new lastCreated is always stored
// We can throw this exception at the end
caughtException = ex;
}
if (!oldLastCreated.Equals(lastCreated))
{
mirrorJson[LastCreatedKey] = lastCreated.ToString(DateTimeFormatSpecifier);
await SetJObject(MirrorBlobContainer, MirrorBlobName, mirrorJson);
Log.NewLastCreatedAtEndOfInvocation(MirrorBlobName, lastCreated.ToString(DateTimeFormatSpecifier), lastCreated.Kind.ToString());
}
if (caughtException != null)
{
throw caughtException;
}
}
