private async Task ThrowDestinationExceptionIfNeeded(DestinationException ex, DataServicePackageWithCreated package, JObject mirrorJson, SqlConnectionStringBuilder cstr, CloudStorageAccount account) { var inner = ex.InnerException; HttpStatusCode?code = (inner != null && inner is InvalidOperationException) ? GetHttpStatusCodeFrom(inner.InnerException) : GetHttpStatusCodeFrom(inner); if (code == null || code != HttpStatusCode.Conflict) { throw ex; } switch (code) { // '409 Conflict' from Destination- Package already exists in destination. Don't rethrow case HttpStatusCode.Conflict: var sourceJObject = GetJObject(mirrorJson, package.Id, package.SemanticVersion); if (sourceJObject == null) { throw new InvalidOperationException("Package" + package.Id + "//" + package.SemanticVersion.ToString() + "is already mirrored, but, not present in mirror.json. WRONG!"); } var oldSourceCreated = sourceJObject[SourceCreatedKey].Value <DateTime>(); var newSourceCreated = new DateTime(package.Created.Value.DateTime.Ticks, DateTimeKind.Utc); if (!newSourceCreated.Equals(oldSourceCreated)) { // This package while already mirrored to the destination, has been deleted from the source and created again to the source // Hence, the different SourceCreated Date // Need to delete the package Log.DeletingOldRevision(package.ToString(), oldSourceCreated.ToString(DateTimeFormatSpecifier), newSourceCreated.ToString(DateTimeFormatSpecifier)); await NuGetV2RepositoryMirrorPackageDeletor.DeletePackage(cstr, account, sourceJObject, package.Id, package.SemanticVersion.ToString()); throw ex; } Log.PackageAlreadyExists(package.ToString()); break; // Any other code or if code is null. Throw default: throw ex; } }
/// <summary> /// Queries for packages created after 'lastCreated'. At most, 40 packages may be returned /// Mirror that batch of packages to the destination server /// </summary> /// <returns>Returns lastMirroredPackage or null</returns> private DataServicePackageWithCreated QueryAndMirrorBatch(DataServices.DataServiceContext serviceContext, PackageServer destinationServer, DateTime lastCreated, string apiKey, int timeOut, JObject mirrorJson, ref int retries, ref int count, ref int skipIndex, SqlConnectionStringBuilder cstr, CloudStorageAccount account) { var newPackages = GetNewPackagesToMirror(serviceContext, lastCreated); // Push packages var tempFolderPath = GetTempFolderPath(serviceContext.BaseUri.DnsSafeHost); Log.TempFolderPath(tempFolderPath); var tempLocalRepo = new LocalPackageRepository(tempFolderPath); var tempPackageManager = new PackageManager(new DataServicePackageRepository(serviceContext.BaseUri), tempFolderPath); DataServicePackageWithCreated currentPackage = null; DataServicePackageWithCreated lastMirroredPackage = null; try { do { // The following code deletes the temp folder if one exists and creates a new one GetTempFolderPath(serviceContext.BaseUri.DnsSafeHost); var newPackagesList = newPackages.Skip(skipIndex).ToList(); Log.PackagesCopyCount(newPackagesList.Count); if (newPackagesList.Count == 0) { break; } foreach (DataServicePackageWithCreated package in newPackagesList) { try { currentPackage = package; MirrorPackage(package, destinationServer, tempPackageManager, tempLocalRepo, apiKey, timeOut); var jObject = AddNewPackage(mirrorJson, package); if (!package.IsListed) { // The new package being pushed is not listed. Mark it as unlisted NuGetV2RepositoryMirrorPackageDeletor.SetListed(cstr, jObject, package.Id, package.SemanticVersion.ToString(), false).Wait(); } Log.PushedToDestination(++count); } catch (SourceException ex) { ThrowSourceExceptionIfNeeded(ex, ref retries, package); } catch (DestinationException ex) { ThrowDestinationExceptionIfNeeded(ex, package, mirrorJson, cstr, account).Wait(); } lastMirroredPackage = package; retries = 0; ++skipIndex; } } while (true); } catch (Exception ex) { retries++; Log.ServerUnreachable(retries, ex.Message); if (currentPackage != null) { Log.MirrorFailed(currentPackage.ToString()); } } // Delete the packages stored locally Directory.Delete(tempFolderPath, recursive: true); Log.DeletedTempFolder(tempFolderPath); return(lastMirroredPackage); }
protected async internal override Task Execute() { // Validate mandatory parameters if (String.IsNullOrEmpty(SourceV2Feed)) { throw new ArgumentException("SourceV2Feed cannot be null or empty"); } // If the SourceV2Feed is not a URI, the following line will throw URIFormatException var sourceV2FeedUri = new Uri(SourceV2Feed); if (String.IsNullOrEmpty(DestinationUri)) { throw new ArgumentException("DestinationUri cannot be null or empty"); } if (String.IsNullOrEmpty(ApiKey)) { throw new ArgumentException("ApiKey cannot be null or empty"); } // Packages are in Legacy account. PackageDatabase is the InitialCatalog in the legacy account var account = Config.Storage.Legacy; var cstr = Config.Sql.Legacy; if (cstr == null) { throw new ArgumentNullException("Legacy sql cannot be null"); } // Arrange or set defaults for parameters that are not provided UserAgent = String.Format("{0}v2FeedMirrorer", sourceV2FeedUri.DnsSafeHost); var timeOutPerPush = TimeSpan.FromMinutes(PushTimeOutInMinutes); MirrorStorage = MirrorStorage ?? Config.Storage.Legacy; if (MirrorStorage == null) { throw new ArgumentNullException("MirrorStorage", "Mirror storage is not provided or present in the config"); } MirrorBlobContainerName = String.IsNullOrEmpty(MirrorBlobContainerName) ? DefaultMirrorBlobContainerName : MirrorBlobContainerName; MirrorBlobContainer = MirrorStorage.CreateCloudBlobClient().GetContainerReference(MirrorBlobContainerName); MirrorBlobName = String.IsNullOrEmpty(MirrorBlobName) ? DefaultMirrorBlobName : MirrorBlobName; var sourceUri = new Uri(sourceV2FeedUri, "/api/v2/"); var serviceContext = new DataServices.DataServiceContext(sourceUri) { MergeOption = DataServices.MergeOption.OverwriteChanges, IgnoreMissingProperties = true, }; var mirrorJson = await GetJObject(MirrorBlobContainer, MirrorBlobName); if (!IsMirrorJsonValid(mirrorJson)) { throw new InvalidOperationException("mirrorJson is not valid. Either packageIndex array is not present. Or, the elements are not sorted by SourceCreatedDate"); } Exception caughtException = null; if (ExecuteDeletes) { try { await NuGetV2RepositoryMirrorPackageDeletor.DeleteAndSetListedPackages(sourceUri, mirrorJson, account, cstr); } catch (Exception ex) { caughtException = ex; } await SetJObject(MirrorBlobContainer, MirrorBlobName, mirrorJson); if (caughtException != null) { throw caughtException; } return; } var oldLastCreated = mirrorJson.Value <DateTime>(LastCreatedKey); var lastCreated = oldLastCreated; Log.PreparingToMirror(MirrorBlobName, lastCreated.ToString(DateTimeFormatSpecifier), lastCreated.Kind.ToString(), sourceUri.AbsoluteUri, DestinationUri); int retries = 0; int count = 0; int skipIndex = 0; // // POSSIBLE ACTIONS when an error is encountered // A) Always Skip to next package // B) Retry 'MaxRetries' times and skip to next package // C) Retry 'MaxRetries' times and fail // // KNOWN ERRORS // 1) '409 Conflict' from Destination- Package already exists in destination // i) Action: (A). Always Skip to next package // ii) Before skipping, Update lastMirroredPackage.Created locally, i.e inside the while loop // 2) '403 Forbidden' from Source. For reasons unknown, certain listed packages are not available for download. Source returns "Access Denied" // i) Action: (B). Retry 'MaxRetries' times and Skip to next package // ii) Log every retry. Before skipping, Update lastMirroredPackage.Created locally, i.e inside the while loop // 3) '404 Not Found' from Source. Package is available on the feed but not available for download already // i) Action: (C). Retry 'MaxRetries' times and Fail // ii) Log every retry. Update lastMirroredPackage.Created in blob storage // 4) Unknown Error // i) Action: (C). Retry 'MaxRetries' times and Fail // ii) Log every retry. Update lastMirroredPackage.Created in blob storage // // Test Cases // 1) Add a new package to the source. COVERED // Result: The new package should be present on the destination // 2) Add a package as unlisted to the source. COVERED // Result: The new package should be present on the destination as unlisted // 3) Delete a package version from the source. COVERED // Result: The package should be deleted from the destination // 4) Delete a package version from the source which is the last version with the package Id. NOT COVERED // Result: The package should be deleted from the destination. And, PackageRegistration should be deleted too as appropriate // 5) Delete a package version from the source. And, add a new package with same Id and version as the deleted one. COVERED // Result: Old package with Id and version must be deleted from the destination. And, the new package with the same Id and Version must be added // 6) Mark a listed package as unlisted. COVERED // Result: The package should be unlisted in the destination too // 7) Mark an unlisted package as listed. COVERED // Result: The package should be listed in the destination too // try { do { if ((Invocation.NextVisibleAt - DateTimeOffset.UtcNow) < TimeSpan.FromMinutes(120)) { // Based on default configuration on the repository, there are at most 40 packages returned by a single query (which will get downloaded and pushed) // With a conservative estimate of 3 minutes per package, expecting a minimum of // 120 minutes to run 1 iteration of this do-while loop await Extend(TimeSpan.FromMinutes(120)); } // In each query, at most, 40 packages may be returned. So, continue performing the queries // in this do-while, so long as there are results returned // Query for packages created since oldLastCreated PackageServer destinationServer = new PackageServer(DestinationUri, UserAgent); var lastMirroredPackage = QueryAndMirrorBatch(serviceContext, destinationServer, oldLastCreated, ApiKey, timeOutPerPush.Milliseconds, mirrorJson, ref retries, ref count, ref skipIndex, cstr, account); if (lastMirroredPackage != null) { if (!lastMirroredPackage.Created.HasValue) { throw new InvalidOperationException("Last mirrored package : " + lastMirroredPackage.ToString() + "has a null Created Time.. WRONG!!!"); } // Note that the Created DateTime is always stored in UTC, but the DateTimeKind of the value obtained is Unspecified // So, store it as UTC lastCreated = new DateTime(lastMirroredPackage.Created.Value.DateTime.Ticks, DateTimeKind.Utc); Log.EndOfIteration(lastCreated.ToString(DateTimeFormatSpecifier)); } else if (retries == 0 || retries > MaxRetries) { break; } } while (true); } catch (Exception ex) { // Catch the exception here so that new lastCreated is always stored // We can throw this exception at the end caughtException = ex; } if (!oldLastCreated.Equals(lastCreated)) { mirrorJson[LastCreatedKey] = lastCreated.ToString(DateTimeFormatSpecifier); await SetJObject(MirrorBlobContainer, MirrorBlobName, mirrorJson); Log.NewLastCreatedAtEndOfInvocation(MirrorBlobName, lastCreated.ToString(DateTimeFormatSpecifier), lastCreated.Kind.ToString()); } if (caughtException != null) { throw caughtException; } }