private void ThrowSourceExceptionIfNeeded(SourceException ex, ref int retries, DataServicePackageWithCreated package) { // TO BE DELETED if (ex.InnerException is PathTooLongException) { Log.LogMessage(String.Format("PathTooLongException on package {0}", package.ToString())); return; } HttpStatusCode?code = GetHttpStatusCodeFrom(ex.InnerException); switch (code) { // '403 Forbidden' from Source. For reasons unknown, certain listed packages are not available for download. Source returns "Access Denied" case HttpStatusCode.Forbidden: if (retries < MaxRetries) { throw ex; } // Since, retries >= MaxRetries // Set retries to and 0 and don't rethrow // This accomplishes max retries and skipping to next package retries = 0; Log.SkippedForbiddenPackage(package.ToString()); break; // '404 Not Found' from Source. Package is available on the feed but not available for download already case HttpStatusCode.NotFound: // Any other code or if code is null. Throw default: throw ex; } }
/// <summary> /// Installs the package locally and pushes the package onto the destination server /// Uses OptimizedPackage much like the 'nuget push' command to prevent issues caused by /// holding onto package as a stream in memory /// </summary> private void MirrorPackage(DataServicePackageWithCreated package, PackageServer destinationServer, IPackageManager tempPackageManager, LocalPackageRepository tempLocalRepo, string apiKey, int timeOut) { // Download the package locally into a temp folder. This prevents storing the package in memory // which becomes an issue with large packages. Push command uses OptimizedZipPackage and we will too string localPackagePath = String.Empty; OptimizedZipPackage localPackage = null; try { Log.AddingPackageLocally(package.ToString(), package.Created.Value.DateTime.ToString()); tempPackageManager.InstallPackage(package.Id, package.SemanticVersion, ignoreDependencies: true, allowPrereleaseVersions: true); Log.AddedPackageLocally(package.ToString()); } catch (Exception ex) { throw new SourceException(ex); } try { // Push the local package onto destination Repository var localInstallPath = tempLocalRepo.PathResolver.GetInstallPath(package); localPackagePath = Path.Combine(localInstallPath, tempLocalRepo.PathResolver.GetPackageFileName(package)); localPackage = new OptimizedZipPackage(localPackagePath); Log.PushingPackage(localPackage.ToString()); destinationServer.PushPackage(apiKey, localPackage, new FileInfo(localPackagePath).Length, timeOut, disableBuffering: false); } catch (Exception ex) { throw new DestinationException(ex); } }
private async Task ThrowDestinationExceptionIfNeeded(DestinationException ex, DataServicePackageWithCreated package, JObject mirrorJson, SqlConnectionStringBuilder cstr, CloudStorageAccount account) { var inner = ex.InnerException; HttpStatusCode?code = (inner != null && inner is InvalidOperationException) ? GetHttpStatusCodeFrom(inner.InnerException) : GetHttpStatusCodeFrom(inner); if (code == null || code != HttpStatusCode.Conflict) { throw ex; } switch (code) { // '409 Conflict' from Destination- Package already exists in destination. Don't rethrow case HttpStatusCode.Conflict: var sourceJObject = GetJObject(mirrorJson, package.Id, package.SemanticVersion); if (sourceJObject == null) { throw new InvalidOperationException("Package" + package.Id + "//" + package.SemanticVersion.ToString() + "is already mirrored, but, not present in mirror.json. WRONG!"); } var oldSourceCreated = sourceJObject[SourceCreatedKey].Value <DateTime>(); var newSourceCreated = new DateTime(package.Created.Value.DateTime.Ticks, DateTimeKind.Utc); if (!newSourceCreated.Equals(oldSourceCreated)) { // This package while already mirrored to the destination, has been deleted from the source and created again to the source // Hence, the different SourceCreated Date // Need to delete the package Log.DeletingOldRevision(package.ToString(), oldSourceCreated.ToString(DateTimeFormatSpecifier), newSourceCreated.ToString(DateTimeFormatSpecifier)); await NuGetV2RepositoryMirrorPackageDeletor.DeletePackage(cstr, account, sourceJObject, package.Id, package.SemanticVersion.ToString()); throw ex; } Log.PackageAlreadyExists(package.ToString()); break; // Any other code or if code is null. Throw default: throw ex; } }
/// <summary> /// Queries for packages created after 'lastCreated'. At most, 40 packages may be returned /// Mirror that batch of packages to the destination server /// </summary> /// <returns>Returns lastMirroredPackage or null</returns> private DataServicePackageWithCreated QueryAndMirrorBatch(DataServices.DataServiceContext serviceContext, PackageServer destinationServer, DateTime lastCreated, string apiKey, int timeOut, JObject mirrorJson, ref int retries, ref int count, ref int skipIndex, SqlConnectionStringBuilder cstr, CloudStorageAccount account) { var newPackages = GetNewPackagesToMirror(serviceContext, lastCreated); // Push packages var tempFolderPath = GetTempFolderPath(serviceContext.BaseUri.DnsSafeHost); Log.TempFolderPath(tempFolderPath); var tempLocalRepo = new LocalPackageRepository(tempFolderPath); var tempPackageManager = new PackageManager(new DataServicePackageRepository(serviceContext.BaseUri), tempFolderPath); DataServicePackageWithCreated currentPackage = null; DataServicePackageWithCreated lastMirroredPackage = null; try { do { // The following code deletes the temp folder if one exists and creates a new one GetTempFolderPath(serviceContext.BaseUri.DnsSafeHost); var newPackagesList = newPackages.Skip(skipIndex).ToList(); Log.PackagesCopyCount(newPackagesList.Count); if (newPackagesList.Count == 0) { break; } foreach (DataServicePackageWithCreated package in newPackagesList) { try { currentPackage = package; MirrorPackage(package, destinationServer, tempPackageManager, tempLocalRepo, apiKey, timeOut); var jObject = AddNewPackage(mirrorJson, package); if (!package.IsListed) { // The new package being pushed is not listed. Mark it as unlisted NuGetV2RepositoryMirrorPackageDeletor.SetListed(cstr, jObject, package.Id, package.SemanticVersion.ToString(), false).Wait(); } Log.PushedToDestination(++count); } catch (SourceException ex) { ThrowSourceExceptionIfNeeded(ex, ref retries, package); } catch (DestinationException ex) { ThrowDestinationExceptionIfNeeded(ex, package, mirrorJson, cstr, account).Wait(); } lastMirroredPackage = package; retries = 0; ++skipIndex; } } while (true); } catch (Exception ex) { retries++; Log.ServerUnreachable(retries, ex.Message); if (currentPackage != null) { Log.MirrorFailed(currentPackage.ToString()); } } // Delete the packages stored locally Directory.Delete(tempFolderPath, recursive: true); Log.DeletedTempFolder(tempFolderPath); return(lastMirroredPackage); }