public MainModel() { AppSettings = SettingsStorage.LoadSettings(Environment.AppSettingsFilePath); if (AppSettings.Advanced.LoggingEnabled) { EnableLogging(); } Mirrors = MirrorStorage.LoadMirrors(Environment.MirrorsFilePath); ValidateAndCorrectSelectedMirrors(); CreateNewHttpClient(); OpenDatabase(AppSettings.DatabaseFileName); }
public MainModel() { AppSettings = SettingsStorage.LoadSettings(Environment.AppSettingsFilePath); if (AppSettings.Advanced.LoggingEnabled) { EnableLogging(); } ValidateAndCorrectDirectoryPaths(); Mirrors = MirrorStorage.LoadMirrors(Path.Combine(Environment.MirrorsDirectoryPath, MIRRORS_FILE_NAME)); ValidateAndCorrectSelectedMirrors(); Localization = new LocalizationStorage(Environment.LanguagesDirectoryPath, AppSettings.General.Language); CreateNewHttpClient(); OpenDatabase(AppSettings.DatabaseFileName); LastApplicationUpdateCheckDateTime = AppSettings.LastUpdate.LastCheckedAt; LastApplicationUpdateCheckResult = null; updater = new Updater(); updater.UpdateCheck += ApplicationUpdateCheck; ConfigureUpdater(); Downloader = new Downloader(); ConfigureDownloader(); }
protected async internal override Task Execute() { // Validate mandatory parameters if (String.IsNullOrEmpty(SourceV2Feed)) { throw new ArgumentException("SourceV2Feed cannot be null or empty"); } // If the SourceV2Feed is not a URI, the following line will throw URIFormatException var sourceV2FeedUri = new Uri(SourceV2Feed); if (String.IsNullOrEmpty(DestinationUri)) { throw new ArgumentException("DestinationUri cannot be null or empty"); } if (String.IsNullOrEmpty(ApiKey)) { throw new ArgumentException("ApiKey cannot be null or empty"); } // Packages are in Legacy account. PackageDatabase is the InitialCatalog in the legacy account var account = Config.Storage.Legacy; var cstr = Config.Sql.Legacy; if (cstr == null) { throw new ArgumentNullException("Legacy sql cannot be null"); } // Arrange or set defaults for parameters that are not provided UserAgent = String.Format("{0}v2FeedMirrorer", sourceV2FeedUri.DnsSafeHost); var timeOutPerPush = TimeSpan.FromMinutes(PushTimeOutInMinutes); MirrorStorage = MirrorStorage ?? Config.Storage.Legacy; if (MirrorStorage == null) { throw new ArgumentNullException("MirrorStorage", "Mirror storage is not provided or present in the config"); } MirrorBlobContainerName = String.IsNullOrEmpty(MirrorBlobContainerName) ? DefaultMirrorBlobContainerName : MirrorBlobContainerName; MirrorBlobContainer = MirrorStorage.CreateCloudBlobClient().GetContainerReference(MirrorBlobContainerName); MirrorBlobName = String.IsNullOrEmpty(MirrorBlobName) ? DefaultMirrorBlobName : MirrorBlobName; var sourceUri = new Uri(sourceV2FeedUri, "/api/v2/"); var serviceContext = new DataServices.DataServiceContext(sourceUri) { MergeOption = DataServices.MergeOption.OverwriteChanges, IgnoreMissingProperties = true, }; var mirrorJson = await GetJObject(MirrorBlobContainer, MirrorBlobName); if (!IsMirrorJsonValid(mirrorJson)) { throw new InvalidOperationException("mirrorJson is not valid. Either packageIndex array is not present. Or, the elements are not sorted by SourceCreatedDate"); } Exception caughtException = null; if (ExecuteDeletes) { try { await NuGetV2RepositoryMirrorPackageDeletor.DeleteAndSetListedPackages(sourceUri, mirrorJson, account, cstr); } catch (Exception ex) { caughtException = ex; } await SetJObject(MirrorBlobContainer, MirrorBlobName, mirrorJson); if (caughtException != null) { throw caughtException; } return; } var oldLastCreated = mirrorJson.Value <DateTime>(LastCreatedKey); var lastCreated = oldLastCreated; Log.PreparingToMirror(MirrorBlobName, lastCreated.ToString(DateTimeFormatSpecifier), lastCreated.Kind.ToString(), sourceUri.AbsoluteUri, DestinationUri); int retries = 0; int count = 0; int skipIndex = 0; // // POSSIBLE ACTIONS when an error is encountered // A) Always Skip to next package // B) Retry 'MaxRetries' times and skip to next package // C) Retry 'MaxRetries' times and fail // // KNOWN ERRORS // 1) '409 Conflict' from Destination- Package already exists in destination // i) Action: (A). Always Skip to next package // ii) Before skipping, Update lastMirroredPackage.Created locally, i.e inside the while loop // 2) '403 Forbidden' from Source. For reasons unknown, certain listed packages are not available for download. Source returns "Access Denied" // i) Action: (B). Retry 'MaxRetries' times and Skip to next package // ii) Log every retry. Before skipping, Update lastMirroredPackage.Created locally, i.e inside the while loop // 3) '404 Not Found' from Source. Package is available on the feed but not available for download already // i) Action: (C). Retry 'MaxRetries' times and Fail // ii) Log every retry. Update lastMirroredPackage.Created in blob storage // 4) Unknown Error // i) Action: (C). Retry 'MaxRetries' times and Fail // ii) Log every retry. Update lastMirroredPackage.Created in blob storage // // Test Cases // 1) Add a new package to the source. COVERED // Result: The new package should be present on the destination // 2) Add a package as unlisted to the source. COVERED // Result: The new package should be present on the destination as unlisted // 3) Delete a package version from the source. COVERED // Result: The package should be deleted from the destination // 4) Delete a package version from the source which is the last version with the package Id. NOT COVERED // Result: The package should be deleted from the destination. And, PackageRegistration should be deleted too as appropriate // 5) Delete a package version from the source. And, add a new package with same Id and version as the deleted one. COVERED // Result: Old package with Id and version must be deleted from the destination. And, the new package with the same Id and Version must be added // 6) Mark a listed package as unlisted. COVERED // Result: The package should be unlisted in the destination too // 7) Mark an unlisted package as listed. COVERED // Result: The package should be listed in the destination too // try { do { if ((Invocation.NextVisibleAt - DateTimeOffset.UtcNow) < TimeSpan.FromMinutes(120)) { // Based on default configuration on the repository, there are at most 40 packages returned by a single query (which will get downloaded and pushed) // With a conservative estimate of 3 minutes per package, expecting a minimum of // 120 minutes to run 1 iteration of this do-while loop await Extend(TimeSpan.FromMinutes(120)); } // In each query, at most, 40 packages may be returned. So, continue performing the queries // in this do-while, so long as there are results returned // Query for packages created since oldLastCreated PackageServer destinationServer = new PackageServer(DestinationUri, UserAgent); var lastMirroredPackage = QueryAndMirrorBatch(serviceContext, destinationServer, oldLastCreated, ApiKey, timeOutPerPush.Milliseconds, mirrorJson, ref retries, ref count, ref skipIndex, cstr, account); if (lastMirroredPackage != null) { if (!lastMirroredPackage.Created.HasValue) { throw new InvalidOperationException("Last mirrored package : " + lastMirroredPackage.ToString() + "has a null Created Time.. WRONG!!!"); } // Note that the Created DateTime is always stored in UTC, but the DateTimeKind of the value obtained is Unspecified // So, store it as UTC lastCreated = new DateTime(lastMirroredPackage.Created.Value.DateTime.Ticks, DateTimeKind.Utc); Log.EndOfIteration(lastCreated.ToString(DateTimeFormatSpecifier)); } else if (retries == 0 || retries > MaxRetries) { break; } } while (true); } catch (Exception ex) { // Catch the exception here so that new lastCreated is always stored // We can throw this exception at the end caughtException = ex; } if (!oldLastCreated.Equals(lastCreated)) { mirrorJson[LastCreatedKey] = lastCreated.ToString(DateTimeFormatSpecifier); await SetJObject(MirrorBlobContainer, MirrorBlobName, mirrorJson); Log.NewLastCreatedAtEndOfInvocation(MirrorBlobName, lastCreated.ToString(DateTimeFormatSpecifier), lastCreated.Kind.ToString()); } if (caughtException != null) { throw caughtException; } }