public MainModel()
{
AppSettings = SettingsStorage.LoadSettings(Environment.AppSettingsFilePath);
if (AppSettings.Advanced.LoggingEnabled)
{
EnableLogging();
}
Mirrors = MirrorStorage.LoadMirrors(Environment.MirrorsFilePath);
ValidateAndCorrectSelectedMirrors();
CreateNewHttpClient();
OpenDatabase(AppSettings.DatabaseFileName);
}
public MainModel()
{
AppSettings = SettingsStorage.LoadSettings(Environment.AppSettingsFilePath);
if (AppSettings.Advanced.LoggingEnabled)
{
EnableLogging();
}
ValidateAndCorrectDirectoryPaths();
Mirrors = MirrorStorage.LoadMirrors(Path.Combine(Environment.MirrorsDirectoryPath, MIRRORS_FILE_NAME));
ValidateAndCorrectSelectedMirrors();
Localization = new LocalizationStorage(Environment.LanguagesDirectoryPath, AppSettings.General.Language);
CreateNewHttpClient();
OpenDatabase(AppSettings.DatabaseFileName);
LastApplicationUpdateCheckDateTime = AppSettings.LastUpdate.LastCheckedAt;
LastApplicationUpdateCheckResult = null;
updater = new Updater();
updater.UpdateCheck += ApplicationUpdateCheck;
ConfigureUpdater();
Downloader = new Downloader();
ConfigureDownloader();
}
protected async internal override Task Execute()
{
// Validate mandatory parameters
if (String.IsNullOrEmpty(SourceV2Feed))
{
throw new ArgumentException("SourceV2Feed cannot be null or empty");
}
// If the SourceV2Feed is not a URI, the following line will throw URIFormatException
var sourceV2FeedUri = new Uri(SourceV2Feed);
if (String.IsNullOrEmpty(DestinationUri))
{
throw new ArgumentException("DestinationUri cannot be null or empty");
}
if (String.IsNullOrEmpty(ApiKey))
{
throw new ArgumentException("ApiKey cannot be null or empty");
}
// Packages are in Legacy account. PackageDatabase is the InitialCatalog in the legacy account
var account = Config.Storage.Legacy;
var cstr = Config.Sql.Legacy;
if (cstr == null)
{
throw new ArgumentNullException("Legacy sql cannot be null");
}
// Arrange or set defaults for parameters that are not provided
UserAgent = String.Format("{0}v2FeedMirrorer", sourceV2FeedUri.DnsSafeHost);
var timeOutPerPush = TimeSpan.FromMinutes(PushTimeOutInMinutes);
MirrorStorage = MirrorStorage ?? Config.Storage.Legacy;
if (MirrorStorage == null)
{
throw new ArgumentNullException("MirrorStorage", "Mirror storage is not provided or present in the config");
}
MirrorBlobContainerName = String.IsNullOrEmpty(MirrorBlobContainerName) ? DefaultMirrorBlobContainerName : MirrorBlobContainerName;
MirrorBlobContainer = MirrorStorage.CreateCloudBlobClient().GetContainerReference(MirrorBlobContainerName);
MirrorBlobName = String.IsNullOrEmpty(MirrorBlobName) ? DefaultMirrorBlobName : MirrorBlobName;
var sourceUri = new Uri(sourceV2FeedUri, "/api/v2/");
var serviceContext = new DataServices.DataServiceContext(sourceUri)
{
MergeOption = DataServices.MergeOption.OverwriteChanges,
IgnoreMissingProperties = true,
};
var mirrorJson = await GetJObject(MirrorBlobContainer, MirrorBlobName);
if (!IsMirrorJsonValid(mirrorJson))
{
throw new InvalidOperationException("mirrorJson is not valid. Either packageIndex array is not present. Or, the elements are not sorted by SourceCreatedDate");
}
Exception caughtException = null;
if (ExecuteDeletes)
{
try
{
await NuGetV2RepositoryMirrorPackageDeletor.DeleteAndSetListedPackages(sourceUri, mirrorJson, account, cstr);
}
catch (Exception ex)
{
caughtException = ex;
}
await SetJObject(MirrorBlobContainer, MirrorBlobName, mirrorJson);
if (caughtException != null)
{
throw caughtException;
}
return;
}
var oldLastCreated = mirrorJson.Value <DateTime>(LastCreatedKey);
var lastCreated = oldLastCreated;
Log.PreparingToMirror(MirrorBlobName, lastCreated.ToString(DateTimeFormatSpecifier), lastCreated.Kind.ToString(), sourceUri.AbsoluteUri, DestinationUri);
int retries = 0;
int count = 0;
int skipIndex = 0;
//
// POSSIBLE ACTIONS when an error is encountered
// A) Always Skip to next package
// B) Retry 'MaxRetries' times and skip to next package
// C) Retry 'MaxRetries' times and fail
//
// KNOWN ERRORS
// 1) '409 Conflict' from Destination- Package already exists in destination
// i) Action: (A). Always Skip to next package
// ii) Before skipping, Update lastMirroredPackage.Created locally, i.e inside the while loop
// 2) '403 Forbidden' from Source. For reasons unknown, certain listed packages are not available for download. Source returns "Access Denied"
// i) Action: (B). Retry 'MaxRetries' times and Skip to next package
// ii) Log every retry. Before skipping, Update lastMirroredPackage.Created locally, i.e inside the while loop
// 3) '404 Not Found' from Source. Package is available on the feed but not available for download already
// i) Action: (C). Retry 'MaxRetries' times and Fail
// ii) Log every retry. Update lastMirroredPackage.Created in blob storage
// 4) Unknown Error
// i) Action: (C). Retry 'MaxRetries' times and Fail
// ii) Log every retry. Update lastMirroredPackage.Created in blob storage
//
// Test Cases
// 1) Add a new package to the source. COVERED
// Result: The new package should be present on the destination
// 2) Add a package as unlisted to the source. COVERED
// Result: The new package should be present on the destination as unlisted
// 3) Delete a package version from the source. COVERED
// Result: The package should be deleted from the destination
// 4) Delete a package version from the source which is the last version with the package Id. NOT COVERED
// Result: The package should be deleted from the destination. And, PackageRegistration should be deleted too as appropriate
// 5) Delete a package version from the source. And, add a new package with same Id and version as the deleted one. COVERED
// Result: Old package with Id and version must be deleted from the destination. And, the new package with the same Id and Version must be added
// 6) Mark a listed package as unlisted. COVERED
// Result: The package should be unlisted in the destination too
// 7) Mark an unlisted package as listed. COVERED
// Result: The package should be listed in the destination too
//
try
{
do
{
if ((Invocation.NextVisibleAt - DateTimeOffset.UtcNow) < TimeSpan.FromMinutes(120))
{
// Based on default configuration on the repository, there are at most 40 packages returned by a single query (which will get downloaded and pushed)
// With a conservative estimate of 3 minutes per package, expecting a minimum of
// 120 minutes to run 1 iteration of this do-while loop
await Extend(TimeSpan.FromMinutes(120));
}
// In each query, at most, 40 packages may be returned. So, continue performing the queries
// in this do-while, so long as there are results returned
// Query for packages created since oldLastCreated
PackageServer destinationServer = new PackageServer(DestinationUri, UserAgent);
var lastMirroredPackage = QueryAndMirrorBatch(serviceContext, destinationServer, oldLastCreated, ApiKey, timeOutPerPush.Milliseconds, mirrorJson, ref retries, ref count, ref skipIndex, cstr, account);
if (lastMirroredPackage != null)
{
if (!lastMirroredPackage.Created.HasValue)
{
throw new InvalidOperationException("Last mirrored package : " + lastMirroredPackage.ToString() + "has a null Created Time.. WRONG!!!");
}
// Note that the Created DateTime is always stored in UTC, but the DateTimeKind of the value obtained is Unspecified
// So, store it as UTC
lastCreated = new DateTime(lastMirroredPackage.Created.Value.DateTime.Ticks, DateTimeKind.Utc);
Log.EndOfIteration(lastCreated.ToString(DateTimeFormatSpecifier));
}
else if (retries == 0 || retries > MaxRetries)
{
break;
}
} while (true);
}
catch (Exception ex)
{
// Catch the exception here so that new lastCreated is always stored
// We can throw this exception at the end
caughtException = ex;
}
if (!oldLastCreated.Equals(lastCreated))
{
mirrorJson[LastCreatedKey] = lastCreated.ToString(DateTimeFormatSpecifier);
await SetJObject(MirrorBlobContainer, MirrorBlobName, mirrorJson);
Log.NewLastCreatedAtEndOfInvocation(MirrorBlobName, lastCreated.ToString(DateTimeFormatSpecifier), lastCreated.Kind.ToString());
}
if (caughtException != null)
{
throw caughtException;
}
}
