private static JObject AddNewPackage(JObject mirrorJson, DataServicePackageWithCreated package)
{
// Should Check for prior existence of the package Id, version and delete the old one if SourceLastCreated is not the same
// This logic is added in the 409 Conflict path
var array = mirrorJson[PackageIndexKey] as JArray;
if (array == null)
{
throw new InvalidOperationException("There is no array of 'packageIndex' in mirror json");
}
var lastItem = array.Count > 0 ? array[array.Count - 1] as JObject : null;
if (lastItem != null)
{
if (CompareToSourceCreated(lastItem, package.Created.Value.DateTime.Ticks) > 0)
{
throw new InvalidOperationException("Last package added has a greater SourceCreated Date than the new package being added");
}
}
var jObject = GetNewPackage(package);
array.Add(jObject);
return(jObject);
}
/// <summary>
/// Installs the package locally and pushes the package onto the destination server
/// Uses OptimizedPackage much like the 'nuget push' command to prevent issues caused by
/// holding onto package as a stream in memory
/// </summary>
private void MirrorPackage(DataServicePackageWithCreated package, PackageServer destinationServer, IPackageManager tempPackageManager, LocalPackageRepository tempLocalRepo,
string apiKey, int timeOut)
{
// Download the package locally into a temp folder. This prevents storing the package in memory
// which becomes an issue with large packages. Push command uses OptimizedZipPackage and we will too
string localPackagePath = String.Empty;
OptimizedZipPackage localPackage = null;
try
{
Log.AddingPackageLocally(package.ToString(), package.Created.Value.DateTime.ToString());
tempPackageManager.InstallPackage(package.Id, package.SemanticVersion, ignoreDependencies: true, allowPrereleaseVersions: true);
Log.AddedPackageLocally(package.ToString());
}
catch (Exception ex)
{
throw new SourceException(ex);
}
try
{
// Push the local package onto destination Repository
var localInstallPath = tempLocalRepo.PathResolver.GetInstallPath(package);
localPackagePath = Path.Combine(localInstallPath, tempLocalRepo.PathResolver.GetPackageFileName(package));
localPackage = new OptimizedZipPackage(localPackagePath);
Log.PushingPackage(localPackage.ToString());
destinationServer.PushPackage(apiKey, localPackage, new FileInfo(localPackagePath).Length, timeOut, disableBuffering: false);
}
catch (Exception ex)
{
throw new DestinationException(ex);
}
}
private static JObject GetNewPackage(DataServicePackageWithCreated package)
{
JObject jObject = new JObject();
var utcdt = new DateTime(package.Created.Value.DateTime.Ticks, DateTimeKind.Utc);
jObject.Add(SourceCreatedKey, utcdt.ToString(DateTimeFormatSpecifier));
jObject.Add(IdKey, package.Id);
jObject.Add(VersionKey, package.SemanticVersion.ToString());
jObject.Add(ListedKey, package.IsListed);
// No need to add Deleted at this point
return(jObject);
}
/// <summary>
/// Queries for packages created after 'lastCreated'. At most, 40 packages may be returned
/// Mirror that batch of packages to the destination server
/// </summary>
/// <returns>Returns lastMirroredPackage or null</returns>
private DataServicePackageWithCreated QueryAndMirrorBatch(DataServices.DataServiceContext serviceContext, PackageServer destinationServer, DateTime lastCreated,
string apiKey, int timeOut, JObject mirrorJson, ref int retries, ref int count, ref int skipIndex, SqlConnectionStringBuilder cstr, CloudStorageAccount account)
{
var newPackages = GetNewPackagesToMirror(serviceContext, lastCreated);
// Push packages
var tempFolderPath = GetTempFolderPath(serviceContext.BaseUri.DnsSafeHost);
Log.TempFolderPath(tempFolderPath);
var tempLocalRepo = new LocalPackageRepository(tempFolderPath);
var tempPackageManager = new PackageManager(new DataServicePackageRepository(serviceContext.BaseUri), tempFolderPath);
DataServicePackageWithCreated currentPackage = null;
DataServicePackageWithCreated lastMirroredPackage = null;
try
{
do
{
// The following code deletes the temp folder if one exists and creates a new one
GetTempFolderPath(serviceContext.BaseUri.DnsSafeHost);
var newPackagesList = newPackages.Skip(skipIndex).ToList();
Log.PackagesCopyCount(newPackagesList.Count);
if (newPackagesList.Count == 0)
{
break;
}
foreach (DataServicePackageWithCreated package in newPackagesList)
{
try
{
currentPackage = package;
MirrorPackage(package, destinationServer, tempPackageManager, tempLocalRepo, apiKey, timeOut);
var jObject = AddNewPackage(mirrorJson, package);
if (!package.IsListed)
{
// The new package being pushed is not listed. Mark it as unlisted
NuGetV2RepositoryMirrorPackageDeletor.SetListed(cstr, jObject, package.Id, package.SemanticVersion.ToString(), false).Wait();
}
Log.PushedToDestination(++count);
}
catch (SourceException ex)
{
ThrowSourceExceptionIfNeeded(ex, ref retries, package);
}
catch (DestinationException ex)
{
ThrowDestinationExceptionIfNeeded(ex, package, mirrorJson, cstr, account).Wait();
}
lastMirroredPackage = package;
retries = 0;
++skipIndex;
}
} while (true);
}
catch (Exception ex)
{
retries++;
Log.ServerUnreachable(retries, ex.Message);
if (currentPackage != null)
{
Log.MirrorFailed(currentPackage.ToString());
}
}
// Delete the packages stored locally
Directory.Delete(tempFolderPath, recursive: true);
Log.DeletedTempFolder(tempFolderPath);
return(lastMirroredPackage);
}
private async Task ThrowDestinationExceptionIfNeeded(DestinationException ex, DataServicePackageWithCreated package, JObject mirrorJson, SqlConnectionStringBuilder cstr, CloudStorageAccount account)
{
var inner = ex.InnerException;
HttpStatusCode?code = (inner != null && inner is InvalidOperationException) ? GetHttpStatusCodeFrom(inner.InnerException) : GetHttpStatusCodeFrom(inner);
if (code == null || code != HttpStatusCode.Conflict)
{
throw ex;
}
switch (code)
{
// '409 Conflict' from Destination- Package already exists in destination. Don't rethrow
case HttpStatusCode.Conflict:
var sourceJObject = GetJObject(mirrorJson, package.Id, package.SemanticVersion);
if (sourceJObject == null)
{
throw new InvalidOperationException("Package" + package.Id + "//" + package.SemanticVersion.ToString() + "is already mirrored, but, not present in mirror.json. WRONG!");
}
var oldSourceCreated = sourceJObject[SourceCreatedKey].Value <DateTime>();
var newSourceCreated = new DateTime(package.Created.Value.DateTime.Ticks, DateTimeKind.Utc);
if (!newSourceCreated.Equals(oldSourceCreated))
{
// This package while already mirrored to the destination, has been deleted from the source and created again to the source
// Hence, the different SourceCreated Date
// Need to delete the package
Log.DeletingOldRevision(package.ToString(), oldSourceCreated.ToString(DateTimeFormatSpecifier), newSourceCreated.ToString(DateTimeFormatSpecifier));
await NuGetV2RepositoryMirrorPackageDeletor.DeletePackage(cstr, account, sourceJObject, package.Id, package.SemanticVersion.ToString());
throw ex;
}
Log.PackageAlreadyExists(package.ToString());
break;
// Any other code or if code is null. Throw
default:
throw ex;
}
}
private void ThrowSourceExceptionIfNeeded(SourceException ex, ref int retries, DataServicePackageWithCreated package)
{
// TO BE DELETED
if (ex.InnerException is PathTooLongException)
{
Log.LogMessage(String.Format("PathTooLongException on package {0}", package.ToString()));
return;
}
HttpStatusCode?code = GetHttpStatusCodeFrom(ex.InnerException);
switch (code)
{
// '403 Forbidden' from Source. For reasons unknown, certain listed packages are not available for download. Source returns "Access Denied"
case HttpStatusCode.Forbidden:
if (retries < MaxRetries)
{
throw ex;
}
// Since, retries >= MaxRetries
// Set retries to and 0 and don't rethrow
// This accomplishes max retries and skipping to next package
retries = 0;
Log.SkippedForbiddenPackage(package.ToString());
break;
// '404 Not Found' from Source. Package is available on the feed but not available for download already
case HttpStatusCode.NotFound:
// Any other code or if code is null. Throw
default:
throw ex;
}
}
public static bool Equals(MinPackage mp, DataServicePackageWithCreated dp)
{
return(mp.SourceCreated.Equals(dp.Created.Value.DateTime) && mp.SemanticVersion.Equals(dp.SemanticVersion) && String.Equals(mp.Id, dp.Id, StringComparison.OrdinalIgnoreCase));
}
