private void ProcessPackageEdits(int packageKey)
{
// Create a fresh entities context so that we work in isolation
var entitiesContext = new EntitiesContext(ConnectionString.ConnectionString, readOnly: false);
// Get the list of edits for this package
// Do edit with a 'most recent edit to this package wins - other edits are deleted' strategy.
var editsForThisPackage = entitiesContext.Set <PackageEdit>()
.Where(pe => pe.PackageKey == packageKey && pe.TriedCount < 3)
.Include(pe => pe.Package)
.Include(pe => pe.Package.PackageRegistration)
.Include(pe => pe.User)
.OrderByDescending(pe => pe.Timestamp)
.ToList();
// List of Work to do:
// 1) Backup old blob, if the original has not been backed up yet
// 2) Downloads blob, create new NUPKG locally
// 3) Upload blob
// 4) Update the database
PackageEdit edit = editsForThisPackage.First();
var blobClient = StorageAccount.CreateCloudBlobClient();
var packagesContainer = Util.GetPackagesBlobContainer(blobClient);
var latestPackageFileName = Util.GetPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version);
var originalPackageFileName = Util.GetBackupOfOriginalPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version);
var originalPackageBackupBlob = packagesContainer.GetBlockBlobReference(originalPackageFileName);
var latestPackageBlob = packagesContainer.GetBlockBlobReference(latestPackageFileName);
var edits = new List <Action <ManifestMetadata> >
{
(m) => { m.Authors = edit.Authors; },
(m) => { m.Copyright = edit.Copyright; },
(m) => { m.Description = edit.Description; },
(m) => { m.IconUrl = edit.IconUrl; },
(m) => { m.LicenseUrl = edit.LicenseUrl; },
(m) => { m.ProjectUrl = edit.ProjectUrl; },
(m) => { m.ReleaseNotes = edit.ReleaseNotes; },
(m) => { m.RequireLicenseAcceptance = edit.RequiresLicenseAcceptance; },
(m) => { m.Summary = edit.Summary; },
(m) => { m.Title = edit.Title; },
(m) => { m.Tags = edit.Tags; },
};
Log.Info(
"Processing Edit Key={0}, PackageId={1}, Version={2}, User={3}",
edit.Key,
edit.Package.PackageRegistration.Id,
edit.Package.Version,
edit.User.Username);
if (!WhatIf)
{
edit.TriedCount += 1;
int nr = entitiesContext.SaveChanges();
if (nr != 1)
{
throw new Exception(
String.Format("Something went terribly wrong, only one entity should be updated but actually {0} entities were updated", nr));
}
}
try
{
ArchiveOriginalPackageBlob(originalPackageBackupBlob, latestPackageBlob);
using (var readWriteStream = new MemoryStream())
{
// Download to memory
CloudBlockBlob downloadSourceBlob = WhatIf ? latestPackageBlob : originalPackageBackupBlob;
Log.Info("Downloading original package blob to memory {0}", downloadSourceBlob.Name);
downloadSourceBlob.DownloadToStream(readWriteStream);
// Rewrite in memory
Log.Info("Rewriting nupkg package in memory", downloadSourceBlob.Name);
NupkgRewriter.RewriteNupkgManifest(readWriteStream, edits);
// Get updated hash code, and file size
Log.Info("Computing updated hash code of memory stream");
var newPackageFileSize = readWriteStream.Length;
var hashAlgorithm = HashAlgorithm.Create("SHA512");
byte[] hashBytes = hashAlgorithm.ComputeHash(readWriteStream.GetBuffer());
var newHash = Convert.ToBase64String(hashBytes);
if (!WhatIf)
{
// Snapshot the blob
var blobSnapshot = latestPackageBlob.CreateSnapshot();
// Build up the changes in the entities context
edit.Apply(hashAlgorithm: "SHA512", hash: newHash, packageFileSize: newPackageFileSize);
foreach (var eachEdit in editsForThisPackage)
{
entitiesContext.DeleteOnCommit(eachEdit);
}
// Upload the blob before doing SaveChanges(). If blob update fails, we won't do SaveChanges() and the edit can be retried.
// If SaveChanges() fails we can undo the blob upload.
try
{
Log.Info("Uploading blob from memory {0}", latestPackageBlob.Name);
readWriteStream.Position = 0;
latestPackageBlob.UploadFromStream(readWriteStream);
}
catch (Exception e)
{
Log.Error("(error) - package edit blob update failed.");
Log.ErrorException("(exception)", e);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
throw; // To handler block that will record error in DB
}
try
{
// SaveChanges tries to commit changes to DB
entitiesContext.SaveChanges();
}
catch (Exception e)
{
// Commit changes to DB probably failed.
// Since our blob update wasn't part of the transaction (and doesn't AFAIK have a 'commit()' operator we can utilize for the type of blobs we are using)
// try, (single attempt) to roll back the blob update by restoring the previous snapshot.
Log.Error("(error) - package edit DB update failed. Trying to roll back the blob to its previous snapshot.");
Log.ErrorException("(exception)", e);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
try
{
latestPackageBlob.StartCopyFromBlob(blobSnapshot);
}
catch (Exception e2)
{
// If blob rollback fails it is not be the end of the world
// - the package metadata mismatches the edit now,
// but there should still an edit in the queue, waiting to be rerun and put everything back in synch.
Log.Error("(error) - rolling back the package blob to its previous snapshot failed.");
Log.ErrorException("(exception)", e2);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
}
throw; // To handler block that will record error in DB
}
}
}
}
catch (Exception e)
{
if (!WhatIf)
{
try
{
Log.Info("Storing the error on package edit with key {0}", edit.Key);
// Try to record the error into the PackageEdit database record
// so that we can actually diagnose failures.
// This must be done on a fresh context to ensure no conflicts.
var errorContext = new EntitiesContext(ConnectionString.ConnectionString, readOnly: false);
var errorEdit = errorContext.Set <PackageEdit>().Where(pe => pe.Key == edit.Key).FirstOrDefault();
if (errorEdit != null)
{
errorEdit.LastError = string.Format("{0} : {1}", e.GetType(), e);
errorContext.SaveChanges();
}
else
{
Log.Info("The package edit with key {0} couldn't be found. It was likely canceled and deleted.", edit.Key);
}
}
catch (Exception errorException)
{
Log.ErrorException("(error) - couldn't save the last error on the edit that was being applied.", errorException);
}
}
}
}
private void ProcessPackageEdits(IEnumerable <PackageEdit> editsForThisPackage, EntitiesContext entitiesContext)
{
// List of Work to do:
// 1) Backup old blob, if the original has not been backed up yet
// 2) Downloads blob, create new NUPKG locally
// 3) Upload blob
// 4) Update the database
PackageEdit edit = editsForThisPackage.OrderByDescending(pe => pe.Timestamp).First();
var blobClient = StorageAccount.CreateCloudBlobClient();
var packagesContainer = Util.GetPackagesBlobContainer(blobClient);
var latestPackageFileName = Util.GetPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version);
var originalPackageFileName = Util.GetBackupOfOriginalPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version);
var originalPackageBackupBlob = packagesContainer.GetBlockBlobReference(originalPackageFileName);
var latestPackageBlob = packagesContainer.GetBlockBlobReference(latestPackageFileName);
var edits = new List <Action <ManifestMetadata> >
{
(m) => { m.Authors = edit.Authors; },
(m) => { m.Copyright = edit.Copyright; },
(m) => { m.Description = edit.Description; },
(m) => { m.IconUrl = edit.IconUrl; },
(m) => { m.LicenseUrl = edit.LicenseUrl; },
(m) => { m.ProjectUrl = edit.ProjectUrl; },
(m) => { m.ReleaseNotes = edit.ReleaseNotes; },
(m) => { m.RequireLicenseAcceptance = edit.RequiresLicenseAcceptance; },
(m) => { m.Summary = edit.Summary; },
(m) => { m.Title = edit.Title; },
(m) => { m.Tags = edit.Tags; },
};
Log.Info(
"Processing Edit Key={0}, PackageId={1}, Version={2}",
edit.Key,
edit.Package.PackageRegistration.Id,
edit.Package.Version);
if (!WhatIf)
{
edit.TriedCount += 1;
int nr = entitiesContext.SaveChanges();
if (nr != 1)
{
throw new ApplicationException(
String.Format("Something went terribly wrong, only one entity should be updated but actually {0} entities were updated", nr));
}
}
ArchiveOriginalPackageBlob(originalPackageBackupBlob, latestPackageBlob);
using (var readWriteStream = new MemoryStream())
{
// Download to memory
CloudBlockBlob downloadSourceBlob = WhatIf ? latestPackageBlob : originalPackageBackupBlob;
Log.Info("Downloading original package blob to memory {0}", downloadSourceBlob.Name);
downloadSourceBlob.DownloadToStream(readWriteStream);
// Rewrite in memory
Log.Info("Rewriting nupkg package in memory", downloadSourceBlob.Name);
NupkgRewriter.RewriteNupkgManifest(readWriteStream, edits);
// Get updated hash code, and file size
Log.Info("Computing updated hash code of memory stream");
var newPackageFileSize = readWriteStream.Length;
var hashAlgorithm = HashAlgorithm.Create("SHA512");
byte[] hashBytes = hashAlgorithm.ComputeHash(readWriteStream.GetBuffer());
var newHash = Convert.ToBase64String(hashBytes);
if (!WhatIf)
{
// Snapshot the blob
var blobSnapshot = latestPackageBlob.CreateSnapshot();
// Start Transaction: Complete the edit in the gallery DB.
// Use explicit SQL transactions instead of EF operation-grouping
// so that we can manually roll the transaction back on a blob related failure.
ObjectContext objectContext = (entitiesContext as IObjectContextAdapter).ObjectContext;
((objectContext.Connection) as EntityConnection).Open(); // must open in order to begin transaction
using (EntityTransaction transaction = ((objectContext.Connection) as EntityConnection).BeginTransaction())
{
edit.Apply(hashAlgorithm: "SHA512", hash: newHash, packageFileSize: newPackageFileSize);
// Add to transaction: delete all the pending edits of this package.
foreach (var eachEdit in editsForThisPackage)
{
entitiesContext.DeleteOnCommit(eachEdit);
}
entitiesContext.SaveChanges(); // (transaction is still not committed, but do some EF legwork up-front of modifying the blob)
try
{
// Reupload blob
Log.Info("Uploading blob from memory {0}", latestPackageBlob.Name);
readWriteStream.Position = 0;
latestPackageBlob.UploadFromStream(readWriteStream);
}
catch (Exception e)
{
// Uploading the updated nupkg failed.
// Rollback the transaction, which restores the Edit to PackageEdits so it can be attempted again.
Log.Error("(error) - package edit blob update failed. Rolling back the DB transaction.");
Log.ErrorException("(exception", e);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
transaction.Rollback();
return;
}
try
{
transaction.Commit();
}
catch (Exception e)
{
// Commit changes to DB failed.
// Since our blob update wasn't part of the transaction (and doesn't AFAIK have a 'commit()' operator we can utilize for the type of blobs we are using)
// try, (single attempt) to roll back the blob update by restoring the previous snapshot.
Log.Error("(error) - package edit DB update failed. Trying to roll back the blob to its previous snapshot.");
Log.ErrorException("(exception", e);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
try
{
latestPackageBlob.StartCopyFromBlob(blobSnapshot);
}
catch (Exception e2)
{
// In this case it may not be the end of the world - the package metadata mismatches the edit now,
// but there's still an edit in the queue, waiting to be rerun and put everything back in synch.
Log.Error("(error) - rolling back the package blob to its previous snapshot failed.");
Log.ErrorException("(exception", e2);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
}
}
}
}
}
}
private void ProcessPackageEdits(int packageKey, IEnumerable<PackageEdit> editsToDelete)
{
// Create a fresh entities context so that we work in isolation
var entitiesContext = new EntitiesContext(ConnectionString.ConnectionString, readOnly: false);
// Get the most recent edit for this package
var edit = entitiesContext.Set<PackageEdit>()
.Where(pe => pe.PackageKey == packageKey && pe.TriedCount < 3)
.Include(pe => pe.Package)
.Include(pe => pe.Package.PackageRegistration)
.Include(pe => pe.User)
.OrderByDescending(pe => pe.Timestamp)
.First();
// List of Work to do:
// 1) Backup old blob, if the original has not been backed up yet
// 2) Downloads blob, create new NUPKG locally
// 3) Upload blob
// 4) Update the database
var blobClient = StorageAccount.CreateCloudBlobClient();
var packagesContainer = Util.GetPackagesBlobContainer(blobClient);
var latestPackageFileName = Util.GetPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version);
var originalPackageFileName = Util.GetBackupOfOriginalPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version);
var originalPackageBackupBlob = packagesContainer.GetBlockBlobReference(originalPackageFileName);
var latestPackageBlob = packagesContainer.GetBlockBlobReference(latestPackageFileName);
var edits = new List<Action<ManifestEdit>>
{
(m) => { m.Authors = edit.Authors; },
(m) => { m.Copyright = edit.Copyright; },
(m) => { m.Description = edit.Description; },
(m) => { m.IconUrl = edit.IconUrl; },
(m) => { m.LicenseUrl = edit.LicenseUrl; },
(m) => { m.ProjectUrl = edit.ProjectUrl; },
(m) => { m.ReleaseNotes = edit.ReleaseNotes; },
(m) => { m.RequireLicenseAcceptance = edit.RequiresLicenseAcceptance; },
(m) => { m.Summary = edit.Summary; },
(m) => { m.Title = edit.Title; },
(m) => { m.Tags = edit.Tags; },
};
Log.Info(
"Processing Edit Key={0}, PackageId={1}, Version={2}, User={3}",
edit.Key,
edit.Package.PackageRegistration.Id,
edit.Package.Version,
edit.User.Username);
if (!WhatIf)
{
edit.TriedCount += 1;
int nr = entitiesContext.SaveChanges();
if (nr != 1)
{
throw new Exception(
$"Something went terribly wrong, only one entity should be updated but actually {nr} entities were updated");
}
}
try
{
ArchiveOriginalPackageBlob(originalPackageBackupBlob, latestPackageBlob);
using (var readWriteStream = new MemoryStream())
{
// Download to memory
CloudBlockBlob downloadSourceBlob = WhatIf ? latestPackageBlob : originalPackageBackupBlob;
Log.Info("Downloading original package blob to memory {0}", downloadSourceBlob.Name);
downloadSourceBlob.DownloadToStream(readWriteStream);
// Rewrite in memory
Log.Info("Rewriting nupkg package in memory", downloadSourceBlob.Name);
NupkgRewriter.RewriteNupkgManifest(readWriteStream, edits);
// Get updated hash code, and file size
Log.Info("Computing updated hash code of memory stream");
var newPackageFileSize = readWriteStream.Length;
var hashAlgorithm = HashAlgorithm.Create("SHA512");
byte[] hashBytes = hashAlgorithm.ComputeHash(readWriteStream.GetBuffer());
var newHash = Convert.ToBase64String(hashBytes);
if (!WhatIf)
{
// Snapshot the blob
var blobSnapshot = latestPackageBlob.CreateSnapshot();
// Build up the changes in the entities context
edit.Apply(hashAlgorithm: "SHA512", hash: newHash, packageFileSize: newPackageFileSize);
foreach (var eachEdit in editsToDelete)
{
entitiesContext.DeleteOnCommit(eachEdit);
}
// Upload the blob before doing SaveChanges(). If blob update fails, we won't do SaveChanges() and the edit can be retried.
// If SaveChanges() fails we can undo the blob upload.
try
{
Log.Info("Uploading blob from memory {0}", latestPackageBlob.Name);
readWriteStream.Position = 0;
latestPackageBlob.UploadFromStream(readWriteStream);
}
catch (Exception e)
{
Log.Error("(error) - package edit blob update failed.");
Log.ErrorException("(exception)", e);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
throw; // To handler block that will record error in DB
}
try
{
// SaveChanges tries to commit changes to DB
entitiesContext.SaveChanges();
}
catch (Exception e)
{
// Commit changes to DB probably failed.
// Since our blob update wasn't part of the transaction (and doesn't AFAIK have a 'commit()' operator we can utilize for the type of blobs we are using)
// try, (single attempt) to roll back the blob update by restoring the previous snapshot.
Log.Error("(error) - package edit DB update failed. Trying to roll back the blob to its previous snapshot.");
Log.ErrorException("(exception)", e);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
try
{
latestPackageBlob.StartCopyFromBlob(blobSnapshot);
}
catch (Exception e2)
{
// If blob rollback fails it is not be the end of the world
// - the package metadata mismatches the edit now,
// but there should still an edit in the queue, waiting to be rerun and put everything back in synch.
Log.Error("(error) - rolling back the package blob to its previous snapshot failed.");
Log.ErrorException("(exception)", e2);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
}
throw; // To handler block that will record error in DB
}
}
}
}
catch (Exception e)
{
if (!WhatIf)
{
try
{
Log.Info("Storing the error on package edit with key {0}", edit.Key);
// Try to record the error into the PackageEdit database record
// so that we can actually diagnose failures.
// This must be done on a fresh context to ensure no conflicts.
var errorContext = new EntitiesContext(ConnectionString.ConnectionString, readOnly: false);
var errorEdit = errorContext.Set<PackageEdit>().Where(pe => pe.Key == edit.Key).FirstOrDefault();
if (errorEdit != null)
{
errorEdit.LastError = $"{e.GetType()} : {e}";
errorContext.SaveChanges();
}
else
{
Log.Info("The package edit with key {0} couldn't be found. It was likely canceled and deleted.", edit.Key);
}
}
catch (Exception errorException)
{
Log.ErrorException("(error) - couldn't save the last error on the edit that was being applied.", errorException);
}
}
}
}
private void ProcessPackageEdits(IEnumerable<PackageEdit> editsForThisPackage, EntitiesContext entitiesContext)
{
// List of Work to do:
// 1) Backup old blob, if the original has not been backed up yet
// 2) Downloads blob, create new NUPKG locally
// 3) Upload blob
// 4) Update the database
PackageEdit edit = editsForThisPackage.OrderByDescending(pe => pe.Timestamp).First();
var blobClient = StorageAccount.CreateCloudBlobClient();
var packagesContainer = Util.GetPackagesBlobContainer(blobClient);
var latestPackageFileName = Util.GetPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version);
var originalPackageFileName = Util.GetBackupOfOriginalPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version);
var originalPackageBackupBlob = packagesContainer.GetBlockBlobReference(originalPackageFileName);
var latestPackageBlob = packagesContainer.GetBlockBlobReference(latestPackageFileName);
var edits = new List<Action<ManifestMetadata>>
{
(m) => { m.Authors = edit.Authors; },
(m) => { m.Copyright = edit.Copyright; },
(m) => { m.Description = edit.Description; },
(m) => { m.IconUrl = edit.IconUrl; },
(m) => { m.LicenseUrl = edit.LicenseUrl; },
(m) => { m.ProjectUrl = edit.ProjectUrl; },
(m) => { m.ReleaseNotes = edit.ReleaseNotes; },
(m) => { m.RequireLicenseAcceptance = edit.RequiresLicenseAcceptance; },
(m) => { m.Summary = edit.Summary; },
(m) => { m.Title = edit.Title; },
(m) => { m.Tags = edit.Tags; },
};
Log.Info(
"Processing Edit Key={0}, PackageId={1}, Version={2}",
edit.Key,
edit.Package.PackageRegistration.Id,
edit.Package.Version);
if (!WhatIf)
{
edit.TriedCount += 1;
int nr = entitiesContext.SaveChanges();
if (nr != 1)
{
throw new ApplicationException(
String.Format("Something went terribly wrong, only one entity should be updated but actually {0} entities were updated", nr));
}
}
ArchiveOriginalPackageBlob(originalPackageBackupBlob, latestPackageBlob);
using (var readWriteStream = new MemoryStream())
{
// Download to memory
CloudBlockBlob downloadSourceBlob = WhatIf ? latestPackageBlob : originalPackageBackupBlob;
Log.Info("Downloading original package blob to memory {0}", downloadSourceBlob.Name);
downloadSourceBlob.DownloadToStream(readWriteStream);
// Rewrite in memory
Log.Info("Rewriting nupkg package in memory", downloadSourceBlob.Name);
NupkgRewriter.RewriteNupkgManifest(readWriteStream, edits);
// Get updated hash code, and file size
Log.Info("Computing updated hash code of memory stream");
var newPackageFileSize = readWriteStream.Length;
var hashAlgorithm = HashAlgorithm.Create("SHA512");
byte[] hashBytes = hashAlgorithm.ComputeHash(readWriteStream.GetBuffer());
var newHash = Convert.ToBase64String(hashBytes);
if (!WhatIf)
{
// Snapshot the blob
var blobSnapshot = latestPackageBlob.CreateSnapshot();
// Start Transaction: Complete the edit in the gallery DB.
// Use explicit SQL transactions instead of EF operation-grouping
// so that we can manually roll the transaction back on a blob related failure.
ObjectContext objectContext = (entitiesContext as IObjectContextAdapter).ObjectContext;
((objectContext.Connection) as EntityConnection).Open(); // must open in order to begin transaction
using (EntityTransaction transaction = ((objectContext.Connection) as EntityConnection).BeginTransaction())
{
edit.Apply(hashAlgorithm: "SHA512", hash: newHash, packageFileSize: newPackageFileSize);
// Add to transaction: delete all the pending edits of this package.
foreach (var eachEdit in editsForThisPackage)
{
entitiesContext.DeleteOnCommit(eachEdit);
}
entitiesContext.SaveChanges(); // (transaction is still not committed, but do some EF legwork up-front of modifying the blob)
try
{
// Reupload blob
Log.Info("Uploading blob from memory {0}", latestPackageBlob.Name);
readWriteStream.Position = 0;
latestPackageBlob.UploadFromStream(readWriteStream);
}
catch (Exception e)
{
// Uploading the updated nupkg failed.
// Rollback the transaction, which restores the Edit to PackageEdits so it can be attempted again.
Log.Error("(error) - package edit blob update failed. Rolling back the DB transaction.");
Log.ErrorException("(exception", e);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
transaction.Rollback();
return;
}
try
{
transaction.Commit();
}
catch (Exception e)
{
// Commit changes to DB failed.
// Since our blob update wasn't part of the transaction (and doesn't AFAIK have a 'commit()' operator we can utilize for the type of blobs we are using)
// try, (single attempt) to roll back the blob update by restoring the previous snapshot.
Log.Error("(error) - package edit DB update failed. Trying to roll back the blob to its previous snapshot.");
Log.ErrorException("(exception", e);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
try
{
latestPackageBlob.StartCopyFromBlob(blobSnapshot);
}
catch (Exception e2)
{
// In this case it may not be the end of the world - the package metadata mismatches the edit now,
// but there's still an edit in the queue, waiting to be rerun and put everything back in synch.
Log.Error("(error) - rolling back the package blob to its previous snapshot failed.");
Log.ErrorException("(exception", e2);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
}
}
}
}
}
}
