public override void ExecuteCommand() { // Work to do: // 0) Find Pending Edits in DB that have been attempted less than 3 times // 1) Backup all old NUPKGS // 2) Generate all new NUPKGs (in place), and tell gallery the edit is completed var connectionString = ConnectionString.ConnectionString; // We group edits together by their package key and process them together - this is a read-only operation var entitiesContext = new EntitiesContext(connectionString, readOnly: true); var editsPerPackage = entitiesContext.Set <PackageEdit>() .GroupBy(pe => pe.PackageKey); // Now that we have our list of packages with pending edits, we'll process the pending edits for each // Note that we're not doing editing in parallel because // a) any particular blob may use a large amount of memory to process. Let's not multiply that! // b) we don't want multithreaded usage of the entitiesContext (and its implied transactions)! foreach (IGrouping <int, PackageEdit> editsGroup in editsPerPackage) { if (editsGroup.Any((pe => pe.TriedCount < 3))) { ProcessPackageEdits(editsGroup.Key, editsGroup); } } }
public override void ExecuteCommand() { // Work to do: // 0) Find Pending Edits in DB that have been attempted less than 3 times // 1) Backup all old NUPKGS // 2) Generate all new NUPKGs (in place), and tell gallery the edit is completed var connectionString = ConnectionString.ConnectionString; var storageAccount = StorageAccount; var entitiesContext = new EntitiesContext(connectionString, readOnly: false); var editsPerPackage = entitiesContext.Set <PackageEdit>() .Where(pe => pe.TriedCount < 3) .Include(pe => pe.Package) .Include(pe => pe.Package.PackageRegistration) .Include(pe => pe.Package.User) .ToList() .GroupBy(pe => pe.PackageKey); // Do edit with a 'most recent edit to this package wins - other edits are deleted' strategy. // Not doing editing in parallel because // a) any particular blob may use a large amount of memory to process. Let's not multiply that! // b) we don't want multithreaded usage of the entitiesContext (and its implied transactions)! foreach (IGrouping <int, PackageEdit> editsGroup in editsPerPackage) { ProcessPackageEdits(editsGroup, entitiesContext); } }
public override void ExecuteCommand() { // Work to do: // 0) Find Pending Edits in DB that have been attempted less than 3 times // 1) Backup all old NUPKGS // 2) Generate all new NUPKGs (in place), and tell gallery the edit is completed var connectionString = ConnectionString.ConnectionString; // We group edits together by their package key and process them together - this is a read-only operation var entitiesContext = new EntitiesContext(connectionString, readOnly: true); var editsPerPackage = entitiesContext.Set<PackageEdit>() .GroupBy(pe => pe.PackageKey); // Now that we have our list of packages with pending edits, we'll process the pending edits for each // Note that we're not doing editing in parallel because // a) any particular blob may use a large amount of memory to process. Let's not multiply that! // b) we don't want multithreaded usage of the entitiesContext (and its implied transactions)! foreach (IGrouping<int, PackageEdit> editsGroup in editsPerPackage) { if (editsGroup.Any((pe => pe.TriedCount < 3))) { ProcessPackageEdits(editsGroup.Key, editsGroup); } } }
public override void ExecuteCommand() { // Work to do: // 0) Find Pending Edits in DB that have been attempted less than 3 times // 1) Backup all old NUPKGS // 2) Generate all new NUPKGs (in place), and tell gallery the edit is completed var connectionString = ConnectionString.ConnectionString; var storageAccount = StorageAccount; var entitiesContext = new EntitiesContext(connectionString, readOnly: false); var editsPerPackage = entitiesContext.Set<PackageEdit>() .Where(pe => pe.TriedCount < 3) .Include(pe => pe.Package) .Include(pe => pe.Package.PackageRegistration) .Include(pe => pe.Package.User) .ToList() .GroupBy(pe => pe.PackageKey); // Do edit with a 'most recent edit to this package wins - other edits are deleted' strategy. // Not doing editing in parallel because // a) any particular blob may use a large amount of memory to process. Let's not multiply that! // b) we don't want multithreaded usage of the entitiesContext (and its implied transactions)! foreach (IGrouping<int, PackageEdit> editsGroup in editsPerPackage) { ProcessPackageEdits(editsGroup, entitiesContext); } }
/// This method is a workaround to the fact that the Seed method /// never seems to get called. So we'll try calling this manually later. public static void SeedDatabase(EntitiesContext context) { var roles = context.Set <Role>(); if (!roles.Any(x => x.Name == Constants.AdminRoleName)) { roles.Add(new Role() { Name = Constants.AdminRoleName }); context.SaveChanges(); } var gallerySettings = context.Set <GallerySetting>(); if (!gallerySettings.Any()) { gallerySettings.Add(new GallerySetting { SmtpHost = null, SmtpPort = null, GalleryOwnerEmail = GalleryOwnerEmail, GalleryOwnerName = GalleryOwnerName, ConfirmEmailAddresses = true }); context.SaveChanges(); } else { var gallerySetting = gallerySettings.First(); if (String.IsNullOrEmpty(gallerySetting.GalleryOwnerEmail)) { gallerySetting.GalleryOwnerEmail = GalleryOwnerEmail; } if (String.IsNullOrEmpty(gallerySetting.GalleryOwnerName)) { gallerySetting.GalleryOwnerName = GalleryOwnerName; } context.SaveChanges(); } }
public override void ExecuteCommand() { //Get all the failed edits. var connectionString = ConnectionString.ConnectionString; var entitiesContext = new EntitiesContext(connectionString, readOnly: true); var failedEdits = entitiesContext.Set <PackageEdit>() .Where(pe => pe.TriedCount == 3).Include(pe => pe.Package).Include(pe => pe.Package.PackageRegistration); //For each ofthe failed edit, send out a support request mail. foreach (PackageEdit edit in failedEdits) { Log.Info( "Sending support request for '{0}'", edit.Package.PackageRegistration.Id); SendMailTask mailTask = new SendMailTask { ConnectionString = this.ConnectionString, UserAccount = this.UserAccount, Password = this.Password, EmailHost = this.EmailHost, ToList = this.UserAccount, ReplyToList = this.UserAccount, MailSubject = $" [NuGet Gallery] : Package Edit Request for {edit.Package.PackageRegistration.Id}", MailContent = $"<b><i>Package:</i></b> {edit.Package.PackageRegistration.Id} </br> <b>Version:</b> {edit.Package.NormalizedVersion} </br> <b>TimeStamp:</b> {edit.Timestamp} </br> <b>LastError:</b> {edit.LastError} </br> <i>Message sent from NuGet Gallery</i> " }; try { mailTask.Execute(); } catch (Exception e) { Log.Error("Creating support request for package {0} failed with error {1}", edit.Package.PackageRegistration.Id, e.Message); } } }
public override void ExecuteCommand() { //Get all the failed edits. var connectionString = ConnectionString.ConnectionString; var entitiesContext = new EntitiesContext(connectionString, readOnly: true); var failedEdits = entitiesContext.Set<PackageEdit>() .Where(pe => pe.TriedCount == 3).Include(pe => pe.Package).Include(pe => pe.Package.PackageRegistration); //For each ofthe failed edit, send out a support request mail. foreach (PackageEdit edit in failedEdits) { Log.Info( "Sending support request for '{0}'", edit.Package.PackageRegistration.Id); SendMailTask mailTask = new SendMailTask { ConnectionString = this.ConnectionString, UserAccount = this.UserAccount, Password = this.Password, EmailHost = this.EmailHost, ToList = this.UserAccount, ReplyToList = this.UserAccount, MailSubject = string.Format(" [NuGet Gallery] : Package Edit Request for {0}", edit.Package.PackageRegistration.Id), MailContent = string.Format("<b><i>Package:</i></b> {0} </br> <b>Version:</b> {1} </br> <b>TimeStamp:</b> {2} </br> <b>LastError:</b> {3} </br> <i>Message sent from NuGet Gallery</i> ", edit.Package.PackageRegistration.Id, edit.Package.NormalizedVersion, edit.Timestamp,edit.LastError) }; try { mailTask.Execute(); } catch (Exception e) { Log.Error("Creating support request for package {0} failed with error {1}", edit.Package.PackageRegistration.Id, e.Message); } } }
private void ProcessPackageEdits(int packageKey) { // Create a fresh entities context so that we work in isolation var entitiesContext = new EntitiesContext(ConnectionString.ConnectionString, readOnly: false); // Get the list of edits for this package // Do edit with a 'most recent edit to this package wins - other edits are deleted' strategy. var editsForThisPackage = entitiesContext.Set <PackageEdit>() .Where(pe => pe.PackageKey == packageKey && pe.TriedCount < 3) .Include(pe => pe.Package) .Include(pe => pe.Package.PackageRegistration) .Include(pe => pe.User) .OrderByDescending(pe => pe.Timestamp) .ToList(); // List of Work to do: // 1) Backup old blob, if the original has not been backed up yet // 2) Downloads blob, create new NUPKG locally // 3) Upload blob // 4) Update the database PackageEdit edit = editsForThisPackage.First(); var blobClient = StorageAccount.CreateCloudBlobClient(); var packagesContainer = Util.GetPackagesBlobContainer(blobClient); var latestPackageFileName = Util.GetPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version); var originalPackageFileName = Util.GetBackupOfOriginalPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version); var originalPackageBackupBlob = packagesContainer.GetBlockBlobReference(originalPackageFileName); var latestPackageBlob = packagesContainer.GetBlockBlobReference(latestPackageFileName); var edits = new List <Action <ManifestMetadata> > { (m) => { m.Authors = edit.Authors; }, (m) => { m.Copyright = edit.Copyright; }, (m) => { m.Description = edit.Description; }, (m) => { m.IconUrl = edit.IconUrl; }, (m) => { m.LicenseUrl = edit.LicenseUrl; }, (m) => { m.ProjectUrl = edit.ProjectUrl; }, (m) => { m.ReleaseNotes = edit.ReleaseNotes; }, (m) => { m.RequireLicenseAcceptance = edit.RequiresLicenseAcceptance; }, (m) => { m.Summary = edit.Summary; }, (m) => { m.Title = edit.Title; }, (m) => { m.Tags = edit.Tags; }, }; Log.Info( "Processing Edit Key={0}, PackageId={1}, Version={2}, User={3}", edit.Key, edit.Package.PackageRegistration.Id, edit.Package.Version, edit.User.Username); if (!WhatIf) { edit.TriedCount += 1; int nr = entitiesContext.SaveChanges(); if (nr != 1) { throw new Exception( String.Format("Something went terribly wrong, only one entity should be updated but actually {0} entities were updated", nr)); } } try { ArchiveOriginalPackageBlob(originalPackageBackupBlob, latestPackageBlob); using (var readWriteStream = new MemoryStream()) { // Download to memory CloudBlockBlob downloadSourceBlob = WhatIf ? latestPackageBlob : originalPackageBackupBlob; Log.Info("Downloading original package blob to memory {0}", downloadSourceBlob.Name); downloadSourceBlob.DownloadToStream(readWriteStream); // Rewrite in memory Log.Info("Rewriting nupkg package in memory", downloadSourceBlob.Name); NupkgRewriter.RewriteNupkgManifest(readWriteStream, edits); // Get updated hash code, and file size Log.Info("Computing updated hash code of memory stream"); var newPackageFileSize = readWriteStream.Length; var hashAlgorithm = HashAlgorithm.Create("SHA512"); byte[] hashBytes = hashAlgorithm.ComputeHash(readWriteStream.GetBuffer()); var newHash = Convert.ToBase64String(hashBytes); if (!WhatIf) { // Snapshot the blob var blobSnapshot = latestPackageBlob.CreateSnapshot(); // Build up the changes in the entities context edit.Apply(hashAlgorithm: "SHA512", hash: newHash, packageFileSize: newPackageFileSize); foreach (var eachEdit in editsForThisPackage) { entitiesContext.DeleteOnCommit(eachEdit); } // Upload the blob before doing SaveChanges(). If blob update fails, we won't do SaveChanges() and the edit can be retried. // If SaveChanges() fails we can undo the blob upload. try { Log.Info("Uploading blob from memory {0}", latestPackageBlob.Name); readWriteStream.Position = 0; latestPackageBlob.UploadFromStream(readWriteStream); } catch (Exception e) { Log.Error("(error) - package edit blob update failed."); Log.ErrorException("(exception)", e); Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri); throw; // To handler block that will record error in DB } try { // SaveChanges tries to commit changes to DB entitiesContext.SaveChanges(); } catch (Exception e) { // Commit changes to DB probably failed. // Since our blob update wasn't part of the transaction (and doesn't AFAIK have a 'commit()' operator we can utilize for the type of blobs we are using) // try, (single attempt) to roll back the blob update by restoring the previous snapshot. Log.Error("(error) - package edit DB update failed. Trying to roll back the blob to its previous snapshot."); Log.ErrorException("(exception)", e); Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri); try { latestPackageBlob.StartCopyFromBlob(blobSnapshot); } catch (Exception e2) { // If blob rollback fails it is not be the end of the world // - the package metadata mismatches the edit now, // but there should still an edit in the queue, waiting to be rerun and put everything back in synch. Log.Error("(error) - rolling back the package blob to its previous snapshot failed."); Log.ErrorException("(exception)", e2); Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri); } throw; // To handler block that will record error in DB } } } } catch (Exception e) { if (!WhatIf) { try { Log.Info("Storing the error on package edit with key {0}", edit.Key); // Try to record the error into the PackageEdit database record // so that we can actually diagnose failures. // This must be done on a fresh context to ensure no conflicts. var errorContext = new EntitiesContext(ConnectionString.ConnectionString, readOnly: false); var errorEdit = errorContext.Set <PackageEdit>().Where(pe => pe.Key == edit.Key).FirstOrDefault(); if (errorEdit != null) { errorEdit.LastError = string.Format("{0} : {1}", e.GetType(), e); errorContext.SaveChanges(); } else { Log.Info("The package edit with key {0} couldn't be found. It was likely canceled and deleted.", edit.Key); } } catch (Exception errorException) { Log.ErrorException("(error) - couldn't save the last error on the edit that was being applied.", errorException); } } } }
private void ProcessPackageEdits(int packageKey, IEnumerable<PackageEdit> editsToDelete) { // Create a fresh entities context so that we work in isolation var entitiesContext = new EntitiesContext(ConnectionString.ConnectionString, readOnly: false); // Get the most recent edit for this package var edit = entitiesContext.Set<PackageEdit>() .Where(pe => pe.PackageKey == packageKey && pe.TriedCount < 3) .Include(pe => pe.Package) .Include(pe => pe.Package.PackageRegistration) .Include(pe => pe.User) .OrderByDescending(pe => pe.Timestamp) .First(); // List of Work to do: // 1) Backup old blob, if the original has not been backed up yet // 2) Downloads blob, create new NUPKG locally // 3) Upload blob // 4) Update the database var blobClient = StorageAccount.CreateCloudBlobClient(); var packagesContainer = Util.GetPackagesBlobContainer(blobClient); var latestPackageFileName = Util.GetPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version); var originalPackageFileName = Util.GetBackupOfOriginalPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version); var originalPackageBackupBlob = packagesContainer.GetBlockBlobReference(originalPackageFileName); var latestPackageBlob = packagesContainer.GetBlockBlobReference(latestPackageFileName); var edits = new List<Action<ManifestEdit>> { (m) => { m.Authors = edit.Authors; }, (m) => { m.Copyright = edit.Copyright; }, (m) => { m.Description = edit.Description; }, (m) => { m.IconUrl = edit.IconUrl; }, (m) => { m.LicenseUrl = edit.LicenseUrl; }, (m) => { m.ProjectUrl = edit.ProjectUrl; }, (m) => { m.ReleaseNotes = edit.ReleaseNotes; }, (m) => { m.RequireLicenseAcceptance = edit.RequiresLicenseAcceptance; }, (m) => { m.Summary = edit.Summary; }, (m) => { m.Title = edit.Title; }, (m) => { m.Tags = edit.Tags; }, }; Log.Info( "Processing Edit Key={0}, PackageId={1}, Version={2}, User={3}", edit.Key, edit.Package.PackageRegistration.Id, edit.Package.Version, edit.User.Username); if (!WhatIf) { edit.TriedCount += 1; int nr = entitiesContext.SaveChanges(); if (nr != 1) { throw new Exception( $"Something went terribly wrong, only one entity should be updated but actually {nr} entities were updated"); } } try { ArchiveOriginalPackageBlob(originalPackageBackupBlob, latestPackageBlob); using (var readWriteStream = new MemoryStream()) { // Download to memory CloudBlockBlob downloadSourceBlob = WhatIf ? latestPackageBlob : originalPackageBackupBlob; Log.Info("Downloading original package blob to memory {0}", downloadSourceBlob.Name); downloadSourceBlob.DownloadToStream(readWriteStream); // Rewrite in memory Log.Info("Rewriting nupkg package in memory", downloadSourceBlob.Name); NupkgRewriter.RewriteNupkgManifest(readWriteStream, edits); // Get updated hash code, and file size Log.Info("Computing updated hash code of memory stream"); var newPackageFileSize = readWriteStream.Length; var hashAlgorithm = HashAlgorithm.Create("SHA512"); byte[] hashBytes = hashAlgorithm.ComputeHash(readWriteStream.GetBuffer()); var newHash = Convert.ToBase64String(hashBytes); if (!WhatIf) { // Snapshot the blob var blobSnapshot = latestPackageBlob.CreateSnapshot(); // Build up the changes in the entities context edit.Apply(hashAlgorithm: "SHA512", hash: newHash, packageFileSize: newPackageFileSize); foreach (var eachEdit in editsToDelete) { entitiesContext.DeleteOnCommit(eachEdit); } // Upload the blob before doing SaveChanges(). If blob update fails, we won't do SaveChanges() and the edit can be retried. // If SaveChanges() fails we can undo the blob upload. try { Log.Info("Uploading blob from memory {0}", latestPackageBlob.Name); readWriteStream.Position = 0; latestPackageBlob.UploadFromStream(readWriteStream); } catch (Exception e) { Log.Error("(error) - package edit blob update failed."); Log.ErrorException("(exception)", e); Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri); throw; // To handler block that will record error in DB } try { // SaveChanges tries to commit changes to DB entitiesContext.SaveChanges(); } catch (Exception e) { // Commit changes to DB probably failed. // Since our blob update wasn't part of the transaction (and doesn't AFAIK have a 'commit()' operator we can utilize for the type of blobs we are using) // try, (single attempt) to roll back the blob update by restoring the previous snapshot. Log.Error("(error) - package edit DB update failed. Trying to roll back the blob to its previous snapshot."); Log.ErrorException("(exception)", e); Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri); try { latestPackageBlob.StartCopyFromBlob(blobSnapshot); } catch (Exception e2) { // If blob rollback fails it is not be the end of the world // - the package metadata mismatches the edit now, // but there should still an edit in the queue, waiting to be rerun and put everything back in synch. Log.Error("(error) - rolling back the package blob to its previous snapshot failed."); Log.ErrorException("(exception)", e2); Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri); } throw; // To handler block that will record error in DB } } } } catch (Exception e) { if (!WhatIf) { try { Log.Info("Storing the error on package edit with key {0}", edit.Key); // Try to record the error into the PackageEdit database record // so that we can actually diagnose failures. // This must be done on a fresh context to ensure no conflicts. var errorContext = new EntitiesContext(ConnectionString.ConnectionString, readOnly: false); var errorEdit = errorContext.Set<PackageEdit>().Where(pe => pe.Key == edit.Key).FirstOrDefault(); if (errorEdit != null) { errorEdit.LastError = $"{e.GetType()} : {e}"; errorContext.SaveChanges(); } else { Log.Info("The package edit with key {0} couldn't be found. It was likely canceled and deleted.", edit.Key); } } catch (Exception errorException) { Log.ErrorException("(error) - couldn't save the last error on the edit that was being applied.", errorException); } } } }
public IDbSet <T> CreateObjectSet <T>() where T : class { return(_context.Set <T>()); }
public TEntity Get(int id) { return(DbContext.Set <TEntity>().Find(id)); }
public IQueryable <T> FindAll(bool trackChanges) => !trackChanges ? EntitiesContext.Set <T>().AsNoTracking() : EntitiesContext.Set <T>();
public void Add(TEntity entity) { Context.Set <TEntity>().Add(entity); }
private static async Task Hash(string connectionString, bool whatIf) { using (var context = new EntitiesContext(connectionString, readOnly: whatIf)) { var allCredentials = (IQueryable <Credential>)context.Set <Credential>(); // Get all V1/V2 credentials that are active: // V1 credentials that have no expiration date, but were used in the last year // V1/V2 credentials that have a future expiration date var validLastUsed = DateTime.UtcNow - TimeSpan.FromDays(365); Expression <Func <Credential, bool> > predicate = x => (x.Type == CredentialTypes.ApiKey.V1 || x.Type == CredentialTypes.ApiKey.V2) && ((x.Expires == null && x.LastUsed != null && x.LastUsed > validLastUsed) || (x.Expires != null && x.Expires > DateTime.UtcNow)); var activeCredentialsCount = allCredentials.Count(predicate); Console.WriteLine($"Found {activeCredentialsCount} active V1/V2 ApiKeys."); IList <Credential> batch; int batchNumber = 1; bool failures = false; do { batch = allCredentials.Where(predicate).OrderBy(x => x.Key).Take(BatchSize).ToList(); if (batch.Count > 0) { Console.WriteLine($"Hashing batch {batchNumber}/{Math.Ceiling((double)activeCredentialsCount / (double)BatchSize)}. Batch size: {batch.Count}..."); foreach (var v1v2ApiKey in batch) { ApiKeyV3 hashedApiKey = null; try { hashedApiKey = ApiKeyV3.CreateFromV1V2ApiKey(v1v2ApiKey.Value); v1v2ApiKey.Type = CredentialTypes.ApiKey.V3; v1v2ApiKey.Value = hashedApiKey.HashedApiKey; } catch (ArgumentException e) { failures = true; Console.WriteLine($"Failed to hash key: {v1v2ApiKey.Key} Type: {v1v2ApiKey.Type} Old value: {v1v2ApiKey.Value}. Reason: {e}"); } } if (!failures && !whatIf) { var stopwatch = Stopwatch.StartNew(); Console.WriteLine($"Saving batch {batchNumber} to DB..."); await context.SaveChangesAsync(); Console.WriteLine($"Saved changes to DB. Took: {stopwatch.Elapsed.TotalSeconds} seconds"); } else { Console.WriteLine("Skipping DB save."); allCredentials = allCredentials.Where(predicate).OrderBy(x => x.Key).Skip(batch.Count); } batchNumber++; } }while (batch.Count > 0); } }
public List <TEntity> Get() { return(_context.Set <TEntity>().AsNoTracking().ToList()); }
public GenericRepository(EntitiesContext _Context) { Context = _Context; dbSet = Context.Set <T>(); }
public void Add(IEntity entity) { _context.Set(entity.GetType()).Add(entity); }
public Repository(EntitiesContext entitiesContext) { EntitiesContext = entitiesContext; DbSet = entitiesContext.Set <T>(); }
public IQueryable <T> GetAll() { return(entityContext.Set <T>()); }