public override void ExecuteCommand()
{
// Work to do:
// 0) Find Pending Edits in DB that have been attempted less than 3 times
// 1) Backup all old NUPKGS
// 2) Generate all new NUPKGs (in place), and tell gallery the edit is completed
var connectionString = ConnectionString.ConnectionString;
// We group edits together by their package key and process them together - this is a read-only operation
var entitiesContext = new EntitiesContext(connectionString, readOnly: true);
var editsPerPackage = entitiesContext.Set<PackageEdit>()
.GroupBy(pe => pe.PackageKey);
// Now that we have our list of packages with pending edits, we'll process the pending edits for each
// Note that we're not doing editing in parallel because
// a) any particular blob may use a large amount of memory to process. Let's not multiply that!
// b) we don't want multithreaded usage of the entitiesContext (and its implied transactions)!
foreach (IGrouping<int, PackageEdit> editsGroup in editsPerPackage)
{
if (editsGroup.Any((pe => pe.TriedCount < 3)))
{
ProcessPackageEdits(editsGroup.Key, editsGroup);
}
}
}
public override void ExecuteCommand()
{
// Work to do:
// 0) Find Pending Edits in DB that have been attempted less than 3 times
// 1) Backup all old NUPKGS
// 2) Generate all new NUPKGs (in place), and tell gallery the edit is completed
var connectionString = ConnectionString.ConnectionString;
var storageAccount = StorageAccount;
var entitiesContext = new EntitiesContext(connectionString, readOnly: false);
var editsPerPackage = entitiesContext.Set<PackageEdit>()
.Where(pe => pe.TriedCount < 3)
.Include(pe => pe.Package)
.Include(pe => pe.Package.PackageRegistration)
.Include(pe => pe.Package.User)
.ToList()
.GroupBy(pe => pe.PackageKey);
// Do edit with a 'most recent edit to this package wins - other edits are deleted' strategy.
// Not doing editing in parallel because
// a) any particular blob may use a large amount of memory to process. Let's not multiply that!
// b) we don't want multithreaded usage of the entitiesContext (and its implied transactions)!
foreach (IGrouping<int, PackageEdit> editsGroup in editsPerPackage)
{
ProcessPackageEdits(editsGroup, entitiesContext);
}
}
/// <summary>
/// Инициализирует новый экземпляр класса <see cref="UnitOfWorkEf" />
/// </summary>
/// <param name="contextFactory">Фабрика контекста доступа к БД</param>
/// <param name="isolationLevel">Уровень изоляции данных</param>
public UnitOfWorkEf(
IDbContextFactory contextFactory,
IsolationLevel isolationLevel = IsolationLevel.ReadCommitted)
{
this._context = contextFactory.CreateDbContext<EntitiesContext>();
// Если БД не была создана вызовет ошибку
this._transaction = this._context.Database.BeginTransaction(isolationLevel);
}
public void SaveChangesFailsInReadOnlyMode()
{
var ec = new EntitiesContext("", readOnly: true);
Assert.Throws<ReadOnlyException>(() => ec.Users.Add(new User
{
Key = -1,
Username = "******",
}));
}
public void CreateEntitiesFact()
{
var context = new EntitiesContext
{
Namespace = @"kkkkkkaaaaaa.DataTransferObjects",
Imports = new[] { "System", },
TypeNameSuffix = @"Entity",
};
var entities = new Entities(context)
.CreateEntities();
}
public void Fact()
{
var entity = new EntitiesContext()
{
Namespace = @"Namespace",
Imports = new [] { @"Import", },
TypeName = @"TypeName",
TypeNameSuffix = @"Suffix",
Inherits = @"BaseClass",
Columns = new []
{
new SqlColumnsSchemaEntity() { ColumnName = @"ColumnName", ColumnSize = 1, DataType = typeof(string), DataTypeName = @"string", NumericScale = 7 },
}
};
var template = new NotifyPropertyChangedTemplate(entity);
var text = template.TransformText();
}
public SimpleMembershipInitializer()
{
Database.SetInitializer<EntitiesContext>(null);
try
{
using (var context = new EntitiesContext())
{
if (!context.Database.Exists())
{
// Create the SimpleMembership database without Entity Framework migration schema
((IObjectContextAdapter)context).ObjectContext.CreateDatabase();
}
}
}
catch (Exception ex)
{
throw new InvalidOperationException("The ASP.NET Simple Membership database could not be initialized. For more information, please see http://go.microsoft.com/fwlink/?LinkId=256588", ex);
}
}
public override void ExecuteCommand()
{
//Get all the failed edits.
var connectionString = ConnectionString.ConnectionString;
var entitiesContext = new EntitiesContext(connectionString, readOnly: true);
var failedEdits = entitiesContext.Set<PackageEdit>()
.Where(pe => pe.TriedCount == 3).Include(pe => pe.Package).Include(pe => pe.Package.PackageRegistration);
//For each ofthe failed edit, send out a support request mail.
foreach (PackageEdit edit in failedEdits)
{
Log.Info(
"Sending support request for '{0}'",
edit.Package.PackageRegistration.Id);
SendMailTask mailTask = new SendMailTask
{
ConnectionString = this.ConnectionString,
UserAccount = this.UserAccount,
Password = this.Password,
EmailHost = this.EmailHost,
ToList = this.UserAccount,
ReplyToList = this.UserAccount,
MailSubject = string.Format(" [NuGet Gallery] : Package Edit Request for {0}", edit.Package.PackageRegistration.Id),
MailContent = string.Format("<b><i>Package:</i></b> {0} </br> <b>Version:</b> {1} </br> <b>TimeStamp:</b> {2} </br> <b>LastError:</b> {3} </br> <i>Message sent from NuGet Gallery</i> ", edit.Package.PackageRegistration.Id, edit.Package.NormalizedVersion, edit.Timestamp,edit.LastError)
};
try
{
mailTask.Execute();
}
catch (Exception e)
{
Log.Error("Creating support request for package {0} failed with error {1}", edit.Package.PackageRegistration.Id, e.Message);
}
}
}
public EmployeesController()
{
this._dbContext = new EntitiesContext();
}
public CheckoutController(EntitiesContext context)
{
_context = context;
}
public UserRepository(EntitiesContext context) : base(context)
{
}
private void ProcessPackageEdits(IEnumerable <PackageEdit> editsForThisPackage, EntitiesContext entitiesContext)
{
// List of Work to do:
// 1) Backup old blob, if the original has not been backed up yet
// 2) Downloads blob, create new NUPKG locally
// 3) Upload blob
// 4) Update the database
PackageEdit edit = editsForThisPackage.OrderByDescending(pe => pe.Timestamp).First();
var blobClient = StorageAccount.CreateCloudBlobClient();
var packagesContainer = Util.GetPackagesBlobContainer(blobClient);
var latestPackageFileName = Util.GetPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version);
var originalPackageFileName = Util.GetBackupOfOriginalPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version);
var originalPackageBackupBlob = packagesContainer.GetBlockBlobReference(originalPackageFileName);
var latestPackageBlob = packagesContainer.GetBlockBlobReference(latestPackageFileName);
var edits = new List <Action <ManifestMetadata> >
{
(m) => { m.Authors = edit.Authors; },
(m) => { m.Copyright = edit.Copyright; },
(m) => { m.Description = edit.Description; },
(m) => { m.IconUrl = edit.IconUrl; },
(m) => { m.LicenseUrl = edit.LicenseUrl; },
(m) => { m.ProjectUrl = edit.ProjectUrl; },
(m) => { m.ReleaseNotes = edit.ReleaseNotes; },
(m) => { m.RequireLicenseAcceptance = edit.RequiresLicenseAcceptance; },
(m) => { m.Summary = edit.Summary; },
(m) => { m.Title = edit.Title; },
(m) => { m.Tags = edit.Tags; },
};
Log.Info(
"Processing Edit Key={0}, PackageId={1}, Version={2}",
edit.Key,
edit.Package.PackageRegistration.Id,
edit.Package.Version);
if (!WhatIf)
{
edit.TriedCount += 1;
int nr = entitiesContext.SaveChanges();
if (nr != 1)
{
throw new ApplicationException(
String.Format("Something went terribly wrong, only one entity should be updated but actually {0} entities were updated", nr));
}
}
ArchiveOriginalPackageBlob(originalPackageBackupBlob, latestPackageBlob);
using (var readWriteStream = new MemoryStream())
{
// Download to memory
CloudBlockBlob downloadSourceBlob = WhatIf ? latestPackageBlob : originalPackageBackupBlob;
Log.Info("Downloading original package blob to memory {0}", downloadSourceBlob.Name);
downloadSourceBlob.DownloadToStream(readWriteStream);
// Rewrite in memory
Log.Info("Rewriting nupkg package in memory", downloadSourceBlob.Name);
NupkgRewriter.RewriteNupkgManifest(readWriteStream, edits);
// Get updated hash code, and file size
Log.Info("Computing updated hash code of memory stream");
var newPackageFileSize = readWriteStream.Length;
var hashAlgorithm = HashAlgorithm.Create("SHA512");
byte[] hashBytes = hashAlgorithm.ComputeHash(readWriteStream.GetBuffer());
var newHash = Convert.ToBase64String(hashBytes);
if (!WhatIf)
{
// Snapshot the blob
var blobSnapshot = latestPackageBlob.CreateSnapshot();
// Start Transaction: Complete the edit in the gallery DB.
// Use explicit SQL transactions instead of EF operation-grouping
// so that we can manually roll the transaction back on a blob related failure.
ObjectContext objectContext = (entitiesContext as IObjectContextAdapter).ObjectContext;
((objectContext.Connection) as EntityConnection).Open(); // must open in order to begin transaction
using (EntityTransaction transaction = ((objectContext.Connection) as EntityConnection).BeginTransaction())
{
edit.Apply(hashAlgorithm: "SHA512", hash: newHash, packageFileSize: newPackageFileSize);
// Add to transaction: delete all the pending edits of this package.
foreach (var eachEdit in editsForThisPackage)
{
entitiesContext.DeleteOnCommit(eachEdit);
}
entitiesContext.SaveChanges(); // (transaction is still not committed, but do some EF legwork up-front of modifying the blob)
try
{
// Reupload blob
Log.Info("Uploading blob from memory {0}", latestPackageBlob.Name);
readWriteStream.Position = 0;
latestPackageBlob.UploadFromStream(readWriteStream);
}
catch (Exception e)
{
// Uploading the updated nupkg failed.
// Rollback the transaction, which restores the Edit to PackageEdits so it can be attempted again.
Log.Error("(error) - package edit blob update failed. Rolling back the DB transaction.");
Log.ErrorException("(exception", e);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
transaction.Rollback();
return;
}
try
{
transaction.Commit();
}
catch (Exception e)
{
// Commit changes to DB failed.
// Since our blob update wasn't part of the transaction (and doesn't AFAIK have a 'commit()' operator we can utilize for the type of blobs we are using)
// try, (single attempt) to roll back the blob update by restoring the previous snapshot.
Log.Error("(error) - package edit DB update failed. Trying to roll back the blob to its previous snapshot.");
Log.ErrorException("(exception", e);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
try
{
latestPackageBlob.StartCopyFromBlob(blobSnapshot);
}
catch (Exception e2)
{
// In this case it may not be the end of the world - the package metadata mismatches the edit now,
// but there's still an edit in the queue, waiting to be rerun and put everything back in synch.
Log.Error("(error) - rolling back the package blob to its previous snapshot failed.");
Log.ErrorException("(exception", e2);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
}
}
}
}
}
}
public CityRepository(EntitiesContext context)
{
this._context = context;
}
public Repository(EntitiesContext entitiesContext)
{
EntitiesContext = entitiesContext;
DbSet = entitiesContext.Set <T>();
}
public AddNewCustomer(EntitiesContext context)
{
_context = context;
}
public GenericRepository(EntitiesContext entityContext)
{
this.entityContext = entityContext;
}
public UnitOfWork(EntitiesContext entities)
{
this.entities = entities;
}
public ActionResult Submission(int id, SubmissionUploadViewModel submissionUploadViewModel)
{
var problemId = id;
var rawFile = new byte[submissionUploadViewModel.FileUpload.Length];
var fileStream = submissionUploadViewModel.FileUpload.OpenReadStream();
fileStream.Read(rawFile, 0, rawFile.Length);
var submission = new Submission
{
FileName = submissionUploadViewModel.FileUpload.FileName,
LoginId = int.Parse(User.Claims.Single(c => c.Type == ClaimTypes.Sid).Value),
ProblemId = problemId,
ProgrammingLanguage = submissionUploadViewModel.ProgrammingLanguage,
RawFile = rawFile,
SubmissionDateTime = DateTime.Now
};
EntitiesContext.Submissions.Add(submission);
EntitiesContext.SaveChanges();
//var connectionFactory = new ConnectionFactory
//{
// Uri = new Uri("amqp://*****:*****@rabbitmq:5672")
//};
//using (var connection = connectionFactory.CreateConnection())
//using (var channel = connection.CreateModel())
//{
// const string exchange = "tsa-coding-challenge";
// const string routingKey = "tsa";
// string queue;
// switch (submissionUploadViewModel.ProgrammingLanguage)
// {
// case ProgrammingLanguage.C:
// queue = QueueNames.C;
// break;
// case ProgrammingLanguage.CPlusPlus:
// queue = QueueNames.CPlusPlus;
// break;
// case ProgrammingLanguage.CSharp:
// queue = QueueNames.CSharp;
// break;
// case ProgrammingLanguage.FSharp:
// queue = QueueNames.FSharp;
// break;
// case ProgrammingLanguage.Java:
// queue = QueueNames.Java;
// break;
// case ProgrammingLanguage.NodeJs:
// queue = QueueNames.NodeJs;
// break;
// case ProgrammingLanguage.Perl:
// queue = QueueNames.Perl;
// break;
// case ProgrammingLanguage.Python:
// queue = QueueNames.Python;
// break;
// case ProgrammingLanguage.Ruby:
// queue = QueueNames.Ruby;
// break;
// case ProgrammingLanguage.VbDotNet:
// queue = QueueNames.VbDotNet;
// break;
// default:
// throw new NotImplementedException();
// }
// channel.ExchangeDeclare(exchange, ExchangeType.Direct);
// channel.QueueDeclare(queue, false, false, false, null);
// channel.QueueBind(queue, exchange, routingKey, null);
// var encodedFile = Convert.ToBase64String(rawFile, 0, rawFile.Length);
// var message = JsonConvert.SerializeObject(new { submissionId = submission.Id, problemId, fileName = submission.FileName, file = encodedFile });
// var body = Encoding.UTF8.GetBytes(message);
// channel.BasicPublish(exchange, routingKey, null, body);
//}
return(RedirectToAction("Index", new { solutionSubmitted = true }));
}
public AvaliacaoDAO()
{
context = new EntitiesContext();
}
public ClassEntityRegistrar(ClassEntityRegistrarFactory classEntityRegistrarFactory, ApiRootClassEntityRegistrar rootApiClassEntityRegistrar, ApiClassEntityRegistrar apiClassEntityRegistrar, EntitiesContext entitiesContext, NamespaceApiToTypeDefinitionConverter namespaceApiToTypeDefinitionConverter, RegistrationOptions registrationOptions)
{
this.classEntityRegistrarFactory = classEntityRegistrarFactory;
this.rootApiClassEntityRegistrar = rootApiClassEntityRegistrar;
this.apiClassEntityRegistrar = apiClassEntityRegistrar;
this.entitiesContext = entitiesContext;
this.namespaceApiToTypeDefinitionConverter = namespaceApiToTypeDefinitionConverter;
this.registrationOptions = registrationOptions;
}
public FakeUnitOfWork(EntitiesContext context)
{
_context = context;
}
protected abstract void UpdatePackage(Package package, TMetadata metadata, EntitiesContext context);
public MenuDAO()
{
context = new EntitiesContext();
}
protected BaseClassEntityRegistrar(EntitiesContext entitiesContext)
{
this.entitiesContext = entitiesContext;
}
public TrimManager(EntitiesContext entitiesContext) : base(entitiesContext)
{
}
public NamespaceEntityRegistrar(EntitiesContext entitiesContext, ILogger logger)
{
this.entitiesContext = entitiesContext;
this.logger = logger;
}
public HomeController(EntitiesContext context)
{
this.context = context;
}
public RentingRepository(EntitiesContext context) : base(context)
{
}
public ArtistDAO()
{
context = new EntitiesContext();
}
public async Task Collect(SqlConnection connection, Uri serviceDiscoveryUri, DateTime?lastCreateTime, string fileName)
{
using (var context = new EntitiesContext(connection, readOnly: true))
using (var cursor = new FileCursor(CursorFileName))
using (var logger = new Logger(ErrorsFileName))
{
context.SetCommandTimeout(300); // large query
var startTime = await cursor.Read();
logger.Log($"Starting metadata collection - Cursor time: {startTime:u}");
var repository = new EntityRepository <Package>(context);
var packages = repository.GetAll()
.Include(p => p.PackageRegistration);
if (QueryIncludes != null)
{
packages = packages.Include(QueryIncludes);
}
packages = packages
.Where(p => p.Created <lastCreateTime && p.Created> startTime)
.Where(p => p.PackageStatusKey == PackageStatus.Available)
.OrderBy(p => p.Created);
if (LimitTo > 0)
{
packages = packages.Take(LimitTo);
}
var flatContainerUri = await GetFlatContainerUri(serviceDiscoveryUri);
using (var csv = CreateCsvWriter(fileName))
using (var http = new HttpClient())
{
// We want these downloads ignored by stats pipelines - this user agent is automatically skipped.
// See https://github.com/NuGet/NuGet.Jobs/blob/262da48ed05d0366613bbf1c54f47879aad96dcd/src/Stats.ImportAzureCdnStatistics/StatisticsParser.cs#L41
http.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; AppInsights) Backfill Job: NuGet.Gallery GalleryTools");
var counter = 0;
var lastCreatedDate = default(DateTime?);
foreach (var package in packages)
{
var id = package.PackageRegistration.Id;
var version = package.NormalizedVersion;
var idLowered = id.ToLowerInvariant();
var versionLowered = version.ToLowerInvariant();
try
{
var metadata = default(TMetadata);
var nuspecUri =
$"{flatContainerUri}/{idLowered}/{versionLowered}/{idLowered}.nuspec";
using (var nuspecStream = await http.GetStreamAsync(nuspecUri))
{
var document = LoadDocument(nuspecStream);
var nuspecReader = new NuspecReader(document);
if (SourceType == MetadataSourceType.NuspecOnly)
{
metadata = ReadMetadata(nuspecReader);
}
else if (SourceType == MetadataSourceType.Nupkg)
{
var nupkgUri =
$"{flatContainerUri}/{idLowered}/{versionLowered}/{idLowered}.{versionLowered}.nupkg";
metadata = await FetchMetadataAsync(http, nupkgUri, nuspecReader, id, version, logger);
}
}
if (ShouldWriteMetadata(metadata))
{
var record = new PackageMetadata(id, version, metadata, package.Created);
csv.WriteRecord(record);
await csv.NextRecordAsync();
logger.LogPackage(id, version, $"Metadata saved");
}
}
catch (Exception e)
{
await logger.LogPackageError(id, version, e);
}
counter++;
if (!lastCreatedDate.HasValue || lastCreatedDate < package.Created)
{
lastCreatedDate = package.Created;
}
if (counter >= CollectBatchSize)
{
logger.Log($"Writing {package.Created:u} to cursor...");
await cursor.Write(package.Created);
counter = 0;
}
}
if (counter > 0 && lastCreatedDate.HasValue)
{
await cursor.Write(lastCreatedDate.Value);
}
}
}
}
public GeneroDAO()
{
context = new EntitiesContext();
}
private void ProcessPackageEdits(IEnumerable<PackageEdit> editsForThisPackage, EntitiesContext entitiesContext)
{
// List of Work to do:
// 1) Backup old blob, if the original has not been backed up yet
// 2) Downloads blob, create new NUPKG locally
// 3) Upload blob
// 4) Update the database
PackageEdit edit = editsForThisPackage.OrderByDescending(pe => pe.Timestamp).First();
var blobClient = StorageAccount.CreateCloudBlobClient();
var packagesContainer = Util.GetPackagesBlobContainer(blobClient);
var latestPackageFileName = Util.GetPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version);
var originalPackageFileName = Util.GetBackupOfOriginalPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version);
var originalPackageBackupBlob = packagesContainer.GetBlockBlobReference(originalPackageFileName);
var latestPackageBlob = packagesContainer.GetBlockBlobReference(latestPackageFileName);
var edits = new List<Action<ManifestMetadata>>
{
(m) => { m.Authors = edit.Authors; },
(m) => { m.Copyright = edit.Copyright; },
(m) => { m.Description = edit.Description; },
(m) => { m.IconUrl = edit.IconUrl; },
(m) => { m.LicenseUrl = edit.LicenseUrl; },
(m) => { m.ProjectUrl = edit.ProjectUrl; },
(m) => { m.ReleaseNotes = edit.ReleaseNotes; },
(m) => { m.RequireLicenseAcceptance = edit.RequiresLicenseAcceptance; },
(m) => { m.Summary = edit.Summary; },
(m) => { m.Title = edit.Title; },
(m) => { m.Tags = edit.Tags; },
};
Log.Info(
"Processing Edit Key={0}, PackageId={1}, Version={2}",
edit.Key,
edit.Package.PackageRegistration.Id,
edit.Package.Version);
if (!WhatIf)
{
edit.TriedCount += 1;
int nr = entitiesContext.SaveChanges();
if (nr != 1)
{
throw new ApplicationException(
String.Format("Something went terribly wrong, only one entity should be updated but actually {0} entities were updated", nr));
}
}
ArchiveOriginalPackageBlob(originalPackageBackupBlob, latestPackageBlob);
using (var readWriteStream = new MemoryStream())
{
// Download to memory
CloudBlockBlob downloadSourceBlob = WhatIf ? latestPackageBlob : originalPackageBackupBlob;
Log.Info("Downloading original package blob to memory {0}", downloadSourceBlob.Name);
downloadSourceBlob.DownloadToStream(readWriteStream);
// Rewrite in memory
Log.Info("Rewriting nupkg package in memory", downloadSourceBlob.Name);
NupkgRewriter.RewriteNupkgManifest(readWriteStream, edits);
// Get updated hash code, and file size
Log.Info("Computing updated hash code of memory stream");
var newPackageFileSize = readWriteStream.Length;
var hashAlgorithm = HashAlgorithm.Create("SHA512");
byte[] hashBytes = hashAlgorithm.ComputeHash(readWriteStream.GetBuffer());
var newHash = Convert.ToBase64String(hashBytes);
if (!WhatIf)
{
// Snapshot the blob
var blobSnapshot = latestPackageBlob.CreateSnapshot();
// Start Transaction: Complete the edit in the gallery DB.
// Use explicit SQL transactions instead of EF operation-grouping
// so that we can manually roll the transaction back on a blob related failure.
ObjectContext objectContext = (entitiesContext as IObjectContextAdapter).ObjectContext;
((objectContext.Connection) as EntityConnection).Open(); // must open in order to begin transaction
using (EntityTransaction transaction = ((objectContext.Connection) as EntityConnection).BeginTransaction())
{
edit.Apply(hashAlgorithm: "SHA512", hash: newHash, packageFileSize: newPackageFileSize);
// Add to transaction: delete all the pending edits of this package.
foreach (var eachEdit in editsForThisPackage)
{
entitiesContext.DeleteOnCommit(eachEdit);
}
entitiesContext.SaveChanges(); // (transaction is still not committed, but do some EF legwork up-front of modifying the blob)
try
{
// Reupload blob
Log.Info("Uploading blob from memory {0}", latestPackageBlob.Name);
readWriteStream.Position = 0;
latestPackageBlob.UploadFromStream(readWriteStream);
}
catch (Exception e)
{
// Uploading the updated nupkg failed.
// Rollback the transaction, which restores the Edit to PackageEdits so it can be attempted again.
Log.Error("(error) - package edit blob update failed. Rolling back the DB transaction.");
Log.ErrorException("(exception", e);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
transaction.Rollback();
return;
}
try
{
transaction.Commit();
}
catch (Exception e)
{
// Commit changes to DB failed.
// Since our blob update wasn't part of the transaction (and doesn't AFAIK have a 'commit()' operator we can utilize for the type of blobs we are using)
// try, (single attempt) to roll back the blob update by restoring the previous snapshot.
Log.Error("(error) - package edit DB update failed. Trying to roll back the blob to its previous snapshot.");
Log.ErrorException("(exception", e);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
try
{
latestPackageBlob.StartCopyFromBlob(blobSnapshot);
}
catch (Exception e2)
{
// In this case it may not be the end of the world - the package metadata mismatches the edit now,
// but there's still an edit in the queue, waiting to be rerun and put everything back in synch.
Log.Error("(error) - rolling back the package blob to its previous snapshot failed.");
Log.ErrorException("(exception", e2);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
}
}
}
}
}
}
static void GeneratePackageDataFile(string[] args)
{
string connectionString = @"Data Source=(LocalDB)\v11.0;Initial Catalog=SearchTesting;Integrated Security=SSPI";
var context = new EntitiesContext(connectionString, readOnly: false);
var packageRepo = new EntityRepository <Package>(context);
var packages = packageRepo.GetAll()
.Where(p => p.IsLatest || p.IsLatestStable) // which implies that p.IsListed by the way!
.Include(p => p.PackageRegistration)
.Include(p => p.PackageRegistration.Owners)
.Include(p => p.SupportedFrameworks).ToList();
var ja = new JArray();
foreach (Package p in packages)
{
var pseudoPackage = new
{
Authors = p.Authors.Select(pa => pa.Name).ToList(),
p.Copyright,
p.Created,
Dependencies = p.Dependencies.Select(pd => new { pd.TargetFramework, pd.VersionSpec }).ToList(),
p.Description,
p.DownloadCount,
p.FlattenedAuthors,
p.FlattenedDependencies,
p.Hash,
p.HashAlgorithm,
p.IconUrl,
p.IsLatest,
p.IsLatestStable,
p.IsPrerelease,
p.Key,
p.Language,
p.LastUpdated,
p.LicenseUrl,
p.Listed,
p.MinClientVersion,
p.PackageFileSize,
PackageRegistration = new
{
DownloadCount = p.PackageRegistration.DownloadCount,
Id = p.PackageRegistration.Id,
Key = p.PackageRegistration.Key,
Owners = p.PackageRegistration.Owners.Select(o => o.Username).ToList(),
},
p.PackageRegistrationKey,
p.ProjectUrl,
p.Published,
p.ReleaseNotes,
p.RequiresLicenseAcceptance,
p.Summary,
SupportedFrameworks = p.SupportedFrameworks.Select(pf => pf.TargetFramework).ToList(),
p.Tags,
p.Title,
p.Version,
};
foreach (var author in p.Authors)
{
author.Package = null;
}
var jo = JObject.FromObject(pseudoPackage);
string json = jo.ToString();
ja.Add(jo);
}
File.WriteAllText(@"samplePackageData.json", ja.ToString());
}
public CharacterController(EntitiesContext db, IHostingEnvironment he, UserManager <IdentityUser> um)
{
_dbContext = db;
_hostingEnvironment = he;
_userManager = um;
}
public PlaceOrderDbAccess(EntitiesContext context)//#A
{
_context = context;
}
public IInterceptor[] SelectInterceptors(Type type, System.Reflection.MethodInfo method, IInterceptor[] interceptors)
{
return(EntitiesContext.SelectInterceptors(method, interceptors));
}
public BasketController(IRepositoriesFactory factory, EntitiesContext context, IEmailSender sender) : base(factory)
{
this.context = context;
this.sender = sender;
}
public async Task Update(SqlConnection connection, string fileName)
{
if (!File.Exists(fileName))
{
throw new ArgumentException($"File '{fileName}' doesn't exist");
}
using (var context = new EntitiesContext(connection, readOnly: false))
using (var cursor = new FileCursor(CursorFileName))
using (var logger = new Logger(ErrorsFileName))
{
var startTime = await cursor.Read();
logger.Log($"Starting database update - Cursor time: {startTime:u}");
var repository = new EntityRepository <Package>(context);
var packages = repository.GetAll().Include(p => p.PackageRegistration);
using (var csv = CreateCsvReader(fileName))
{
var counter = 0;
var lastCreatedDate = default(DateTime?);
var result = await TryReadMetadata(csv);
while (result.Success)
{
var metadata = result.Metadata;
if (metadata.Created >= startTime)
{
var package = packages.FirstOrDefault(p => p.PackageRegistration.Id == metadata.Id && p.NormalizedVersion == metadata.Version);
if (package != null)
{
UpdatePackage(package, metadata.Metadata, context);
logger.LogPackage(metadata.Id, metadata.Version, "Metadata updated.");
counter++;
if (!lastCreatedDate.HasValue || lastCreatedDate < package.Created)
{
lastCreatedDate = metadata.Created;
}
}
else
{
await logger.LogPackageError(metadata.Id, metadata.Version, "Could not find package in the database.");
}
}
if (counter >= UpdateBatchSize)
{
await CommitBatch(context, cursor, logger, metadata.Created);
counter = 0;
}
result = await TryReadMetadata(csv);
}
if (counter > 0)
{
await CommitBatch(context, cursor, logger, lastCreatedDate);
}
}
}
}
private void ProcessPackageEdits(int packageKey, IEnumerable<PackageEdit> editsToDelete)
{
// Create a fresh entities context so that we work in isolation
var entitiesContext = new EntitiesContext(ConnectionString.ConnectionString, readOnly: false);
// Get the most recent edit for this package
var edit = entitiesContext.Set<PackageEdit>()
.Where(pe => pe.PackageKey == packageKey && pe.TriedCount < 3)
.Include(pe => pe.Package)
.Include(pe => pe.Package.PackageRegistration)
.Include(pe => pe.User)
.OrderByDescending(pe => pe.Timestamp)
.First();
// List of Work to do:
// 1) Backup old blob, if the original has not been backed up yet
// 2) Downloads blob, create new NUPKG locally
// 3) Upload blob
// 4) Update the database
var blobClient = StorageAccount.CreateCloudBlobClient();
var packagesContainer = Util.GetPackagesBlobContainer(blobClient);
var latestPackageFileName = Util.GetPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version);
var originalPackageFileName = Util.GetBackupOfOriginalPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version);
var originalPackageBackupBlob = packagesContainer.GetBlockBlobReference(originalPackageFileName);
var latestPackageBlob = packagesContainer.GetBlockBlobReference(latestPackageFileName);
var edits = new List<Action<ManifestEdit>>
{
(m) => { m.Authors = edit.Authors; },
(m) => { m.Copyright = edit.Copyright; },
(m) => { m.Description = edit.Description; },
(m) => { m.IconUrl = edit.IconUrl; },
(m) => { m.LicenseUrl = edit.LicenseUrl; },
(m) => { m.ProjectUrl = edit.ProjectUrl; },
(m) => { m.ReleaseNotes = edit.ReleaseNotes; },
(m) => { m.RequireLicenseAcceptance = edit.RequiresLicenseAcceptance; },
(m) => { m.Summary = edit.Summary; },
(m) => { m.Title = edit.Title; },
(m) => { m.Tags = edit.Tags; },
};
Log.Info(
"Processing Edit Key={0}, PackageId={1}, Version={2}, User={3}",
edit.Key,
edit.Package.PackageRegistration.Id,
edit.Package.Version,
edit.User.Username);
if (!WhatIf)
{
edit.TriedCount += 1;
int nr = entitiesContext.SaveChanges();
if (nr != 1)
{
throw new Exception(
$"Something went terribly wrong, only one entity should be updated but actually {nr} entities were updated");
}
}
try
{
ArchiveOriginalPackageBlob(originalPackageBackupBlob, latestPackageBlob);
using (var readWriteStream = new MemoryStream())
{
// Download to memory
CloudBlockBlob downloadSourceBlob = WhatIf ? latestPackageBlob : originalPackageBackupBlob;
Log.Info("Downloading original package blob to memory {0}", downloadSourceBlob.Name);
downloadSourceBlob.DownloadToStream(readWriteStream);
// Rewrite in memory
Log.Info("Rewriting nupkg package in memory", downloadSourceBlob.Name);
NupkgRewriter.RewriteNupkgManifest(readWriteStream, edits);
// Get updated hash code, and file size
Log.Info("Computing updated hash code of memory stream");
var newPackageFileSize = readWriteStream.Length;
var hashAlgorithm = HashAlgorithm.Create("SHA512");
byte[] hashBytes = hashAlgorithm.ComputeHash(readWriteStream.GetBuffer());
var newHash = Convert.ToBase64String(hashBytes);
if (!WhatIf)
{
// Snapshot the blob
var blobSnapshot = latestPackageBlob.CreateSnapshot();
// Build up the changes in the entities context
edit.Apply(hashAlgorithm: "SHA512", hash: newHash, packageFileSize: newPackageFileSize);
foreach (var eachEdit in editsToDelete)
{
entitiesContext.DeleteOnCommit(eachEdit);
}
// Upload the blob before doing SaveChanges(). If blob update fails, we won't do SaveChanges() and the edit can be retried.
// If SaveChanges() fails we can undo the blob upload.
try
{
Log.Info("Uploading blob from memory {0}", latestPackageBlob.Name);
readWriteStream.Position = 0;
latestPackageBlob.UploadFromStream(readWriteStream);
}
catch (Exception e)
{
Log.Error("(error) - package edit blob update failed.");
Log.ErrorException("(exception)", e);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
throw; // To handler block that will record error in DB
}
try
{
// SaveChanges tries to commit changes to DB
entitiesContext.SaveChanges();
}
catch (Exception e)
{
// Commit changes to DB probably failed.
// Since our blob update wasn't part of the transaction (and doesn't AFAIK have a 'commit()' operator we can utilize for the type of blobs we are using)
// try, (single attempt) to roll back the blob update by restoring the previous snapshot.
Log.Error("(error) - package edit DB update failed. Trying to roll back the blob to its previous snapshot.");
Log.ErrorException("(exception)", e);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
try
{
latestPackageBlob.StartCopyFromBlob(blobSnapshot);
}
catch (Exception e2)
{
// If blob rollback fails it is not be the end of the world
// - the package metadata mismatches the edit now,
// but there should still an edit in the queue, waiting to be rerun and put everything back in synch.
Log.Error("(error) - rolling back the package blob to its previous snapshot failed.");
Log.ErrorException("(exception)", e2);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
}
throw; // To handler block that will record error in DB
}
}
}
}
catch (Exception e)
{
if (!WhatIf)
{
try
{
Log.Info("Storing the error on package edit with key {0}", edit.Key);
// Try to record the error into the PackageEdit database record
// so that we can actually diagnose failures.
// This must be done on a fresh context to ensure no conflicts.
var errorContext = new EntitiesContext(ConnectionString.ConnectionString, readOnly: false);
var errorEdit = errorContext.Set<PackageEdit>().Where(pe => pe.Key == edit.Key).FirstOrDefault();
if (errorEdit != null)
{
errorEdit.LastError = $"{e.GetType()} : {e}";
errorContext.SaveChanges();
}
else
{
Log.Info("The package edit with key {0} couldn't be found. It was likely canceled and deleted.", edit.Key);
}
}
catch (Exception errorException)
{
Log.ErrorException("(error) - couldn't save the last error on the edit that was being applied.", errorException);
}
}
}
}
public BaseRepository(EntitiesContext context)
{
this.Context = context;
}
