/// <summary> /// Initializes a new instance of the <see cref="LeadQuoteRepository"/> class. /// </summary> /// <param name="repositoryContext">The repository context.</param> /// <param name="mapper">The mapper.</param> /// <param name="sortHelper">The sort helper.</param> /// <param name="dataShaper">The data shaper.</param> public LeadCountRepository( LeadContext repositoryContext, IMapper mapper) : base(repositoryContext) { this.mapper = mapper; }
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline. public void Configure(IApplicationBuilder app, IWebHostEnvironment env, LeadContext context) { if (env.IsDevelopment()) { app.UseDeveloperExceptionPage(); } app.UseSwagger(c => { c.SerializeAsV2 = true; }); app.UseSwaggerUI(c => { c.SwaggerEndpoint("/swagger/v1/swagger.json", "My API V1"); c.RoutePrefix = string.Empty; }); app.UseRouting(); app.UseAuthorization(); app.UseMiddleware <HttpExceptionMiddleware>(); app.UseEndpoints(endpoints => { endpoints.MapControllers(); }); if (!env.IsEnvironment("integration")) { context.Database.EnsureCreated(); } }
/// <summary> /// Initializes a new instance of the <see cref="LeadDataCollectionRepository"/> class. /// </summary> /// <param name="repositoryContext">The repository context.</param> /// <param name="mapper">The mapper.</param> public LeadDataCollectionRepository( LeadContext repositoryContext, IMapper mapper, ISortHelper <LeadDataCollectionResponse> sortHelper, IDataShaper <LeadDataCollectionResponse> dataShaper) : base(repositoryContext) { this.mapper = mapper; this.sortHelper = sortHelper; this.dataShaper = dataShaper; }
/// <summary> /// Initializes a new instance of the <see cref="MultiDetailRepository" /> class. /// </summary> /// <param name="repositoryContext">The repository context.</param> /// <param name="mapper">The mapper.</param> /// <param name="sortHelper">The sort helper.</param> /// <param name="dataShaper">The data shaper.</param> public MultiDetailRepository( LeadContext repositoryContext, IMapper mapper, ISortHelper <MultiDetailResponse> sortHelper, IDataShaper <MultiDetailResponse> dataShaper) : base(repositoryContext) { this.mapper = mapper; this.sortHelper = sortHelper; this.dataShaper = dataShaper; }
private async Task DataSeeds(LeadContext context) { context.Contractors.Add(new Api.DataModel.Models.Contractor { ContractorID = 1, CompanyName = "Test 1" }); context.Contractors.Add(new Api.DataModel.Models.Contractor { ContractorID = 2, CompanyName = "Test 2" }); context.Contractors.Add(new Api.DataModel.Models.Contractor { ContractorID = 3, CompanyName = "Test 3" }); context.Contractors.Add(new Api.DataModel.Models.Contractor { ContractorID = 4, CompanyName = "Test 4" }); context.Contractors.Add(new Api.DataModel.Models.Contractor { ContractorID = 5, CompanyName = "Test 5" }); await context.SaveChangesAsync(); }
static void Main(string[] args) { Console.WriteLine("Service Started"); var container = RegisterContainers(); Console.WriteLine("Preparing Leads to Insert"); var leads = new List <Lead>() { new Lead(Guid.NewGuid(), "111111111", "Customer1", 32, "11 5566-7788"), new Lead(Guid.NewGuid(), "222222222", "Customer1", 43, "11 4321-7766"), new Lead(Guid.NewGuid(), "333333333", "Customer1", 55, "11 98789-1243"), }; // Inserting in Redis var leadContext = new LeadContext(); leadContext.InsertLeads(leads); Console.WriteLine("Leads Inserted in Redis"); // Get value by key in Redis var firstLeadInserted = leadContext.GetLeadByKey($"id-{leads.First().Id}"); Console.WriteLine($"First lead Inserted: {firstLeadInserted}"); // Get values by keys in Redis var allLeads = leadContext.GetLeadsByKeys(leads.Select(s => $"id-{s.Id}").ToList()); Console.WriteLine("All Leads inserted:"); allLeads.ForEach(s => Console.WriteLine(s)); AppDomain.CurrentDomain.ProcessExit += (o, e) => { Console.WriteLine("Terminating..."); autoResetEvent.Set(); }; autoResetEvent.WaitOne(); }
/// <summary> /// Initializes a new instance of the <see cref="RepositoryWrapper"/> class. /// </summary> /// <param name="repositoryContext">The repository context.</param> /// <param name="mapper">The mapper.</param> /// <param name="multicodeSortHelper">The multi code sort helper.</param> /// <param name="multicodeDataShaper">The multi code data shaper.</param> /// <param name="multidetailSortHelper">The multi detail sort helper.</param> /// <param name="multidetailDataShaper">The multi detail data shaper.</param> /// <param name="leadDataCollectionSortHelper">The lead Data Collection sort helper.</param> /// <param name="leadDataCollectionDataShaper">The lead Data Collection data shaper.</param> /// <param name="leadQuoteSortHelper">The lead quote sort helper.</param> /// <param name="leadQuoteDataShaper">The lead quote data shaper.</param> /// <param name="leadValidationSortHelper">The lead Validation sort helper.</param> /// <param name="leadValidationDataShaper">The lead Validation data shaper.</param> /// <param name="leadAssignSortHelper">The lead Assign sort helper.</param> /// <param name="leadAssignDataShaper">The lead Assign data shaper.</param> public RepositoryWrapper( LeadContext repositoryContext, IMapper mapper, ISortHelper <MultiCodeResponse> multicodeSortHelper, IDataShaper <MultiCodeResponse> multicodeDataShaper, ISortHelper <MultiDetailResponse> multidetailSortHelper, IDataShaper <MultiDetailResponse> multidetailDataShaper, ISortHelper <LeadDataCollectionResponse> leadDataCollectionSortHelper, IDataShaper <LeadDataCollectionResponse> leadDataCollectionDataShaper, ISortHelper <LeadQuoteResponse> leadQuoteSortHelper, IDataShaper <LeadQuoteResponse> leadQuoteDataShaper, ISortHelper <LeadValidationResponse> leadValidationSortHelper, IDataShaper <LeadValidationResponse> leadValidationDataShaper, ISortHelper <LeadAssignResponse> leadAssignSortHelper, IDataShaper <LeadAssignResponse> leadAssignDataShaper, ISortHelper <LeadStatusResponse> leadStatusSortHelper, IDataShaper <LeadStatusResponse> leadStatusDataShaper ) { this.repositoryContext = repositoryContext; this.mapper = mapper; this.multicodeSortHelper = multicodeSortHelper; this.multicodeDataShaper = multicodeDataShaper; this.multidetailSortHelper = multidetailSortHelper; this.multidetailDataShaper = multidetailDataShaper; this.leadDataCollectionSortHelper = leadDataCollectionSortHelper; this.leadDataCollectionDataShaper = leadDataCollectionDataShaper; this.leadQuoteSortHelper = leadQuoteSortHelper; this.leadQuoteDataShaper = leadQuoteDataShaper; this.leadValidationSortHelper = leadValidationSortHelper; this.leadValidationDataShaper = leadValidationDataShaper; this.leadAssignSortHelper = leadAssignSortHelper; this.leadAssignDataShaper = leadAssignDataShaper; this.leadStatusSortHelper = leadStatusSortHelper; this.leadStatusDataShaper = leadStatusDataShaper; }
/// <summary> /// A simple function that takes a string and does a ToUpper /// </summary> /// <param name="context"></param> /// <returns></returns> public void FunctionHandler(ILambdaContext context) { try { Stopwatch sw = new Stopwatch(); sw.Start(); Logger.Init(TRANSFORMER_NAME); Logger.Log("Started"); // todo: switch to S3 var filename = "C:\\temp\\country.csv"; Logger.Log($"Creating DB contexts."); var silverContext = new SilverContext(); var leadContext = new LeadContext(); Dictionary <string, Country> countries = silverContext.Countries.ToDictionary(x => x.Code, x => x); if (countries.Count == 0) { throw new InvalidOperationException("Countries are not loaded"); } List <string> leis; leis = leadContext.GleifEntity.Select(x => x.Lei) //.Take(1000) .ToList(); //using (StreamWriter sw = new StreamWriter("C:\\temp\\leis.dat")) //{ // foreach (var s in leis) // { // sw.WriteLine(s); // } //} //leis = new List<string>(1700000); //using (StreamReader sr = new StreamReader("C:\\temp\\leis.dat")) //{ // while(!sr.EndOfStream) // { // leis.Add(sr.ReadLine()); // } //} var chunks = new List <List <string> >(); int size = 10000; for (int i = 0; i < leis.Count; i += size) { chunks.Add(leis.GetRange(i, Math.Min(size, leis.Count - i))); } chunks .AsParallel().WithDegreeOfParallelism(24).ForAll( //.Skip(20).ToList() //.ForEach( x => LoadChunk(x, chunks.IndexOf(x), countries)); sw.Stop(); Logger.Log($"Finished successfully in {sw.ElapsedMilliseconds} ms"); } catch (Exception ex) { Logger.Log($"ERROR: Unhandled exception: {ex.ToString()}"); } }
public void LoadChunk(List <string> chunk, int index, Dictionary <string, Country> countries) { try { Logger.Log($"Loading chunk {index}"); var leadContext = new LeadContext(); //var factory = LoggerFactory.Create(builder => //{ // builder.AddProvider(new TraceLoggerProvider()); //}); var silverContext = new SilverContext(); silverContext.ChangeTracker.LazyLoadingEnabled = true; silverContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; silverContext.ChangeTracker.AutoDetectChangesEnabled = false; List <Company> companiesToSave = new List <Company>(); List <CompanyName> namesToSave = new List <CompanyName>(); foreach (var entity in leadContext.GleifEntity.AsNoTracking().Where(x => chunk.Contains(x.Lei)).Include(x => x.GleifAddresses).Include(x => x.GleifEntityNames)) { //var targetEntity = silverContext.Company.Where(x => x.Lei == entity.Lei).FirstOrDefault(); //if (targetEntity == null) //{ if (!countries.ContainsKey(entity.LegalJurisdiction.Split('-')[0])) { // we have no country data for it continue; } var targetEntity = new Company(); companiesToSave.Add(targetEntity); //} var names = entity.GleifEntityNames; var legalName = names.Where(x => x.Type == "LEGAL").First(); //legal name should exist var otherNames = names.Where(x => x.Type != "LEGAL" && x.Type != "PREVIOUS_LEGAL_NAME" && !string.IsNullOrEmpty(x.Name)).ToList(); targetEntity.LegalName = legalName.Name; foreach (var name in otherNames) { var otherName = new CompanyName() { Company = targetEntity, Name = name.Name, Type = name.Type, }; targetEntity.CompanyNames.Add(otherName); namesToSave.Add(otherName); } targetEntity.Status = entity.Status; targetEntity.Lei = entity.Lei; var legalAddress = entity.GleifAddresses.Where(x => x.Type == "LEGAL").First(); if (targetEntity.Legal == null) { targetEntity.Legal = new Address(); silverContext.Add(targetEntity.Legal); //targetEntity.Legal.CompanyLegal.Add(targetEntity); } Address targetLegalAddress = targetEntity.Legal; targetLegalAddress.City = legalAddress.City; if (countries.ContainsKey(legalAddress.Country)) { targetLegalAddress.Country = countries[legalAddress.Country]; targetLegalAddress.CountryId = targetLegalAddress.Country.Id; } targetLegalAddress.AddressNumber = legalAddress.Addressnumber; targetLegalAddress.PostalCode = legalAddress.Postalcode; targetLegalAddress.Region = legalAddress.Region; targetLegalAddress.AddressLine = legalAddress.Firstaddressline + (string.IsNullOrEmpty(legalAddress.Additionaladdressline1) ? string.Empty : " " + legalAddress.Additionaladdressline1) + (string.IsNullOrEmpty(legalAddress.Additionaladdressline2) ? string.Empty : " " + legalAddress.Additionaladdressline2) + (string.IsNullOrEmpty(legalAddress.Additionaladdressline3) ? string.Empty : " " + legalAddress.Additionaladdressline3); targetLegalAddress.AddressLine = targetLegalAddress.AddressLine ?? string.Empty; var hqAddress = entity.GleifAddresses.Where(x => x.Type == "HQ").First(); if (targetEntity.Hq == null) { targetEntity.Hq = new Address(); silverContext.Add(targetEntity.Hq); //targetEntity.Hq.CompanyHq.Add(targetEntity); } Address targetHqAddress = targetEntity.Hq; targetHqAddress.City = hqAddress.City; if (countries.ContainsKey(hqAddress.Country)) { targetHqAddress.Country = countries[hqAddress.Country]; targetHqAddress.CountryId = targetHqAddress.Country.Id; } targetHqAddress.AddressNumber = hqAddress.Addressnumber; targetHqAddress.PostalCode = hqAddress.Postalcode; targetHqAddress.Region = hqAddress.Region; targetHqAddress.AddressLine = hqAddress.Firstaddressline + (string.IsNullOrEmpty(hqAddress.Additionaladdressline1) ? string.Empty : " " + hqAddress.Additionaladdressline1) + (string.IsNullOrEmpty(hqAddress.Additionaladdressline2) ? string.Empty : " " + hqAddress.Additionaladdressline2) + (string.IsNullOrEmpty(hqAddress.Additionaladdressline3) ? string.Empty : " " + hqAddress.Additionaladdressline3); targetHqAddress.AddressLine = targetHqAddress.AddressLine ?? string.Empty; targetEntity.CompanyCountries.Add( new CompanyCountry() { IsPrimary = true, LegalJurisdiction = true, Company = targetEntity, CountryId = countries[entity.LegalJurisdiction.Split('-')[0]].Id }); } silverContext.SaveChanges(); foreach (var company in companiesToSave) { silverContext.Add(company); } foreach (var name in namesToSave) { silverContext.Add(name); } silverContext.SaveChanges(); silverContext.Dispose(); leadContext.Dispose(); Logger.Log($"Chunk {index} loaded"); } catch (Exception ex) { Logger.Log($"ERROR: Unhandled exception: {ex.ToString()}"); throw; } }
public LeadController(LeadContext leadContext, IMessageSession messageSession) { _context = leadContext; _messageSession = messageSession; }
/// <summary> /// Initializes a new instance of the <see cref="RepositoryBase{T}"/> class. /// </summary> /// <param name="repositoryContext">The repository context.</param> public RepositoryBase(LeadContext repositoryContext) { RepositoryContext = repositoryContext; }
/// <summary> /// A simple function that takes a string and does a ToUpper /// </summary> /// <param name="input"></param> /// <param name="context"></param> /// <returns></returns> public void FunctionHandler(ILambdaContext context) { try { Stopwatch sw = new Stopwatch(); sw.Start(); Logger.Init(CRAWLER_NAME); Logger.Log("Started"); string fileUrl = string.Empty; Logger.Log("Asking GLEIF service for file URL"); WebRequest fileAddressRequest = WebRequest.Create(GLEIF_SERVICE); using (var stream = fileAddressRequest.GetResponse().GetResponseStream()) { using (var sr = new StreamReader(stream)) { string fileAddressStringResponse = sr.ReadToEnd(); Logger.Log("Recieved GLEIF service response for file URL"); var fileAddressResponse = JsonConvert.DeserializeObject <ServiceResponse>(fileAddressStringResponse); Logger.Log("Successfully parsed GLEIF service response for file URL"); fileUrl = fileAddressResponse.Data.First().Lei2.FullFile.Csv.Url; } } if (string.IsNullOrEmpty(fileUrl)) { throw new InvalidOperationException("GLEIF file URL was not obtained properly!"); } else { Logger.Log($"GLEIF file URL is {fileUrl}"); } //Log("Downloading GLEIF file to S3"); string s3FileKey = $"GLEIF/{DateTime.Now.ToString("yyyyMMdd")}.csv"; var s3Client = new AmazonS3Client(RegionEndpoint.USEast2); Logger.Log("Starting reading Golden Copy file contents"); WebRequest fileRequest = WebRequest.Create(fileUrl); var fileResponse = fileRequest.GetResponse(); using (Stream pageStream = fileResponse.GetResponseStream()) //using (Stream pageStream = new FileStream("C:\\temp\\20201102.zip", FileMode.Open, FileAccess.Read)) { var archive = new ZipArchive(pageStream, ZipArchiveMode.Read); var entry = archive.Entries.First(); using (var zippedFileStream = entry.Open()) { using (var fileReader = new StreamReader(zippedFileStream)) { CsvReader csvReader = ConfigureCsvReader(fileReader); Logger.Log("Reading infrastructure for Golden Copy file contents initialized."); Logger.Log("Configuring DB context."); //using (var sw = new StreamWriter("C:\\temp\\allnames.dat")) //{ // while (csvReader.Read()) // { // sw.WriteLine(csvReader.GetRecord<LegalEntityName>().Name.ToUpperInvariant()); // } //} LeadContext dbContext = new LeadContext(); dbContext.ChangeTracker.AutoDetectChangesEnabled = false; dbContext.ChangeTracker.LazyLoadingEnabled = true; dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; Logger.Log("Clearing Lead GLEIF tables."); //dbContext.Database.ExecuteSqlRaw("TRUNCATE TABLE gleif_validation_authority"); dbContext.Database.ExecuteSqlRaw("TRUNCATE TABLE gleif_address"); dbContext.Database.ExecuteSqlRaw("DELETE FROM gleif_entity_name"); dbContext.Database.ExecuteSqlRaw("DELETE FROM gleif_entity"); dbContext.Dispose(); Logger.Log("Lead GLEIF tables cleared."); for (int i = 0; i < THREAD_COUNT; i++) { new Thread(x => UploadChunks(csvReader)).Start(); } for (int i = 0; i < THREAD_COUNT; i++) { total.WaitOne(); } Logger.Log($"Processing finished."); } } } sw.Stop(); Logger.Log($"Finished successfully in {sw.ElapsedMilliseconds} ms"); } catch (Exception ex) { Logger.Log($"ERROR: Unhandled exception: {ex.ToString()}"); } }
private void UploadChunks(CsvReader csvReader) { while (true) { LeadContext localDbContext = new LeadContext(); localDbContext.ChangeTracker.AutoDetectChangesEnabled = false; localDbContext.ChangeTracker.LazyLoadingEnabled = true; localDbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; bool canRead = true; lock (this) { int counter = 0; while ((canRead = csvReader.Read()) && counter < DB_FLUSH_THRESHOLD) { var entity = csvReader.GetRecord <GleifEntity>(); new List <GleifEntityName>() { csvReader.GetRecord <LegalEntityName>(), csvReader.GetRecord <OtherEntityName1>(), csvReader.GetRecord <OtherEntityName2>(), csvReader.GetRecord <OtherEntityName3>(), csvReader.GetRecord <OtherEntityName4>(), csvReader.GetRecord <OtherEntityName5>(), csvReader.GetRecord <TransliteratedOtherEntityName1>(), csvReader.GetRecord <TransliteratedOtherEntityName2>(), csvReader.GetRecord <TransliteratedOtherEntityName3>(), csvReader.GetRecord <TransliteratedOtherEntityName4>(), csvReader.GetRecord <TransliteratedOtherEntityName5>() } .Where(x => !string.IsNullOrEmpty(x.Name)).ToList() .ForEach(x => { entity.GleifEntityNames.Add(x); x.Entity = entity; localDbContext.GleifEntityName.Add(x); }); new List <GleifAddress>() { csvReader.GetRecord <LegalAddress>(), csvReader.GetRecord <HeadquartersAddress>(), /*csvReader.GetRecord<OtherAddress1>(), * csvReader.GetRecord<OtherAddress2>(), * csvReader.GetRecord<OtherAddress3>(), * csvReader.GetRecord<OtherAddress4>(), * csvReader.GetRecord<OtherAddress5>(),*/ /*csvReader.GetRecord<TransliteratedOtherAddress1>(), * csvReader.GetRecord<TransliteratedOtherAddress2>(), * csvReader.GetRecord<TransliteratedOtherAddress3>(), * csvReader.GetRecord<TransliteratedOtherAddress4>(), * csvReader.GetRecord<TransliteratedOtherAddress5>(),*/ } .Where(x => !string.IsNullOrEmpty(x.Firstaddressline)).ToList() .ForEach(x => { entity.GleifAddresses.Add(x); x.Entity = entity; localDbContext.GleifAddress.Add(x); } ); /* * new List<GleifValidationAuthority>() * { * csvReader.GetRecord<ValidationAuthority>(), * csvReader.GetRecord<OtherValidationAuthority1>(), * csvReader.GetRecord<OtherValidationAuthority2>(), * csvReader.GetRecord<OtherValidationAuthority3>(), * csvReader.GetRecord<OtherValidationAuthority4>(), * csvReader.GetRecord<OtherValidationAuthority5>(), * } * .Where(x => !string.IsNullOrEmpty(x.ValidationAuthorityId)).ToList() * .ForEach(x => * { * entity.GleifValidationAuthorities.Add(x); * localDbContext.GleifValidationAuthority.Add(x); * } * ); */ localDbContext.GleifEntity.Add(entity); counter++; } } localDbContext.SaveChanges(); localDbContext.Dispose(); if (!canRead) { break; } } total.Release(1); }