/// <summary>
/// Initializes a new instance of the <see cref="LeadQuoteRepository"/> class.
/// </summary>
/// <param name="repositoryContext">The repository context.</param>
/// <param name="mapper">The mapper.</param>
/// <param name="sortHelper">The sort helper.</param>
/// <param name="dataShaper">The data shaper.</param>
public LeadCountRepository(
LeadContext repositoryContext,
IMapper mapper)
: base(repositoryContext)
{
this.mapper = mapper;
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IWebHostEnvironment env, LeadContext context)
{
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
}
app.UseSwagger(c =>
{
c.SerializeAsV2 = true;
});
app.UseSwaggerUI(c =>
{
c.SwaggerEndpoint("/swagger/v1/swagger.json", "My API V1");
c.RoutePrefix = string.Empty;
});
app.UseRouting();
app.UseAuthorization();
app.UseMiddleware <HttpExceptionMiddleware>();
app.UseEndpoints(endpoints =>
{
endpoints.MapControllers();
});
if (!env.IsEnvironment("integration"))
{
context.Database.EnsureCreated();
}
}
/// <summary>
/// Initializes a new instance of the <see cref="LeadDataCollectionRepository"/> class.
/// </summary>
/// <param name="repositoryContext">The repository context.</param>
/// <param name="mapper">The mapper.</param>
public LeadDataCollectionRepository(
LeadContext repositoryContext,
IMapper mapper,
ISortHelper <LeadDataCollectionResponse> sortHelper,
IDataShaper <LeadDataCollectionResponse> dataShaper)
: base(repositoryContext)
{
this.mapper = mapper;
this.sortHelper = sortHelper;
this.dataShaper = dataShaper;
}
/// <summary>
/// Initializes a new instance of the <see cref="MultiDetailRepository" /> class.
/// </summary>
/// <param name="repositoryContext">The repository context.</param>
/// <param name="mapper">The mapper.</param>
/// <param name="sortHelper">The sort helper.</param>
/// <param name="dataShaper">The data shaper.</param>
public MultiDetailRepository(
LeadContext repositoryContext,
IMapper mapper,
ISortHelper <MultiDetailResponse> sortHelper,
IDataShaper <MultiDetailResponse> dataShaper)
: base(repositoryContext)
{
this.mapper = mapper;
this.sortHelper = sortHelper;
this.dataShaper = dataShaper;
}
private async Task DataSeeds(LeadContext context)
{
context.Contractors.Add(new Api.DataModel.Models.Contractor {
ContractorID = 1, CompanyName = "Test 1"
});
context.Contractors.Add(new Api.DataModel.Models.Contractor {
ContractorID = 2, CompanyName = "Test 2"
});
context.Contractors.Add(new Api.DataModel.Models.Contractor {
ContractorID = 3, CompanyName = "Test 3"
});
context.Contractors.Add(new Api.DataModel.Models.Contractor {
ContractorID = 4, CompanyName = "Test 4"
});
context.Contractors.Add(new Api.DataModel.Models.Contractor {
ContractorID = 5, CompanyName = "Test 5"
});
await context.SaveChangesAsync();
}
static void Main(string[] args)
{
Console.WriteLine("Service Started");
var container = RegisterContainers();
Console.WriteLine("Preparing Leads to Insert");
var leads = new List <Lead>()
{
new Lead(Guid.NewGuid(), "111111111", "Customer1", 32, "11 5566-7788"),
new Lead(Guid.NewGuid(), "222222222", "Customer1", 43, "11 4321-7766"),
new Lead(Guid.NewGuid(), "333333333", "Customer1", 55, "11 98789-1243"),
};
// Inserting in Redis
var leadContext = new LeadContext();
leadContext.InsertLeads(leads);
Console.WriteLine("Leads Inserted in Redis");
// Get value by key in Redis
var firstLeadInserted = leadContext.GetLeadByKey($"id-{leads.First().Id}");
Console.WriteLine($"First lead Inserted: {firstLeadInserted}");
// Get values by keys in Redis
var allLeads = leadContext.GetLeadsByKeys(leads.Select(s => $"id-{s.Id}").ToList());
Console.WriteLine("All Leads inserted:");
allLeads.ForEach(s => Console.WriteLine(s));
AppDomain.CurrentDomain.ProcessExit += (o, e) =>
{
Console.WriteLine("Terminating...");
autoResetEvent.Set();
};
autoResetEvent.WaitOne();
}
/// <summary>
/// Initializes a new instance of the <see cref="RepositoryWrapper"/> class.
/// </summary>
/// <param name="repositoryContext">The repository context.</param>
/// <param name="mapper">The mapper.</param>
/// <param name="multicodeSortHelper">The multi code sort helper.</param>
/// <param name="multicodeDataShaper">The multi code data shaper.</param>
/// <param name="multidetailSortHelper">The multi detail sort helper.</param>
/// <param name="multidetailDataShaper">The multi detail data shaper.</param>
/// <param name="leadDataCollectionSortHelper">The lead Data Collection sort helper.</param>
/// <param name="leadDataCollectionDataShaper">The lead Data Collection data shaper.</param>
/// <param name="leadQuoteSortHelper">The lead quote sort helper.</param>
/// <param name="leadQuoteDataShaper">The lead quote data shaper.</param>
/// <param name="leadValidationSortHelper">The lead Validation sort helper.</param>
/// <param name="leadValidationDataShaper">The lead Validation data shaper.</param>
/// <param name="leadAssignSortHelper">The lead Assign sort helper.</param>
/// <param name="leadAssignDataShaper">The lead Assign data shaper.</param>
public RepositoryWrapper(
LeadContext repositoryContext,
IMapper mapper,
ISortHelper <MultiCodeResponse> multicodeSortHelper,
IDataShaper <MultiCodeResponse> multicodeDataShaper,
ISortHelper <MultiDetailResponse> multidetailSortHelper,
IDataShaper <MultiDetailResponse> multidetailDataShaper,
ISortHelper <LeadDataCollectionResponse> leadDataCollectionSortHelper,
IDataShaper <LeadDataCollectionResponse> leadDataCollectionDataShaper,
ISortHelper <LeadQuoteResponse> leadQuoteSortHelper,
IDataShaper <LeadQuoteResponse> leadQuoteDataShaper,
ISortHelper <LeadValidationResponse> leadValidationSortHelper,
IDataShaper <LeadValidationResponse> leadValidationDataShaper,
ISortHelper <LeadAssignResponse> leadAssignSortHelper,
IDataShaper <LeadAssignResponse> leadAssignDataShaper,
ISortHelper <LeadStatusResponse> leadStatusSortHelper,
IDataShaper <LeadStatusResponse> leadStatusDataShaper
)
{
this.repositoryContext = repositoryContext;
this.mapper = mapper;
this.multicodeSortHelper = multicodeSortHelper;
this.multicodeDataShaper = multicodeDataShaper;
this.multidetailSortHelper = multidetailSortHelper;
this.multidetailDataShaper = multidetailDataShaper;
this.leadDataCollectionSortHelper = leadDataCollectionSortHelper;
this.leadDataCollectionDataShaper = leadDataCollectionDataShaper;
this.leadQuoteSortHelper = leadQuoteSortHelper;
this.leadQuoteDataShaper = leadQuoteDataShaper;
this.leadValidationSortHelper = leadValidationSortHelper;
this.leadValidationDataShaper = leadValidationDataShaper;
this.leadAssignSortHelper = leadAssignSortHelper;
this.leadAssignDataShaper = leadAssignDataShaper;
this.leadStatusSortHelper = leadStatusSortHelper;
this.leadStatusDataShaper = leadStatusDataShaper;
}
/// <summary>
/// A simple function that takes a string and does a ToUpper
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public void FunctionHandler(ILambdaContext context)
{
try
{
Stopwatch sw = new Stopwatch();
sw.Start();
Logger.Init(TRANSFORMER_NAME);
Logger.Log("Started");
// todo: switch to S3
var filename = "C:\\temp\\country.csv";
Logger.Log($"Creating DB contexts.");
var silverContext = new SilverContext();
var leadContext = new LeadContext();
Dictionary <string, Country> countries = silverContext.Countries.ToDictionary(x => x.Code, x => x);
if (countries.Count == 0)
{
throw new InvalidOperationException("Countries are not loaded");
}
List <string> leis;
leis = leadContext.GleifEntity.Select(x => x.Lei)
//.Take(1000)
.ToList();
//using (StreamWriter sw = new StreamWriter("C:\\temp\\leis.dat"))
//{
// foreach (var s in leis)
// {
// sw.WriteLine(s);
// }
//}
//leis = new List<string>(1700000);
//using (StreamReader sr = new StreamReader("C:\\temp\\leis.dat"))
//{
// while(!sr.EndOfStream)
// {
// leis.Add(sr.ReadLine());
// }
//}
var chunks = new List <List <string> >();
int size = 10000;
for (int i = 0; i < leis.Count; i += size)
{
chunks.Add(leis.GetRange(i, Math.Min(size, leis.Count - i)));
}
chunks
.AsParallel().WithDegreeOfParallelism(24).ForAll(
//.Skip(20).ToList()
//.ForEach(
x => LoadChunk(x, chunks.IndexOf(x), countries));
sw.Stop();
Logger.Log($"Finished successfully in {sw.ElapsedMilliseconds} ms");
}
catch (Exception ex)
{
Logger.Log($"ERROR: Unhandled exception: {ex.ToString()}");
}
}
public void LoadChunk(List <string> chunk, int index, Dictionary <string, Country> countries)
{
try
{
Logger.Log($"Loading chunk {index}");
var leadContext = new LeadContext();
//var factory = LoggerFactory.Create(builder =>
//{
// builder.AddProvider(new TraceLoggerProvider());
//});
var silverContext = new SilverContext();
silverContext.ChangeTracker.LazyLoadingEnabled = true;
silverContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking;
silverContext.ChangeTracker.AutoDetectChangesEnabled = false;
List <Company> companiesToSave = new List <Company>();
List <CompanyName> namesToSave = new List <CompanyName>();
foreach (var entity in leadContext.GleifEntity.AsNoTracking().Where(x => chunk.Contains(x.Lei)).Include(x => x.GleifAddresses).Include(x => x.GleifEntityNames))
{
//var targetEntity = silverContext.Company.Where(x => x.Lei == entity.Lei).FirstOrDefault();
//if (targetEntity == null)
//{
if (!countries.ContainsKey(entity.LegalJurisdiction.Split('-')[0]))
{
// we have no country data for it
continue;
}
var targetEntity = new Company();
companiesToSave.Add(targetEntity);
//}
var names = entity.GleifEntityNames;
var legalName = names.Where(x => x.Type == "LEGAL").First(); //legal name should exist
var otherNames = names.Where(x => x.Type != "LEGAL" && x.Type != "PREVIOUS_LEGAL_NAME" && !string.IsNullOrEmpty(x.Name)).ToList();
targetEntity.LegalName = legalName.Name;
foreach (var name in otherNames)
{
var otherName = new CompanyName()
{
Company = targetEntity,
Name = name.Name,
Type = name.Type,
};
targetEntity.CompanyNames.Add(otherName);
namesToSave.Add(otherName);
}
targetEntity.Status = entity.Status;
targetEntity.Lei = entity.Lei;
var legalAddress = entity.GleifAddresses.Where(x => x.Type == "LEGAL").First();
if (targetEntity.Legal == null)
{
targetEntity.Legal = new Address();
silverContext.Add(targetEntity.Legal);
//targetEntity.Legal.CompanyLegal.Add(targetEntity);
}
Address targetLegalAddress = targetEntity.Legal;
targetLegalAddress.City = legalAddress.City;
if (countries.ContainsKey(legalAddress.Country))
{
targetLegalAddress.Country = countries[legalAddress.Country];
targetLegalAddress.CountryId = targetLegalAddress.Country.Id;
}
targetLegalAddress.AddressNumber = legalAddress.Addressnumber;
targetLegalAddress.PostalCode = legalAddress.Postalcode;
targetLegalAddress.Region = legalAddress.Region;
targetLegalAddress.AddressLine = legalAddress.Firstaddressline
+ (string.IsNullOrEmpty(legalAddress.Additionaladdressline1) ? string.Empty : " " + legalAddress.Additionaladdressline1)
+ (string.IsNullOrEmpty(legalAddress.Additionaladdressline2) ? string.Empty : " " + legalAddress.Additionaladdressline2)
+ (string.IsNullOrEmpty(legalAddress.Additionaladdressline3) ? string.Empty : " " + legalAddress.Additionaladdressline3);
targetLegalAddress.AddressLine = targetLegalAddress.AddressLine ?? string.Empty;
var hqAddress = entity.GleifAddresses.Where(x => x.Type == "HQ").First();
if (targetEntity.Hq == null)
{
targetEntity.Hq = new Address();
silverContext.Add(targetEntity.Hq);
//targetEntity.Hq.CompanyHq.Add(targetEntity);
}
Address targetHqAddress = targetEntity.Hq;
targetHqAddress.City = hqAddress.City;
if (countries.ContainsKey(hqAddress.Country))
{
targetHqAddress.Country = countries[hqAddress.Country];
targetHqAddress.CountryId = targetHqAddress.Country.Id;
}
targetHqAddress.AddressNumber = hqAddress.Addressnumber;
targetHqAddress.PostalCode = hqAddress.Postalcode;
targetHqAddress.Region = hqAddress.Region;
targetHqAddress.AddressLine = hqAddress.Firstaddressline
+ (string.IsNullOrEmpty(hqAddress.Additionaladdressline1) ? string.Empty : " " + hqAddress.Additionaladdressline1)
+ (string.IsNullOrEmpty(hqAddress.Additionaladdressline2) ? string.Empty : " " + hqAddress.Additionaladdressline2)
+ (string.IsNullOrEmpty(hqAddress.Additionaladdressline3) ? string.Empty : " " + hqAddress.Additionaladdressline3);
targetHqAddress.AddressLine = targetHqAddress.AddressLine ?? string.Empty;
targetEntity.CompanyCountries.Add(
new CompanyCountry()
{
IsPrimary = true,
LegalJurisdiction = true,
Company = targetEntity,
CountryId = countries[entity.LegalJurisdiction.Split('-')[0]].Id
});
}
silverContext.SaveChanges();
foreach (var company in companiesToSave)
{
silverContext.Add(company);
}
foreach (var name in namesToSave)
{
silverContext.Add(name);
}
silverContext.SaveChanges();
silverContext.Dispose();
leadContext.Dispose();
Logger.Log($"Chunk {index} loaded");
}
catch (Exception ex)
{
Logger.Log($"ERROR: Unhandled exception: {ex.ToString()}");
throw;
}
}
public LeadController(LeadContext leadContext, IMessageSession messageSession)
{
_context = leadContext;
_messageSession = messageSession;
}
/// <summary>
/// Initializes a new instance of the <see cref="RepositoryBase{T}"/> class.
/// </summary>
/// <param name="repositoryContext">The repository context.</param>
public RepositoryBase(LeadContext repositoryContext)
{
RepositoryContext = repositoryContext;
}
/// <summary>
/// A simple function that takes a string and does a ToUpper
/// </summary>
/// <param name="input"></param>
/// <param name="context"></param>
/// <returns></returns>
public void FunctionHandler(ILambdaContext context)
{
try
{
Stopwatch sw = new Stopwatch();
sw.Start();
Logger.Init(CRAWLER_NAME);
Logger.Log("Started");
string fileUrl = string.Empty;
Logger.Log("Asking GLEIF service for file URL");
WebRequest fileAddressRequest = WebRequest.Create(GLEIF_SERVICE);
using (var stream = fileAddressRequest.GetResponse().GetResponseStream())
{
using (var sr = new StreamReader(stream))
{
string fileAddressStringResponse = sr.ReadToEnd();
Logger.Log("Recieved GLEIF service response for file URL");
var fileAddressResponse = JsonConvert.DeserializeObject <ServiceResponse>(fileAddressStringResponse);
Logger.Log("Successfully parsed GLEIF service response for file URL");
fileUrl = fileAddressResponse.Data.First().Lei2.FullFile.Csv.Url;
}
}
if (string.IsNullOrEmpty(fileUrl))
{
throw new InvalidOperationException("GLEIF file URL was not obtained properly!");
}
else
{
Logger.Log($"GLEIF file URL is {fileUrl}");
}
//Log("Downloading GLEIF file to S3");
string s3FileKey = $"GLEIF/{DateTime.Now.ToString("yyyyMMdd")}.csv";
var s3Client = new AmazonS3Client(RegionEndpoint.USEast2);
Logger.Log("Starting reading Golden Copy file contents");
WebRequest fileRequest = WebRequest.Create(fileUrl);
var fileResponse = fileRequest.GetResponse();
using (Stream pageStream = fileResponse.GetResponseStream())
//using (Stream pageStream = new FileStream("C:\\temp\\20201102.zip", FileMode.Open, FileAccess.Read))
{
var archive = new ZipArchive(pageStream, ZipArchiveMode.Read);
var entry = archive.Entries.First();
using (var zippedFileStream = entry.Open())
{
using (var fileReader = new StreamReader(zippedFileStream))
{
CsvReader csvReader = ConfigureCsvReader(fileReader);
Logger.Log("Reading infrastructure for Golden Copy file contents initialized.");
Logger.Log("Configuring DB context.");
//using (var sw = new StreamWriter("C:\\temp\\allnames.dat"))
//{
// while (csvReader.Read())
// {
// sw.WriteLine(csvReader.GetRecord<LegalEntityName>().Name.ToUpperInvariant());
// }
//}
LeadContext dbContext = new LeadContext();
dbContext.ChangeTracker.AutoDetectChangesEnabled = false;
dbContext.ChangeTracker.LazyLoadingEnabled = true;
dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking;
Logger.Log("Clearing Lead GLEIF tables.");
//dbContext.Database.ExecuteSqlRaw("TRUNCATE TABLE gleif_validation_authority");
dbContext.Database.ExecuteSqlRaw("TRUNCATE TABLE gleif_address");
dbContext.Database.ExecuteSqlRaw("DELETE FROM gleif_entity_name");
dbContext.Database.ExecuteSqlRaw("DELETE FROM gleif_entity");
dbContext.Dispose();
Logger.Log("Lead GLEIF tables cleared.");
for (int i = 0; i < THREAD_COUNT; i++)
{
new Thread(x => UploadChunks(csvReader)).Start();
}
for (int i = 0; i < THREAD_COUNT; i++)
{
total.WaitOne();
}
Logger.Log($"Processing finished.");
}
}
}
sw.Stop();
Logger.Log($"Finished successfully in {sw.ElapsedMilliseconds} ms");
}
catch (Exception ex)
{
Logger.Log($"ERROR: Unhandled exception: {ex.ToString()}");
}
}
private void UploadChunks(CsvReader csvReader)
{
while (true)
{
LeadContext localDbContext = new LeadContext();
localDbContext.ChangeTracker.AutoDetectChangesEnabled = false;
localDbContext.ChangeTracker.LazyLoadingEnabled = true;
localDbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking;
bool canRead = true;
lock (this)
{
int counter = 0;
while ((canRead = csvReader.Read()) && counter < DB_FLUSH_THRESHOLD)
{
var entity = csvReader.GetRecord <GleifEntity>();
new List <GleifEntityName>()
{
csvReader.GetRecord <LegalEntityName>(),
csvReader.GetRecord <OtherEntityName1>(),
csvReader.GetRecord <OtherEntityName2>(),
csvReader.GetRecord <OtherEntityName3>(),
csvReader.GetRecord <OtherEntityName4>(),
csvReader.GetRecord <OtherEntityName5>(),
csvReader.GetRecord <TransliteratedOtherEntityName1>(),
csvReader.GetRecord <TransliteratedOtherEntityName2>(),
csvReader.GetRecord <TransliteratedOtherEntityName3>(),
csvReader.GetRecord <TransliteratedOtherEntityName4>(),
csvReader.GetRecord <TransliteratedOtherEntityName5>()
}
.Where(x => !string.IsNullOrEmpty(x.Name)).ToList()
.ForEach(x =>
{
entity.GleifEntityNames.Add(x);
x.Entity = entity;
localDbContext.GleifEntityName.Add(x);
});
new List <GleifAddress>()
{
csvReader.GetRecord <LegalAddress>(),
csvReader.GetRecord <HeadquartersAddress>(),
/*csvReader.GetRecord<OtherAddress1>(),
* csvReader.GetRecord<OtherAddress2>(),
* csvReader.GetRecord<OtherAddress3>(),
* csvReader.GetRecord<OtherAddress4>(),
* csvReader.GetRecord<OtherAddress5>(),*/
/*csvReader.GetRecord<TransliteratedOtherAddress1>(),
* csvReader.GetRecord<TransliteratedOtherAddress2>(),
* csvReader.GetRecord<TransliteratedOtherAddress3>(),
* csvReader.GetRecord<TransliteratedOtherAddress4>(),
* csvReader.GetRecord<TransliteratedOtherAddress5>(),*/
}
.Where(x => !string.IsNullOrEmpty(x.Firstaddressline)).ToList()
.ForEach(x =>
{
entity.GleifAddresses.Add(x);
x.Entity = entity;
localDbContext.GleifAddress.Add(x);
}
);
/*
* new List<GleifValidationAuthority>()
* {
* csvReader.GetRecord<ValidationAuthority>(),
* csvReader.GetRecord<OtherValidationAuthority1>(),
* csvReader.GetRecord<OtherValidationAuthority2>(),
* csvReader.GetRecord<OtherValidationAuthority3>(),
* csvReader.GetRecord<OtherValidationAuthority4>(),
* csvReader.GetRecord<OtherValidationAuthority5>(),
* }
* .Where(x => !string.IsNullOrEmpty(x.ValidationAuthorityId)).ToList()
* .ForEach(x =>
* {
* entity.GleifValidationAuthorities.Add(x);
* localDbContext.GleifValidationAuthority.Add(x);
* }
* );
*/
localDbContext.GleifEntity.Add(entity);
counter++;
}
}
localDbContext.SaveChanges();
localDbContext.Dispose();
if (!canRead)
{
break;
}
}
total.Release(1);
}