public async Task Map(string supplier, Tracer tracer, TelemetrySpan parentSpan, CancellationToken cancellationToken)
{
using var countryMappingSpan = tracer.StartActiveSpan("Map Countries", SpanKind.Internal, parentSpan);
_logger.LogMappingCountriesStart(supplier);
var countries = await _locationMapperDataRetrieveService.GetNormalizedCountries(cancellationToken);
var countryPairsChanged = new Dictionary <int, int>();
var notSuppliersCountries = countries.Where(c => !c.SupplierCountryCodes.ContainsKey(supplier)).ToList();
var suppliersCountries = countries.Where(c => c.SupplierCountryCodes.ContainsKey(supplier)).ToList();
var countriesToMap = await _context.RawAccommodations.Where(ac => ac.Supplier == supplier)
.Select(ac
=> new
{
Names = ac.CountryNames,
Code = ac.CountryCode
})
.Distinct().ToListAsync(cancellationToken);
countriesToMap = countriesToMap.GroupBy(c => c.Code).Select(c => c.First()).ToList();
var countriesToUpdate = new List <Country>();
var newCountries = new List <Country>();
var utcDate = DateTimeOffset.UtcNow;
foreach (var country in countriesToMap)
{
cancellationToken.ThrowIfCancellationRequested();
var defaultName = country.Names.GetValueOrDefault(Constants.DefaultLanguageCode);
if (!defaultName.IsValid())
{
continue;
}
var code = _locationNameNormalizer.GetNormalizedCountryCode(defaultName, country.Code);
var dbCountry = new Country
{
Code = code,
Names = _multilingualDataHelper.NormalizeCountryMultiLingualNames(country.Names),
IsActive = true,
Modified = utcDate
};
var dbNotSuppliersCountry = notSuppliersCountries.FirstOrDefault(c => c.Code == code);
var dbSuppliersCountry = suppliersCountries.FirstOrDefault(c => c.Code == code);
if (dbNotSuppliersCountry != default)
{
dbCountry.Id = dbNotSuppliersCountry.Id;
dbCountry.Names = MultiLanguageHelpers.MergeMultilingualStrings(dbCountry.Names, dbNotSuppliersCountry.Names);
dbCountry.SupplierCountryCodes =
new Dictionary <string, string>(dbNotSuppliersCountry.SupplierCountryCodes);
dbCountry.SupplierCountryCodes.TryAdd(supplier, code);
if (dbSuppliersCountry != default)
{
countryPairsChanged.Add(dbSuppliersCountry.Id, dbCountry.Id);
foreach (var sup in dbSuppliersCountry.SupplierCountryCodes)
{
dbCountry.SupplierCountryCodes.TryAdd(sup.Key, sup.Value);
}
dbSuppliersCountry.IsActive = false;
countriesToUpdate.Add(dbSuppliersCountry);
}
countriesToUpdate.Add(dbCountry);
}
else if (dbSuppliersCountry == default)
{
dbCountry.SupplierCountryCodes = new Dictionary <string, string> {
{ supplier, code }
};
dbCountry.Created = utcDate;
newCountries.Add(dbCountry);
}
}
// TODO: Remove Distinct ( in connectors may be the same data in different forms normalized or not that is why needed distinct here )
_context.UpdateRange(countriesToUpdate.Distinct(new CountryComparer()));
_context.AddRange(newCountries.Distinct(new CountryComparer()));
await ChangeCountryDependencies(countryPairsChanged, cancellationToken);
await _context.SaveChangesAsync(cancellationToken);
await _countryChangePublisher.PublishAddedCountries(newCountries
.Distinct(new CountryComparer())
.Select(c => new CountryData(c.Id, c.Names.En, c.Code))
.ToList());
await _countryChangePublisher.PublishRemovedCountries(countryPairsChanged.Keys.ToList());
_logger.LogMappingCountriesFinish(supplier);
}
public async Task Map(string supplier, Tracer tracer, TelemetrySpan parentSpan, CancellationToken cancellationToken)
{
using var localityZoneMappingSpan =
tracer.StartActiveSpan("Map Locality zones", SpanKind.Internal, parentSpan);
_logger.LogMappingLocalityZonesStart(supplier);
var countries = await _locationMapperDataRetrieveService.GetCountries(cancellationToken);
foreach (var country in countries)
{
_logger.LogMappingLocalityZonesOfSpecifiedCountryStart(supplier, country.Code);
var changedLocalityZonesPairs = new Dictionary <int, int>();
var countryLocalities = await _locationMapperDataRetrieveService.GetNormalizedLocalitiesByCountry(country.Code, cancellationToken);
var dbNormalizedLocalityZones =
await _locationMapperDataRetrieveService.GetNormalizedLocalityZonesByCountry(country.Code, cancellationToken);
var notSuppliersLocalityZones = dbNormalizedLocalityZones
.Where(l => !l.LocalityZone.SupplierLocalityZoneCodes.ContainsKey(supplier)).ToList();
var suppliersLocalityZones = dbNormalizedLocalityZones
.Where(l => l.LocalityZone.SupplierLocalityZoneCodes.ContainsKey(supplier)).ToList();
var localityZonesToMap = await _context.RawAccommodations
.Where(ac => ac.Supplier == supplier && ac.LocalityZoneNames != null &&
ac.CountryCode == country.Code)
.Select(ac
=> new
{
LocalityNames = ac.LocalityNames,
CountryCode = ac.CountryCode,
CountryNames = ac.CountryNames,
LocalityZoneNames = ac.LocalityZoneNames,
LocalityZoneCode = ac.SupplierLocalityZoneCode
})
.Distinct().ToListAsync(cancellationToken);
localityZonesToMap = localityZonesToMap
.GroupBy(lz => new { LocalityName = lz.LocalityNames.En, LocalityZoneName = lz.LocalityZoneNames.En })
.Select(lz => lz.First())
.ToList();
var localityZonesToUpdate = new List <LocalityZone>();
var localityZonesToAdd = new List <LocalityZone>();
var utcDate = DateTimeOffset.UtcNow;
foreach (var zone in localityZonesToMap)
{
cancellationToken.ThrowIfCancellationRequested();
var defaultCountryName = zone.CountryNames.GetValueOrDefault(Constants.DefaultLanguageCode);
var normalizedCountryName = _locationNameNormalizer.GetNormalizedCountryName(defaultCountryName);
var defaultLocalityName = zone.LocalityNames.GetValueOrDefault(Constants.DefaultLanguageCode);
var normalizedLocalityName =
_locationNameNormalizer.GetNormalizedLocalityName(normalizedCountryName, defaultLocalityName);
var defaultLocalityZone = zone.LocalityZoneNames.GetValueOrDefault(Constants.DefaultLanguageCode);
var normalizedLocalityZone = defaultLocalityZone.ToNormalizedName();
if (!normalizedLocalityZone.IsValid())
{
continue;
}
var dbNotSuppliersZone = notSuppliersLocalityZones.FirstOrDefault(lz
=> lz.DefaultLocality == normalizedLocalityName &&
lz.LocalityZone.Names.En == normalizedLocalityZone);
var dbSuppliersZone = suppliersLocalityZones.FirstOrDefault(lz
=> lz.DefaultLocality == normalizedLocalityName &&
lz.LocalityZone.Names.En == normalizedLocalityZone);
var dbLocalityZone = new LocalityZone
{
Names = _multilingualDataHelper.NormalizeCountryMultiLingualNames(zone.LocalityZoneNames),
IsActive = true,
Modified = utcDate
};
if (dbNotSuppliersZone != default)
{
dbLocalityZone.Id = dbNotSuppliersZone.LocalityZone.Id;
dbLocalityZone.LocalityId = dbNotSuppliersZone.LocalityZone.LocalityId;
dbLocalityZone.Names =
MultiLanguageHelpers.MergeMultilingualStrings(dbLocalityZone.Names,
dbNotSuppliersZone.LocalityZone.Names);
dbLocalityZone.SupplierLocalityZoneCodes =
new Dictionary <string, string>(dbNotSuppliersZone.LocalityZone
.SupplierLocalityZoneCodes);
dbLocalityZone.SupplierLocalityZoneCodes.TryAdd(supplier, zone.LocalityZoneCode);
if (dbSuppliersZone != default)
{
changedLocalityZonesPairs.Add(dbSuppliersZone.LocalityZone.Id,
dbNotSuppliersZone.LocalityZone.Id);
foreach (var sup in dbSuppliersZone.LocalityZone.SupplierLocalityZoneCodes)
{
dbLocalityZone.SupplierLocalityZoneCodes.TryAdd(sup.Key, sup.Value);
}
dbSuppliersZone.LocalityZone.IsActive = false;
localityZonesToUpdate.Add(dbSuppliersZone.LocalityZone);
}
localityZonesToUpdate.Add(dbLocalityZone);
}
else if (dbSuppliersZone == default)
{
dbLocalityZone.Created = utcDate;
if (!_localityValidator.IsNormalizedValid(normalizedCountryName, normalizedLocalityName))
{
continue;
}
dbLocalityZone.LocalityId = countryLocalities.First(l => l.Names.En == normalizedLocalityName).Id;
dbLocalityZone.SupplierLocalityZoneCodes = new Dictionary <string, string>
{
{ supplier, zone.LocalityZoneCode }
};
localityZonesToAdd.Add(dbLocalityZone);
}
}
_context.UpdateRange(localityZonesToUpdate.Distinct(new LocalityZoneComparer()));
_context.AddRange(localityZonesToAdd.Distinct(new LocalityZoneComparer()));
await ChangeLocalityZoneDependencies(changedLocalityZonesPairs, cancellationToken);
await _context.SaveChangesAsync(cancellationToken);
// await _locationsChangePublisher.PublishRemovedLocalityZones(changedLocalityZonesPairs.Keys.ToList());
// await _locationsChangePublisher.PublishAddedLocalityZones(localityZonesToAdd
// .Distinct(new LocalityZoneComparer())
// .Select(lz => new LocalityZoneData(lz.Id, lz.Names.En,
// countryLocalities.First(l => l.Id == lz.LocalityId).Names.En, country.Name, country.Code))
// .ToList());
_context.ChangeTracker.Entries()
.Where(e => e.Entity != null)
.Where(e => e.State != EntityState.Detached)
.ToList()
.ForEach(e => e.State = EntityState.Detached);
localityZoneMappingSpan.AddEvent($"Done mapping locality zones of country with code {country.Code}");
_logger.LogMappingLocalityZonesOfSpecifiedCountryFinish(supplier, country.Code);
}
_logger.LogMappingLocalityZonesFinish(supplier);
}
private async Task MapCountryLocalities(string supplier, int countryId, string countryCode, string countryName, TelemetrySpan localityMappingSpan,
CancellationToken cancellationToken)
{
_logger.LogMappingLocalitiesOfSpecifiedCountryStart(supplier, countryCode);
var utcDate = DateTimeOffset.UtcNow;
var changedLocalityPairs = new Dictionary <int, int>();
var localitiesToUpdate = new List <Locality>();
var newLocalities = new List <Locality>();
var normalizedCountryName = _locationNameNormalizer.GetNormalizedCountryName(countryName);
var dbNormalizedLocalities = await _locationMapperDataRetrieveService.GetNormalizedLocalitiesByCountry(countryCode, cancellationToken);
var notSuppliersLocalities = dbNormalizedLocalities
.Where(l => !l.SupplierLocalityCodes.ContainsKey(supplier)).ToList();
var suppliersLocalities = dbNormalizedLocalities
.Where(l => l.SupplierLocalityCodes.ContainsKey(supplier)).ToList();
var localities = await _context.RawAccommodations
.Where(ac => ac.Supplier == supplier && ac.LocalityNames != null && ac.CountryCode == countryCode)
.Select(ac
=> new
{
LocalityCode = ac.SupplierLocalityCode,
LocalityNames = ac.LocalityNames
})
.Distinct().ToListAsync(cancellationToken);
localities = localities.GroupBy(l => _locationNameNormalizer.GetNormalizedLocalityName(normalizedCountryName, l.LocalityNames.En))
.Select(l => l.First()).ToList();
foreach (var locality in localities)
{
cancellationToken.ThrowIfCancellationRequested();
var defaultLocalityName = locality.LocalityNames.GetValueOrDefault(Constants.DefaultLanguageCode);
var normalizedLocalityName = _locationNameNormalizer.GetNormalizedLocalityName(countryName, defaultLocalityName);
if (!_localityValidator.IsNormalizedValid(normalizedCountryName, normalizedLocalityName))
{
_logger.LogMappingInvalidLocality(defaultLocalityName, normalizedCountryName, supplier,
JsonSerializer.Serialize(new { countryCode, countryId, countryName }), JsonSerializer.Serialize(locality));
continue;
}
var dbNotSuppliersLocality = notSuppliersLocalities.FirstOrDefault(l => l.Names.En == normalizedLocalityName);
var dbSuppliersLocalities = suppliersLocalities.Where(l => l.Names.En == normalizedLocalityName).ToList();
var richLocality = new RichLocality(supplierCode: locality.LocalityCode,
defaultNormalizedName: normalizedLocalityName,
defaultNormalizedCountryName: normalizedCountryName,
names: locality.LocalityNames);
if (dbSuppliersLocalities.Any())
{
ProcessIfSupplierLocalitiesExist(richLocality, dbSuppliersLocalities, dbNotSuppliersLocality);
}
else if (dbNotSuppliersLocality != default)
{
ProcessIfNotSupplierLocalityExists(richLocality, dbNotSuppliersLocality);
}
else
{
ProcessNewLocality(richLocality);
}
}
_context.UpdateRange(localitiesToUpdate);
_context.AddRange(newLocalities);
await ChangeLocalityDependencies(changedLocalityPairs, cancellationToken);
await _context.SaveChangesAsync(cancellationToken);
await _localityChangePublisher.PublishRemovedLocalities(changedLocalityPairs.Keys.ToList());
await _localityChangePublisher.PublishAddedLocalities(newLocalities
.Select(l => new LocalityData(l.Id, l.Names.En, countryName, countryCode))
.ToList());
_context.ChangeTracker.Entries()
.Where(e => e.Entity != null)
.Where(e => e.State != EntityState.Detached)
.ToList()
.ForEach(e => e.State = EntityState.Detached);
localityMappingSpan.AddEvent($"Done mapping localities of country with code {countryCode}");
_logger.LogMappingLocalitiesOfSpecifiedCountryFinish(supplier, countryCode);
void ProcessIfSupplierLocalitiesExist(RichLocality locality, List <Locality> supplierLocalities, Locality?notSupplierLocality)
{
var dbSuppliersLocality = supplierLocalities.First();
var dbSuppliersLocalitiesToDeactivate = supplierLocalities.GetRange(1, supplierLocalities.Count - 1);
var names = _multilingualDataHelper.NormalizeLocalityMultilingualNames(locality.DefaultNormalizedCountryName, locality.Names);
var supplierCodes = dbSuppliersLocality.SupplierLocalityCodes;
foreach (var supplierLocality in dbSuppliersLocalitiesToDeactivate)
{
names = MultiLanguageHelpers.MergeMultilingualStrings(names, supplierLocality.Names);
foreach (var supplierCode in supplierLocality.SupplierLocalityCodes)
{
supplierCodes.TryAdd(supplierCode.Key, supplierCode.Value);
}
supplierLocality.IsActive = false;
supplierLocality.Modified = utcDate;
localitiesToUpdate.Add(supplierLocality);
changedLocalityPairs.Add(supplierLocality.Id, dbSuppliersLocality.Id);
}
if (notSupplierLocality != default)
{
names = MultiLanguageHelpers.MergeMultilingualStrings(names, notSupplierLocality.Names);
changedLocalityPairs.Add(notSupplierLocality.Id, dbSuppliersLocality.Id);
foreach (var sup in notSupplierLocality.SupplierLocalityCodes)
{
supplierCodes.TryAdd(sup.Key, sup.Value);
}
notSupplierLocality.IsActive = false;
notSupplierLocality.Modified = utcDate;
localitiesToUpdate.Add(notSupplierLocality);
}
names = _multilingualDataHelper.NormalizeLocalityMultilingualNames(locality.DefaultNormalizedCountryName, names);
var dbLocality = new Locality
{
Id = dbSuppliersLocality.Id,
CountryId = countryId,
Names = names,
SupplierLocalityCodes = supplierCodes,
Modified = utcDate,
IsActive = true
};
localitiesToUpdate.Add(dbLocality);
}
void ProcessIfNotSupplierLocalityExists(RichLocality locality, Locality notSupplierLocality)
{
var supplierCodes = new Dictionary <string, string?>(notSupplierLocality.SupplierLocalityCodes);
supplierCodes.TryAdd(supplier, locality.SupplierCode);
var names = _multilingualDataHelper.NormalizeLocalityMultilingualNames(locality.DefaultNormalizedCountryName,
MultiLanguageHelpers.MergeMultilingualStrings(locality.Names, notSupplierLocality.Names));
var dbLocality = new Locality
{
Id = notSupplierLocality !.Id,
CountryId = countryId,
Names = names,
SupplierLocalityCodes = supplierCodes,
IsActive = true,
Modified = utcDate
};
localitiesToUpdate.Add(dbLocality);
}
void ProcessNewLocality(RichLocality locality)
{
var dbLocality = new Locality
{
CountryId = countryId,
Names = _multilingualDataHelper.NormalizeLocalityMultilingualNames(locality.DefaultNormalizedCountryName, locality.Names),
IsActive = true,
Modified = utcDate,
Created = utcDate,
SupplierLocalityCodes = new Dictionary <string, string?>
{
{ supplier, locality.SupplierCode }
}
};
newLocalities.Add(dbLocality);
}
}
