/// <summary>
/// Создание экземпляра класса
/// </summary>
/// <param name="account">Аккаунт</param>
/// <param name="repository">Репозиторий</param>
public AddressParser(RoyaltyRepository.Models.Account account, RoyaltyRepository.Repository repository)
{
if (repository == null)
throw new ArgumentNullException("repository");
if (account == null)
throw new ArgumentNullException("account");
Account = account;
Repository = repository;
}
public static IEnumerable<string> FileGetLines(this RoyaltyRepository.Repository repository, IFileStorage storage, RoyaltyRepository.Models.File repositoryFile)
{
if (repository == null)
throw new ArgumentNullException(nameof(repository));
if (repositoryFile == null)
throw new ArgumentNullException(nameof(repositoryFile));
if (storage == null)
throw new ArgumentNullException(nameof(storage));
var res = new List<string>();
using (var fileStream = storage.FileGet(repositoryFile.FileUID))
using (var sr = new System.IO.StreamReader(fileStream, repositoryFile.Encoding))
{
var line = string.Empty;
while ((line = sr.ReadLine()) != null)
res.Add(line);
}
return res;
}
private void UpdateFileInfoes(RoyaltyRepository.Repository rep, RoyaltyRepository.Models.ImportQueueRecord destination, IEnumerable<ImportQueueRecordFileInfo> fileInfoes)
{
throw new NotImplementedException();
}
private void LoadFiles(RoyaltyRepository.Repository rep, RoyaltyRepository.Models.ImportQueueRecordFileInfo destination, IEnumerable<ImportQueueRecordFileInfoFile> files)
{
foreach (var item in files)
{
var dbItem = rep.New<RoyaltyRepository.Models.ImportQueueRecordFileInfoFile>((i) => {
i.FileUID = item.File != null ? item.File.FileUID : item.FileUID;
i.File = rep.Get<RoyaltyRepository.Models.File>(f => f.FileUID == i.FileUID).FirstOrDefault();
i.ImportQueueRecordFileInfo = destination;
i.Type = RoyaltyRepository.Models.ImportQueueRecordFileInfoFileType.Import;
});
rep.Add(dbItem, saveAfterInsert: false);
}
}
private void LoadFileInfoes(RoyaltyRepository.Repository rep, RoyaltyRepository.Models.ImportQueueRecord destination, IEnumerable<ImportQueueRecordFileInfo> fileInfoes)
{
var stateDafault = rep.ImportQueueRecordStateGetDefault();
foreach(var item in fileInfoes)
{
var dbItem = rep.New<RoyaltyRepository.Models.ImportQueueRecordFileInfo>((i) => {
i.ImportQueueRecordFileInfoUID = Guid.NewGuid();
i.ForAnalize = item.ForAnalize;
i.ImportQueueRecord = destination;
i.SourceFilePath = item.SourceFilePath;
i.ImportQueueRecordState = stateDafault;
});
rep.Add(dbItem, saveAfterInsert: false);
LoadFiles(rep, dbItem, item.Files);
}
}
/// <summary>
/// Получение списка улиц в соответствии с адресами
/// </summary>
/// <param name="incomingAddresses">Адреса, для которых нужно определить улицы</param>
/// <param name="city">Город</param>
/// <param name="reportProgress">Action для отслеживания процесса выполнения</param>
/// <returns>Список соответствия входящему адресу найденной улицы</returns>
public IDictionary<Address, RoyaltyRepository.Models.Street> GetStreets(
IEnumerable<Address> incomingAddresses,
RoyaltyRepository.Models.City city,
Action<decimal> reportProgress = null,
Action<string> verboseLog = null)
{
return GetStreets(incomingAddresses, city, true, reportProgress, verboseLog);
}
/// <summary>
/// Получение списка улиц в соответствии с адресами
/// </summary>
/// <param name="incomingAddresses">Адреса, для которых нужно определить улицы</param>
/// <param name="city">Город</param>
/// <param name="doNotAddAnyDataToDictionary">Флаг для отмены добавления данных в словарь</param>
/// <param name="reportProgress">Action для отслеживания процесса выполнения</param>
/// <returns>Список соответствия входящему адресу найденной улицы</returns>
internal IDictionary<Address, RoyaltyRepository.Models.Street> GetStreets(
IEnumerable<Address> incomingAddresses,
RoyaltyRepository.Models.City city,
bool doNotAddAnyDataToDictionary = false,
Action<decimal> reportProgress = null,
Action<string> verboseLog = null)
{
var pp = new Helpers.PercentageProgress();
var ppLoad = pp.GetChild(weight: 0.5m);
var ppSubitem = pp.GetChild(weight: 9m);
var ppEnd = pp.GetChild(weight: 0.5m);
pp.Change += (s, e) => { if (reportProgress != null) reportProgress(e.Value); };
verboseLog = verboseLog ?? new Action<string>(s => { });
using (var logSession = Helpers.Log.Session($"{GetType().Name}.{nameof(GetStreets)}()"))
try
{
logSession.Output = new Action<IEnumerable<string>>( (strs) => strs.ToList().ForEach(s => verboseLog(s)) );
var findStreet = incomingAddresses
.Distinct()
.OrderByDescending(a => a.Area)
.ThenByDescending(a => a.Street)
.ThenByDescending(a => a.House.ToString())
.LeftOuterJoin(Account.Dictionary.Records, //.Where(r => r.Street != null && r.Street.Area != null),
s => new { StreetName = s.Street, AreaName = s.Area },
d => new { StreetName = d.Street.Name, AreaName = d.Street.Area.Name },
(s, d) => new
{
IncomingAddress = s,
Street = (d != null ? d.ChangeStreetTo ?? d.Street : null),
ConditionsScore = GetConditionsScore(s.House.Number, d == null ? null : d.Conditions, doNotAddAnyDataToDictionary),
})
.ToArray();
var count = findStreet.Length;
int current = 0;
ppLoad.Value = 90;
var res = findStreet
.Select(i =>
{
var subRes = new
{
i.IncomingAddress,
Street = i.Street
?? GetStreetByDictionary(i.IncomingAddress, city, doNotAddAnyDataToDictionary, (s) => logSession.Add(s))
?? GetNewStreet(i.IncomingAddress, city, doNotAddAnyDataToDictionary, (s) => logSession.Add(s))
};
current++;
ppSubitem.Value = (decimal)current / (decimal)count * 100m;
return subRes;
}
)
.GroupBy(i => i.IncomingAddress)
.Select(g => new { IncomingAddress = g.Key, Items = g })
//Исключаем случаи, когда в словаре более одной записи на входящий адрес (что-то пошло не так, и мы генерируем ошибку)
.Select(g =>
{
if (g.Items.Count() > 1)
{
var ex = new Exception($"For incoming address '{g.IncomingAddress.ToString()}' found more then 1 street. See data for details");
int index = 0;
foreach (var i in g.Items.Select(n => n.Street))
ex.Data.Add(index++, i.ToString());
throw ex;
}
return new
{
g.IncomingAddress,
g.Items.FirstOrDefault().Street
};
})
.ToArray();
ppLoad.Value = 100;
logSession.Add($"Array constructed with {res.Length} elements for city '{city}'. Try to create dictionary.");
return res.ToDictionary(i => i.IncomingAddress, i => i.Street);
}
catch(Exception ex)
{
logSession.Add(ex);
logSession.Enabled = true;
throw;
}
finally
{
ppEnd.Value = 100;
}
}
/// <summary>
/// Создает новую улицу и добавляет в словарь, если это необходимо
/// </summary>
/// <param name="address">Адрес</param>
/// <param name="city">Город</param>
/// <param name="doNotAddAnyDataToDictionary">Флаг для отмены добавления данных в словарь</param>
/// <param name="verboseLog">Action по логированию метода</param>
/// <returns>Улицу</returns>
internal RoyaltyRepository.Models.Street GetNewStreet(Address address, RoyaltyRepository.Models.City city, bool doNotAddAnyDataToDictionary, Action<string> verboseLog = null)
{
if (address == null)
throw new ArgumentNullException(nameof(address));
if (city == null)
throw new ArgumentNullException(nameof(city));
var log = new Action<string>((str) => { if (verboseLog != null) verboseLog($"{GetType().Name}.{nameof(GetNewStreet)}() {str}"); });
bool isNewArea = false;
try
{
log($"Incoming parameters: address is '{address}' and city is '{city}'");
RoyaltyRepository.Models.Area a = null;
if (string.IsNullOrWhiteSpace(address.Area))
{
log($"Area is unsetted. Use default aread for city: '{city.UndefinedArea}'");
a = city.UndefinedArea;
} else
{
log($"Try get area with name '{address.Area}' from database");
a = Repository.AreaGet(address.Area, city);
if (a == null)
{
log($"Area not found. Create new");
a = Repository.AreaNew(address.Area, city: city);
log($"New area for addres is {a}");
isNewArea = true;
} else
log($"Area found: {a}");
}
RoyaltyRepository.Models.Street s = null;
if (!isNewArea)
{
log($"Try to get street from database");
s = Repository.StreetGet(address.Street, a);
if (s != null)
log($"Street found in database: {s}"); else
log($"Street '{address.Street}' with area '{a}' not found in database");
}
if (s == null)
{
s = a != city.UndefinedArea ? Repository.StreetGet(address.Street, city.UndefinedArea) : null;
if (s != null)
{
log($"Street found in undefined area in database: {s}. Change area for this street to '{a}'");
s.Area = a;
}
else
{
s = Repository.StreetNew(address.Street, a);
log($"New street created: {s}");
}
}
else
log($"Street found in database: {s}");
if (Account.Dictionary.AllowAddToDictionaryAutomatically && !doNotAddAnyDataToDictionary)
lock (Account.Dictionary)
AddNewOrUpdateDictionaryRecord(address, s, (str) => log(str));
return s;
}
catch(Exception)
{
throw;
}
}
/// <summary>
/// Добавляет новую запись или обновляет существующую запись в словаре для адреса
/// </summary>
/// <param name="address">Адрес</param>
/// <param name="street">Улица</param>
/// <param name="verboseLog">Action по логированию метода</param>
internal void AddNewOrUpdateDictionaryRecord(Address address, RoyaltyRepository.Models.Street street, Action<string> verboseLog = null)
{
if (address == null)
throw new ArgumentNullException(nameof(address));
var log = new Action<string>((str) => { if (verboseLog != null) verboseLog($"{GetType().Name}.{nameof(AddNewOrUpdateDictionaryRecord)}() {str}"); });
log("Try get record to dictionary");
var dictionaryRecords = Account.Dictionary.Records.Where(ad => ad.Street == street);
if (!dictionaryRecords.Any())
{
log("Dictionary record not found. Create new and add it");
var adr = Repository.AccountDictionaryRecordNew(Account.Dictionary, street: street);
if (address.House.Number.HasValue)
{
var cond = Repository.AccountDictionaryRecordConditionNew(adr, address.House.Number, address.House.Number);
log($"Add condition for dictionary record {cond.ToString()}");
}
log($"Add dictionary record: {adr.ToString()}");
}
else
{
ConcatDictionaryRecordConditions(address, dictionaryRecords, (str) => log(str));
}
}
/// <summary>
/// Получает улицу по адресу в соответствии со словарем
/// </summary>
/// <param name="address">Адрес для поиска</param>
/// <param name="city">Город</param>
/// <param name="verboseLog">Action по логированию поиска</param>
/// <returns>Улица из БД. Если NULL, значит улица в соответствии со словарем не найдена</returns>
public RoyaltyRepository.Models.Street GetStreetByDictionary(Address address, RoyaltyRepository.Models.City city, Action<string> verboseLog = null)
{
return GetStreetByDictionary(address, city, true, verboseLog);
}
/// <summary>
/// Получает улицу по адресу в соответствии со словарем
/// </summary>
/// <param name="address">Адрес для поиска</param>
/// <param name="city">Город</param>
/// <param name="doNotAddAnyDataToDictionary">Флаг для отмены добавления данных в словарь</param>
/// <param name="log">Action по логированию поиска</param>
/// <returns>Улица из БД. Если NULL, значит улица в соответствии со словарем не найдена</returns>
internal RoyaltyRepository.Models.Street GetStreetByDictionary(Address address, RoyaltyRepository.Models.City city, bool doNotAddAnyDataToDictionary, Action<string> verboseLog = null)
{
if (address == null)
throw new ArgumentNullException(nameof(address));
if (city == null)
throw new ArgumentNullException(nameof(city));
var log = new Action<string>((str) => { if (verboseLog != null) verboseLog($"{GetType().Name}.{nameof(GetStreetByDictionary)}() {str}"); });
//lock (Account.Dictionary)
try
{
log($"Get all account records");
ICollection<RoyaltyRepository.Models.AccountDictionaryRecord> recs = null;
lock (Account.Dictionary)
recs = Account.Dictionary.Records.ToArray();
log($"Get all streets in city '{city}'");
var dictionary = city.Areas
//.AsParallel()
.SelectMany(a => a.Streets.Select(s => new
{
Area = a,
Street = s
}))
.LeftOuterJoin(recs, s => s.Street, r => r.Street, (s, r) => new
{
Street = s.Street,
Area = s.Area,
ChangeStreetTo = r?.ChangeStreetTo,
Conditions = r?.Conditions,
DictionaryRecord = r,
})
.ToArray();
log($"Try to add street for sort dictionary");
var dS = dictionary.AsParallel().Where(i => i.Street.Name.ToUpper() == address.Street.ToUpper()).ToArray();
if (dS.Any())
{
dictionary = dS;
log($"Street added for sort");
}
if (!string.IsNullOrWhiteSpace(address.Area))
{
log($"Try to add area for sort dictionary");
var dA = dictionary.AsParallel().Where(i => i.Area.Name.ToUpper() == address.Area.ToUpper()).ToArray();
if (dA.Any())
{
dictionary = dA;
log($"Area added for sort");
}
}
log($"Get scores for dictionary and incoming address '{address}'");
var subRes = dictionary
.AsParallel()
.Select(i => new
{
Street = i.ChangeStreetTo ?? i.Street,
i.Street.Area,
ConditionsScore = GetConditionsScore(address.House.Number, i.Conditions, doNotAddAnyDataToDictionary),
StreetScore = (decimal)new WordsMatching.MatchsMaker(address.Street, i.Street.Name).Score,
AreaScore = string.IsNullOrWhiteSpace(address.Area)
? Account.Dictionary.SimilarityForTrust
: (decimal)new WordsMatching.MatchsMaker(address.Area, i.Street.Area.Name).Score,
i.DictionaryRecord,
});
log($"Get nearest data");
var res = subRes
.Where(i => i.StreetScore + i.AreaScore >= Account.Dictionary.SimilarityForTrust * 2 && i.ConditionsScore >= Account.Dictionary.ConditionsScoreForTrust)
.OrderByDescending(i => i.StreetScore + i.AreaScore + i.ConditionsScore / 2m)
.ThenByDescending(i => i.DictionaryRecord != null ? 1 : 0)
.ThenByDescending(i => i.Area.Streets.Count)
.FirstOrDefault();
if (res == null)
log($"Street not found for address '{address}'");
else
{
log($"Founded street {(res.DictionaryRecord != null ? "in dictionary" : string.Empty)}for address '{address}': '{res}'");
if (Account.Dictionary.AllowAddToDictionaryAutomatically && !doNotAddAnyDataToDictionary)
{
if (res.DictionaryRecord == null)
{
lock (Account.Dictionary)
AddNewOrUpdateDictionaryRecord(address, res.Street, (str) => log(str));
}
else
ConcatDictionaryRecordConditions(address, new RoyaltyRepository.Models.AccountDictionaryRecord[] { res.DictionaryRecord }, (str) => log(str));
}
}
return res?.Street;
}
catch(Exception ex)
{
log(ex.GetExceptionText());
throw;
}
}