/// <summary>
/// Получить соответствия данным из загружаемой таблицы
/// </summary>
/// <param name="dataTable">Загружаемая таблица</param>
/// <param name="progressAction">Действие для отображения прогресса</param>
/// <param name="logAction">Действие для отображения лога</param>
/// <returns>Словарь соответсвия найденных данных для каждой строки таблицы</returns>
public IDictionary<DataRow, DataPreviewRow> Preview(DataTable dataTable, Action<decimal> progressAction = null, Action<string> logAction = null)
{
progressAction = progressAction ?? new Action<decimal>((i) => { });
logAction = logAction ?? new Action<string>((i) => { });
using (var logSession = Log.Session($"{this.GetType().Name}.{nameof(Preview)}()", VerboseLog))
try
{
logSession.Output = (strs) => strs.ToList().ForEach(s => logAction(s));
if (dataTable == null)
throw new ArgumentNullException(nameof(dataTable));
var pp = new Helpers.PercentageProgress();
var ppPrepare = pp.GetChild(weight: 0.1m);
var ppHostes = pp.GetChild(weight: 0.1m);
var ppPhones = pp.GetChild(weight: 0.1m);
var ppMarks = pp.GetChild(weight: 0.1m);
var ppCities = pp.GetChild(weight: 0.1m);
var ppStreets = pp.GetChild(weight: 0.7m);
var ppAddresses = pp.GetChild(weight: 0.15m);
var ppRows = pp.GetChild(weight: 0.3m);
pp.Change += (s, e) => progressAction(e.Value);
#region Get column names by column types
logSession.Add("Get column names by column types");
var columnNames = dataTable.Columns.OfType<DataColumn>().Select(c => c.ColumnName.ToUpper()).ToArray();
var columnByTypes = typeof(ColumnTypes)
.GetEnumValues()
.Cast<ColumnTypes>()
.Select(ct => new
{
Type = ct,
Column = Account.Settings.GetColumnFor(ct),
})
.Where(c => c.Column != null)
.Select(i => new
{
i.Type,
i.Column,
ExistsInDataTable = columnNames.Contains(i.Column.ColumnName.ToUpper()),
}).ToArray();
var badColumns = columnByTypes.Where(c => c.Column == null).Concat(i => RoyaltyRepository.Extensions.Extensions.GetEnumNameFromType(i.Type), ", ");
if (!string.IsNullOrWhiteSpace(badColumns))
throw new Exception(string.Format(Resources.DataCalculator_Preview_ColumnInSettingsNotSetted, badColumns));
var columnDict = columnByTypes.ToDictionary(i => i.Type, i => new { i.Column.ColumnName, i.ExistsInDataTable });
#endregion
#region Prepare data
logSession.Add("Parse incoming data");
var excludes = Account.Dictionary.Excludes
.Select(e => e.Exclude)
.ToArray();
var subRes0 = dataTable.Rows
.Cast<DataRow>()
.AsParallel()
.Select(dr => new
{
Row = dr,
IncomingAddressValue = columnDict[ColumnTypes.Address].ExistsInDataTable ? dr[columnDict[ColumnTypes.Address].ColumnName] : null,
IncomingHostValue = columnDict[ColumnTypes.Host].ExistsInDataTable ? dr[columnDict[ColumnTypes.Host].ColumnName] : null,
IncomingPhoneValue = columnDict[ColumnTypes.Phone].ExistsInDataTable ? dr[columnDict[ColumnTypes.Phone].ColumnName] : null,
IncomingCityValue = columnDict[ColumnTypes.City].ExistsInDataTable ? dr[columnDict[ColumnTypes.City].ColumnName] : null,
IncomingAreaValue = columnDict[ColumnTypes.Area].ExistsInDataTable ? dr[columnDict[ColumnTypes.Area].ColumnName] : null,
IncomingMarkValue = columnDict[ColumnTypes.Mark].ExistsInDataTable ? dr[columnDict[ColumnTypes.Mark].ColumnName] : null,
})
.Select(dr => new
{
dr.Row,
IncomingAddress = (dr.IncomingAddressValue == DBNull.Value || dr.IncomingAddressValue == null) ? string.Empty : dr.IncomingAddressValue.ToString(),
IncomingHost = (dr.IncomingHostValue == DBNull.Value || dr.IncomingHostValue == null) ? string.Empty : dr.IncomingHostValue.ToString(),
IncomingPhone = (dr.IncomingPhoneValue == DBNull.Value || dr.IncomingPhoneValue == null) ? string.Empty : dr.IncomingPhoneValue.ToString(),
IncomingCity = (dr.IncomingCityValue == DBNull.Value || dr.IncomingCityValue == null) ? string.Empty : dr.IncomingCityValue.ToString(),
IncomingArea = (dr.IncomingAreaValue == DBNull.Value || dr.IncomingAreaValue == null) ? string.Empty : dr.IncomingAreaValue.ToString(),
IncomingMark = (dr.IncomingMarkValue == DBNull.Value || dr.IncomingMarkValue == null) ? string.Empty : dr.IncomingMarkValue.ToString(),
})
.Select(dr => new
{
dr.Row,
IncomingAddress = Parser.Address.FromString(dr.IncomingAddress, dr.IncomingArea, excludes),
IncomingHost = Parser.Host.FromString(dr.IncomingHost),
IncomingPhone = Parser.Phone.FromString(dr.IncomingPhone),
dr.IncomingCity,
dr.IncomingMark,
})
.ToArray();
ppPrepare.Value = 100;
#endregion
#region Join hostes or create new
logSession.Add("Join hostes or create new");
var hosts = subRes0
.AsParallel()
.Select(i => i.IncomingHost.Hostname)
.Distinct()
.LeftOuterJoin(Repository.HostGet(), h => h.ToUpper(), h => h.Name.ToUpper(), (h, host) => new { HostName = h, Host = host })
.ToArray()
.Select(i => i.Host ?? Repository.HostNew(i.HostName))
.ToArray();
ppHostes.Value = 50;
var subRes1 = subRes0
.Join(hosts, i => i.IncomingHost.Hostname, h => h.Name, (i, h) => new
{
i.Row,
i.IncomingAddress,
i.IncomingCity,
i.IncomingPhone,
i.IncomingHost,
i.IncomingMark,
Host = h,
});
ppHostes.Value = 100;
#endregion
#region Join phones or create new
logSession.Add("Join phones or create new");
var phones = subRes0
.AsParallel()
.Select(i => i.IncomingPhone.PhoneNumber)
.Distinct()
.LeftOuterJoin(Repository.PhoneGet(), p => p, p => p.PhoneNumber, (p, phone) => new { PhoneNumber = p, Phone = phone })
.ToArray()
.Select(i => i.Phone ?? Repository.PhoneNew(i.PhoneNumber, Account))
.ToArray();
ppPhones.Value = 50;
var subRes2 = subRes1
.Join(phones, i => i.IncomingPhone.PhoneNumber, p => p.PhoneNumber, (i, p) => new
{
i.Row,
i.IncomingPhone,
i.IncomingAddress,
i.IncomingCity,
i.IncomingHost,
i.IncomingMark,
i.Host,
Phone = p,
});
ppPhones.Value = 100;
#endregion
#region Join marks
logSession.Add("Join marks");
var defMark = Repository.MarkGet(MarkTypes.Unknown);
#pragma warning disable 618
var subRes3 = subRes2
.LeftOuterJoin(Repository.MarkGet(), i => i.IncomingMark.ToUpper(), m => m.SystemName.ToUpper(), (i, m) => new
{
i.Row,
i.IncomingPhone,
i.IncomingAddress,
i.IncomingCity,
i.IncomingHost,
i.IncomingMark,
i.Host,
i.Phone,
Mark = m ?? defMark,
});
#pragma warning restore 618
ppMarks.Value = 100;
#endregion
#region Join cities or create new
logSession.Add("Join cities or create new");
var cities = subRes0
.AsParallel()
.Select(i => i.IncomingCity)
.Distinct()
.LeftOuterJoin(Repository.CityGet(), c => c.ToUpper(), c => c.Name.ToUpper(), (c, city) => new { CityName = c, City = city })
.ToArray()
.Select(i => i.City ?? Repository.CityNew(i.CityName))
.ToArray();
ppCities.Value = 50;
var subRes4 = subRes3
.Join(cities, i => i.IncomingCity.ToUpper(), c => c.Name.ToUpper(), (i, c) => new
{
i.Row,
i.IncomingPhone,
i.IncomingAddress,
i.IncomingCity,
i.IncomingHost,
i.IncomingMark,
i.Host,
i.Phone,
i.Mark,
City = c
});
ppCities.Value = 100;
#endregion
#region Join streets
logSession.Add("Join streets");
var aP = new Parser.AddressParser(Account, Repository);
var aPR = aP.Parse(subRes4
.GroupBy(i => new { i.City, i.IncomingAddress.Street, House = i.IncomingAddress.House.ToString(), i.IncomingAddress.Area })
.Select(g => new
{
City = g.Key.City,
Address = new Parser.Address(g.Key.Street, g.Key.House, g.Key.Area)
})
.Select(g => new Parser.AddressParserIncomingParameter()
{
City = g.City,
Address = g.Address
})
.ToArray(),
!UseDictionary,
(progress) => ppStreets.Value = progress,
(str) => logSession.Add(str));
var subRes5 = subRes4
.LeftOuterJoin(aPR,
r => new { r.IncomingAddress.Street, House = r.IncomingAddress.House.ToString(), r.IncomingAddress.Area, r.City.Name },
i => new { i.Key.Address.Street, House = i.Key.Address.House.ToString(), i.Key.Address.Area, i.Key.City.Name },
(r, i) => new
{
r.Row,
LoadedRow = new
{
i.Value.Address, //change old address to new address
i.Value.Street,
r.Mark,
r.Phone,
r.City,
r.Host,
},
});
#endregion
#region Update addresses
logSession.Add("Update house numbers");
var expectedData = Account.Data
.Where(d => !string.IsNullOrWhiteSpace(d.HouseNumber))
.GroupBy(d => new { d.Phone, d.Street })
.Select(d => new { d.Key.Phone, d.Key.Street, Houses = d.Select(i => i.HouseNumber), Cnt = d.Count() })
.Where(i => i.Cnt == 1)
.Select(i => new { i.Phone, i.Street, HouseNumber = i.Houses.FirstOrDefault() });
var currentData = subRes5
.Where(d => !string.IsNullOrWhiteSpace(d.LoadedRow.Address.House.ToString()))
.GroupBy(d => new { d.LoadedRow.Phone, d.LoadedRow.Street })
.Select(d => new { d.Key.Phone, d.Key.Street, Houses = d.Select(i => i.LoadedRow.Address.House.Number), Cnt = d.Count() })
.Where(i => i.Cnt == 1)
.Select(i => new { i.Phone, i.Street, HouseNumber = i.Houses.FirstOrDefault() });
ppAddresses.Value = 50;
var subRes6 = subRes5
.LeftOuterJoin(currentData, r => new { r.LoadedRow.Phone.PhoneNumber, r.LoadedRow.Street }, d => new { d?.Phone?.PhoneNumber, d?.Street }, (r, d) => new { Data = r, Grouped = d })
.LeftOuterJoin(expectedData, r => new { r.Data.LoadedRow.Phone.PhoneNumber, r.Data.LoadedRow.Street }, d => new { d?.Phone?.PhoneNumber, d?.Street },
(r, d) => new
{
r.Data.Row,
LoadedRow = new
{
//Address = new Parser.Address(r.Data.LoadedRow.Address.Street, (r.Data.LoadedRow.Address.House.Number?.ToString() ?? r.Grouped?.HouseNumber?.ToString() ?? d?.HouseNumber ?? string.Empty), r.Data.LoadedRow.Address.Area),
HouseNumber = (r.Data.LoadedRow.Address.House.Number?.ToString() ?? r.Grouped?.HouseNumber?.ToString() ?? d?.HouseNumber ?? string.Empty),
r.Data.LoadedRow.Street,
r.Data.LoadedRow.Mark,
r.Data.LoadedRow.Phone,
r.Data.LoadedRow.City,
r.Data.LoadedRow.Host,
}
});
ppAddresses.Value = 100;
#endregion
#region Load rows
logSession.Add("Load rows");
var ppRowsPrepare = ppRows.GetChild();
var ppRowsLoad = ppRows.GetChild();
var rowCount0 = (decimal)subRes6.Count() - 1;
var currentIndex0 = 0m;
var res = subRes6.Select(r => new
{
r.Row,
DataRecord = Repository.AccountDataRecordNew(Account, r.LoadedRow),
Index = (ppRowsPrepare.Value = (currentIndex0++) / rowCount0 * 100m)
})
.Select(r => new
{
r.Row,
LoadedRow = new DataPreviewRow()
{
DataRecord = r.DataRecord,
DataRecordAdditional = Repository.AccountDataRecordAdditionalNew(r.DataRecord),
},
}
)
.ToArray();
var rowCount1 = (decimal)dataTable.Rows.Count - 1;
var currentIndex1 = 0m;
var columns = Account.AdditionalColumns.Where(c => !string.IsNullOrWhiteSpace(c.ColumnName));
res.ToList()
.ForEach(r =>
{
foreach (var c in columns)
{
var pi = r.LoadedRow.DataRecordAdditional.GetType().GetProperty(c.ColumnSystemName);
pi.SetValue(r.LoadedRow.DataRecordAdditional, r.Row[c.ColumnName]);
}
ppRowsLoad.Value = currentIndex1++ / rowCount1 * 100m;
});
#endregion
return res.ToDictionary(i => i.Row, i => i.LoadedRow);
}
catch (Exception ex)
{
logSession.Add(ex);
logSession.Enabled = true;
throw;
}
}
