public void DataCalculator_Preview_ExceptOnTableValidation() { var a = Rep.GetAccount(defAccountName, eagerLoad: new string[] { "Settings.Columns" }); if (a != null) using (var dc = new DataCalculator(a, Rep)) { var columns = string.Empty; var colValues = a.Settings.Columns .Where(c => c.ColumnType.ImportTableValidation) .Select(c => c.ColumnName.ToLower()) .ToArray() .Skip(1); //Skip one required row for take exception foreach (var colName in colValues) columns += (string.IsNullOrWhiteSpace(columns) ? string.Empty : ";") + colName; var csvLines = new List<string>(); csvLines.Add(columns); try { var l = Helpers.CSV.CSVFile.Load(csvLines.AsEnumerable<string>(), tableName: "{virtual}", filePath: "{virtual}", tableValidator: dc.TableValidator, rowFilter: dc.RowFilter); } catch(Exception ex) { var exText = ex.GetExceptionText(); Console.WriteLine(exText); Assert.AreEqual(true, exText.Contains(colValues.First()), "Exception must contains required column name"); throw; } } }
public void DataCalculator_Preview_RowFilterAll() { var a = Rep.GetAccount(defAccountName); if (a != null) using (var dc = new DataCalculator(a, Rep)) { var columns = string.Empty; var colValues = a.Settings.Columns .Where(c => c.ColumnType.ImportTableValidation) .Select(c => c.ColumnName.ToLower()) .ToArray(); var csvLines = new List<string>(); foreach (var colName in colValues) columns += (string.IsNullOrWhiteSpace(columns) ? string.Empty : ";") + colName; csvLines.Add(columns); var cnt = 100000; var goodCnt = 0; for(int i=0; i< cnt; i++) { columns = string.Empty; foreach (var colName in colValues.Skip(1)) columns += (string.IsNullOrWhiteSpace(columns) ? string.Empty : ";") + string.Format("data_{0}", i); if (i % 2 == 0) { columns += ";bad_data"; } else { columns += ";good_data"; goodCnt++; } csvLines.Add(columns); } var l = Helpers.CSV.CSVFile.Load(csvLines, tableName: "{virtual}", filePath: "{virtual}", tableValidator: dc.TableValidator, verboseLogAction: (s) => { Console.WriteLine(s); if (s.Contains("good_data")) throw new Exception("Good data filtered: " + s); }, rowFilter: r => r.Table.Columns.OfType<DataColumn>().Any(c => r[c].ToString().Contains("bad_data")) ); Assert.AreEqual(goodCnt, l.Table.Rows.Count, "Good row count must equals"); } }
public void DataCalculator_Prepare() { SqlLogEnabled = false; try { var acc = Rep.GetAccount(defAccountName, eagerLoad: new string[] { "Settings.Columns.ColumnType" }); var csvLines = new List<string>(); var colValues = acc.Settings.Columns .Where(c => c.ColumnType.ImportTableValidation) .Select(c => new { Name = c.ColumnName.ToLower(), Type = c.ColumnType }) .ToArray(); var columns = string.Empty; foreach (var colName in colValues) columns += (string.IsNullOrWhiteSpace(columns) ? string.Empty : ";") + colName.Name; columns += ";test_column"; csvLines.Add(columns); using (var dc = new DataCalculator(acc, Rep)) { var prg0 = new Helpers.PercentageProgress(); prg0.Change += (s, e) => Helpers.Log.Add($"Progress: {e.Value.ToString("N2")}"); var log = new Action<string>((s) => Helpers.Log.Add(s)); var l = Helpers.CSV.CSVFile.Load(csvLines.AsEnumerable(), tableValidator: dc.TableValidator, rowFilter: dc.RowFilter); dc.Prepare(l.Table, i => prg0.Value = i, log); Assert.AreEqual(true, acc.AdditionalColumns.Any(ad => ad.ColumnName == "test_column")); Rep.SaveChanges(); Rep.AccountDataRecordAdditionalColumnRemove(acc.AdditionalColumns); } } finally { SqlLogEnabled = true; } }
public void DataCalculator_Preview() { SqlLogEnabled = false; var rnd = new Random(); rnd.Next(); var alph = "йцукенгшщзфывапролдячсмить"; var maxCnt = 100; var cityNames = Enumerable.Range(0, 3).Select(i => $"TestCity{i}").ToArray(); var areaNames = Enumerable.Range(0, 10).Select(i => $"TestArea{i}").ToArray(); var streetNames = Enumerable.Range(0, 100).Select(i => Enumerable.Range(0, 6).Select(n => alph[rnd.Next(alph.Length)].ToString()).Concat(c => c)).ToArray(); var hostNames = new string[] { "testhost0.ru", "testhost1.ru" }; try { hostNames.ToList().ForEach((h) => { Rep.HostRemove(Rep.HostGet(h), saveAfterRemove: false); var host = Rep.HostNew(h); Rep.HostAdd(host, saveAfterInsert: false); }); Rep.SaveChanges(); var acc = Rep.GetAccount(defAccountName, eagerLoad: new string[] { "Settings.Columns.ColumnType" }); cityNames.ToList().ForEach((s) => { Rep.CityRemove(Rep.CityGet(s), saveAfterRemove: false); }); Rep.SaveChanges(); cityNames.ToList().ForEach((s) => { var city = Rep.CityNew(s); areaNames.ToList().ForEach((a) => { var area = Rep.AreaNew(a, city: city); streetNames.ToList().ForEach((ss) => { if (rnd.Next(5) != 0) Rep.StreetNew(RoyaltyDataCalculator.Parser.Address.FromString(ss, area.Name, acc.Dictionary.Excludes.Select(e => e.Exclude)).Street, area); }); }); }); Rep.SaveChanges(); var addrCol = acc.Settings.GetColumnFor(RoyaltyRepository.Models.ColumnTypes.Address); var areaCol = acc.Settings.GetColumnFor(RoyaltyRepository.Models.ColumnTypes.Area); var cityCol = acc.Settings.GetColumnFor(RoyaltyRepository.Models.ColumnTypes.City); var cityHost = acc.Settings.GetColumnFor(RoyaltyRepository.Models.ColumnTypes.Host); var cityPhone = acc.Settings.GetColumnFor(RoyaltyRepository.Models.ColumnTypes.Phone); var csvLines = new List<string>(); var colValues = acc.Settings.Columns .Where(c => c.ColumnType.ImportTableValidation) .Select(c => new { Name = c.ColumnName.ToLower(), Type = c.ColumnType }) .ToArray(); var columns = string.Empty; foreach (var colName in colValues) columns += (string.IsNullOrWhiteSpace(columns) ? string.Empty : ";") + colName.Name; csvLines.Add(columns); var lastAddedPhone = string.Empty; var genNewData = new Func<RoyaltyRepository.Models.ColumnTypes, string>( (ct) => { var res = string.Empty; switch(ct) { case RoyaltyRepository.Models.ColumnTypes.Phone: if (!string.IsNullOrWhiteSpace(lastAddedPhone) && rnd.Next(0, 5) == 0) res = lastAddedPhone; else for (int n = 0; n < 5; n++) res += rnd.Next(10, 99).ToString(); break; case RoyaltyRepository.Models.ColumnTypes.Address: res = streetNames.Union(new string[] { "defaultPreviewStreet2" }).ToArray()[rnd.Next(0, streetNames.Length + 1)] + ((rnd.Next(0, 5) != 0) ? ", " + rnd.Next(1,99) : string.Empty); break; case RoyaltyRepository.Models.ColumnTypes.Area: res = areaNames.Union(new string[] { string.Empty, "defaultPreviewArea2" }).ToArray()[rnd.Next(0, areaNames.Length + 2)]; break; case RoyaltyRepository.Models.ColumnTypes.City: res = cityNames.Union(new string[] { "defaultPreviewCity1" }).ToArray()[rnd.Next(0, cityNames.Length + 1)]; break; case RoyaltyRepository.Models.ColumnTypes.Host: res = (rnd.Next(3) == 0 ? string.Empty : @"http://") + hostNames.Union(new string[] { "testhost2.ru" }).ToArray()[rnd.Next(0, hostNames.Length + 1)] + "/" + Guid.NewGuid().ToString("N"); break; } return res; } ); for (int i = 0; i < maxCnt; i++ ) { columns = string.Empty; foreach (var colName in colValues) columns += (string.IsNullOrWhiteSpace(columns) ? string.Empty : ";") + genNewData(colName.Type.Type); csvLines.Add(columns); } Rep.SaveChanges(); using (var dc = new DataCalculator(acc, Rep)) { var prg0 = new Helpers.PercentageProgress(); var prg1 = prg0.GetChild(); var prg2 = prg0.GetChild(); var prg3 = prg0.GetChild(); prg0.Change += (s, e) => Helpers.Log.Add($"Progress: {e.Value.ToString("N2")}"); var log = new Action<string>((s) => Helpers.Log.Add(s)); var l = Helpers.CSV.CSVFile.Load(csvLines.AsEnumerable(), tableValidator: dc.TableValidator, rowFilter: dc.RowFilter); dc.Prepare(l.Table, i => prg1.Value = i, log); var previewRes = dc.Preview(l.Table, i => prg2.Value = i, log); Assert.AreEqual(l.Table.Rows.Count, previewRes.Count()); var readyToExport = dc.Import(previewRes.Values, null, i => prg3.Value = i, log); Rep.SaveChanges(); Helpers.Log.Add($"Data imported and ready to export"); Rep.AccountDataRecordRemove(acc.Data); } } finally { hostNames.ToList().ForEach((s) => { Rep.HostRemove(Rep.HostGet(s), saveAfterRemove: false); }); cityNames.ToList().ForEach((s) => { Rep.CityRemove(Rep.CityGet(s), saveAfterRemove: false); }); Rep.SaveChanges(); SqlLogEnabled = true; } }