public void DetermineColumnFormatGetSampleValues2()
{
using (var dt = new DataTable())
{
dt.Columns.Add(new DataColumn
{
ColumnName = "ID",
DataType = typeof(string)
});
for (var i = 0; i < 150; i++)
{
var row = dt.NewRow();
if (i == 10 || i == 47)
{
row[0] = "NULL";
}
row[0] = (i / 3).ToString(CultureInfo.InvariantCulture);
dt.Rows.Add(row);
}
using (var processDisplay = new DummyProcessDisplay())
{
var res = DetermineColumnFormat.GetSampleValues(dt, 0, 20, string.Empty, processDisplay.CancellationToken);
Assert.AreEqual(20, res.Count());
}
}
}
public async Task GetSourceColumnInformationTestAsync()
{
var setting = new CsvFile
{
ID = "ID122",
FileName = UnitTestInitializeCsv.GetTestPath("BasicCSV.txt"),
HasFieldHeader = true,
DisplayStartLineNo = false,
SqlStatement = "ID122",
FileFormat = { FieldDelimiter = "," }
};
using (var reader = new CsvFileReader(setting, null))
{
UnitTestInitializeCsv.MimicSQLReader.AddSetting(setting.ID,
await reader.GetDataTableAsync(0, false, setting.DisplayStartLineNo, setting.DisplayRecordNo,
setting.DisplayEndLineNo, false, null, UnitTestInitializeCsv.Token));
}
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
var res1 = await DetermineColumnFormat.GetWriterColumnInformationAsync(setting.SqlStatement, setting.Timeout,
setting.FileFormat.ValueFormatMutable, setting.ColumnCollection.ReadonlyCopy(),
processDisplay.CancellationToken);
Assert.AreEqual(6, res1.Count());
setting.SqlStatement = null;
var res2 = await DetermineColumnFormat.GetSqlColumnNamesAsync(setting.SqlStatement, setting.Timeout,
processDisplay.CancellationToken);
Assert.AreEqual(0, res2.Count());
}
}
public void StructuredFileWriterJSONEncodeTest()
{
var writeFile = new StructuredFile
{
ID = "Write",
FileName = Path.Combine(m_ApplicationDirectory, "StructuredFileOutputJSON.txt"),
SqlStatement = "Read",
InOverview = true,
JSONEncode = true
};
var cols = DetermineColumnFormat.GetWriterSourceColumns(writeFile, CancellationToken.None);
writeFile.Header = "{\"rowset\":[\n";
var sb = new StringBuilder("{");
// { "firstName":"John", "lastName":"Doe"},
foreach (var col in cols)
{
sb.AppendFormat("\"{0}\":\"{1}\", ", HTMLStyle.JsonElementName(col.Header),
string.Format(System.Globalization.CultureInfo.InvariantCulture, StructuredFileWriter.cFieldPlaceholderByName, col.Header));
}
if (sb.Length > 1)
{
sb.Length -= 2;
}
sb.AppendLine("},");
writeFile.Row = sb.ToString();
var writer = new StructuredFileWriter(writeFile, CancellationToken.None);
_ = writer.Write();
}
public void GuessColumnFormatNumbersAsDate()
{
string[] values =
{
"-130.66", "-7.02", "-19.99", "-131.73", "43478.5037152778", "35634.7884722222", "35717.2918634259",
"36858.2211226852", "43177.1338425925", "40568.3131481481", "37576.1801273148", "42573.3813078704",
"44119.8574189815", "40060.7079976852", "43840.2724884259", "38013.3021759259", "40422.7830671296",
"37365.5321643519", "34057.8838773148", "36490.4011805556", "40911.5474189815"
};
var res = DetermineColumnFormat.GuessValueFormat(
values,
4,
null,
"False",
false,
false,
false,
true,
false,
true,
false,
null,
UnitTestInitializeCsv.Token);
Assert.AreEqual(DataType.DateTime, res.FoundValueFormat?.DataType);
}
public void GuessColumnFormatNoSamples()
{
string[] values = { };
try
{
DetermineColumnFormat.GuessValueFormat(
values,
4,
null,
"False",
true,
false,
true,
true,
true,
false,
false,
null,
UnitTestInitializeCsv.Token);
Assert.Fail("Expected Exception not thrown");
}
catch (ArgumentNullException)
{
}
catch (Exception ex)
{
Assert.Fail("Wrong or exception thrown exception is : " + ex.GetType().Name);
}
}
public async Task DetermineColumnFormatGetSampleValuesAsync2Async()
{
using (var dt = new DataTable())
{
dt.Columns.Add(new DataColumn {
ColumnName = "ID", DataType = typeof(string)
});
for (var i = 0; i < 150; i++)
{
var row = dt.NewRow();
if (i == 10 || i == 47)
{
row[0] = "NULL";
}
row[0] = (i / 3).ToString(CultureInfo.InvariantCulture);
dt.Rows.Add(row);
}
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
using (var reader = new DataTableWrapper(dt))
{
string treatAsNull = string.Empty;
var res = await DetermineColumnFormat.GetSampleValuesAsync(reader, 0, new[] { 0 }, 20,
treatAsNull, processDisplay.CancellationToken);
Assert.IsTrue(res[0].RecordsRead >= 20);
Assert.AreEqual(20, res[0].Values.Count());
}
}
}
}
public void StructuredFileWriterXMLEncodeTest()
{
var writeFile = new StructuredFile
{
ID = "Write",
FileName = Path.Combine(m_ApplicationDirectory, "StructuredFileOutputXML.txt"),
SqlStatement = "Read",
InOverview = true,
JSONEncode = false
};
var cols = DetermineColumnFormat.GetWriterSourceColumns(writeFile, CancellationToken.None);
var sb = new StringBuilder();
sb.AppendLine("<?xml version=\"1.0\"?>\n");
sb.AppendLine("<rowset>");
writeFile.Header = sb.ToString();
sb.Clear();
sb.AppendLine(" <row>");
foreach (var col in cols)
{
sb.AppendFormat(" <{0}>{1}</{0}>\n", HTMLStyle.XmlElementName(col.Header),
string.Format(System.Globalization.CultureInfo.InvariantCulture, StructuredFileWriter.cFieldPlaceholderByName, col.Header));
}
sb.AppendLine(" </row>");
writeFile.Row = sb.ToString();
writeFile.Footer = "</rowset>";
var writer = new StructuredFileWriter(writeFile, CancellationToken.None);
_ = writer.Write();
}
public async Task GetSqlColumnNamesAsyncParameter()
{
using (var dummy = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
var backup = FunctionalDI.SQLDataReader;
try
{
FunctionalDI.SQLDataReader = null;
await DetermineColumnFormat.GetSqlColumnNamesAsync("Nonsense SQL", 60, dummy.CancellationToken);
Assert.Fail("Expected Exception not thrown");
}
catch (FileWriterException)
{
// add good
}
catch (Exception ex)
{
Assert.Fail("Wrong Exception Type: " + ex.GetType());
}
finally
{
FunctionalDI.SQLDataReader = backup;
}
}
}
public async Task GetSampleValuesAsyncTest()
{
using (var dt = UnitTestStatic.GetDataTable(1000))
{
using (var dummy = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
var reader = new DataTableWrapper(dt);
foreach (DataColumn col in dt.Columns)
{
var res = await DetermineColumnFormat.GetSampleValuesAsync(reader, 10,
new[] { col.Ordinal }, 10, "null",
dummy.CancellationToken);
if (col.ColumnName != "AllEmpty")
{
Assert.AreNotEqual(0, res[col.Ordinal].Values.Count, col.ColumnName);
}
else
{
Assert.AreEqual(0, res[col.Ordinal].Values.Count, col.ColumnName);
}
}
}
}
}
public void GuessColumnFormatMMddyyyySuggestion()
{
string[] values = { "01/02/2010", "02/12/2012" };
var res = DetermineColumnFormat.GuessValueFormat(
values,
4,
null,
"false",
true,
false,
true,
true,
true,
false,
false,
new ValueFormatMutable()
{
DataType = DataType.DateTime, DateFormat = "MM/dd/yyyy", DateSeparator = "/"
},
UnitTestInitializeCsv.Token);
Assert.AreEqual(DataType.DateTime, res.FoundValueFormat?.DataType);
Assert.AreEqual(@"MM/dd/yyyy", res.FoundValueFormat?.DateFormat);
Assert.AreEqual("/", res.FoundValueFormat?.DateSeparator);
}
public void GuessColumnFormatMMddyyyyNotenough()
{
string[] values = { "01/02/2010", "02/12/2012" };
var res = DetermineColumnFormat.GuessValueFormat(values, 4, null, "false", true, false, true,
true, true, false, false, null, CancellationToken.None);
Assert.IsFalse(res?.PossibleMatch ?? false);
}
public void GuessColumnFormatNumeric()
{
string[] values = { "1", "2.5", "3", "4", "5.3" };
var res = DetermineColumnFormat.GuessValueFormat(values, 4, null, "False", true, false, true,
true, true, false, false, null, CancellationToken.None);
Assert.AreEqual(DataType.Numeric, res.FoundValueFormat.DataType);
}
public void GuessColumnFormatNoSamples()
{
string[] values = { };
var res = DetermineColumnFormat.GuessValueFormat(values, 4, null, "False", true, false, true,
true, true, false, false, null, CancellationToken.None);
Assert.IsNull(res);
}
public void GuessColumnFormatInteger2()
{
string[] values = { "-1", " 2", "3 ", "4", "100", "10" };
var res = DetermineColumnFormat.GuessValueFormat(values, 4, null, "False", true, false, true,
true, true, false, false, null, CancellationToken.None);
Assert.AreEqual(DataType.Integer, res.FoundValueFormat.DataType);
}
public void GuessColumnFormatGuid()
{
string[] values = { "{0799A029-8B85-4589-8341-C7038AFF5B48}", "99DDD263-2E2D-434F-9265-33CF893B02DF" };
var res = DetermineColumnFormat.GuessValueFormat(values, 4, null, "false", false, true, false,
false, false, false, false, null, CancellationToken.None);
Assert.AreEqual(DataType.Guid, res.FoundValueFormat.DataType);
}
public void GuessColumnFormatDateNotMatching()
{
string[] values = { "01/02/2010", "14/02/2012", "02/14/2012" };
var res = DetermineColumnFormat.GuessValueFormat(values, 4, null, "false", true, false, true,
true, true, false, false, null, CancellationToken.None);
Assert.IsNull(res);
}
public void GuessColumnFormatBoolean2()
{
string[] values = { "Yes", "No" };
var res = DetermineColumnFormat.GuessValueFormat(values, 2, null, "False", true, false, true,
true, true, false, false, null, CancellationToken.None);
Assert.AreEqual(DataType.Boolean, res.FoundValueFormat.DataType);
}
public void GuessColumnFormatVersionNumbers()
{
string[] values = { "1.0.1.2", "1.0.2.1", "1.0.2.2", "1.0.2.3", "1.0.2.3" };
var res = DetermineColumnFormat.GuessValueFormat(values, 4, null, "False", false, false, true,
false, false, false, false, null, CancellationToken.None);
Assert.IsTrue(res == null || res.FoundValueFormat.DataType != DataType.Integer);
}
public void GuessColumnFormatText()
{
string[] values = { "Hallo", "Welt" };
var res = DetermineColumnFormat.GuessValueFormat(values, 4, null, "false", true, false, true,
true, true, false, false, null, CancellationToken.None);
Assert.AreEqual(DataType.String, res.FoundValueFormat.DataType);
}
public void GuessColumnFormatMMddyyyy()
{
string[] values = { "01/02/2010", "02/14/2012", "02/17/2012", "02/22/2012", "03/01/2012" };
var res = DetermineColumnFormat.GuessValueFormat(values, 4, null, "false", true, false, true,
true, true, false, false, null, CancellationToken.None);
Assert.AreEqual(DataType.DateTime, res.FoundValueFormat.DataType);
Assert.AreEqual(@"MM/dd/yyyy", res.FoundValueFormat.DateFormat);
Assert.AreEqual("/", res.FoundValueFormat.DateSeparator);
}
public void GuessColumnFormatddMMyyyy2()
{
string[] values = { "01.02.2010", "14.02.2012", "16.02.2012", "01.04.2014", "31.12.2010" };
var res = DetermineColumnFormat.GuessValueFormat(values, 4, null, "false", true, false, true,
true, true, false, false, null, CancellationToken.None);
Assert.AreEqual(DataType.DateTime, res.FoundValueFormat.DataType);
Assert.AreEqual(@"dd/MM/yyyy", res.FoundValueFormat.DateFormat);
Assert.AreEqual(".", res.FoundValueFormat.DateSeparator);
}
public void DetermineColumnFormatGetSampleValues()
{
using (var dt = new DataTable())
{
using (var processDisplay = new DummyProcessDisplay())
{
var res = DetermineColumnFormat.GetSampleValues(dt, 0, 20, string.Empty, processDisplay.CancellationToken);
Assert.AreEqual(0, res.Count());
}
}
}
public void GetAllPossibleFormatsTest()
{
var res1 = DetermineColumnFormat.GetAllPossibleFormats("1/1/2020");
Assert.AreEqual(2, res1.Count());
var res2 = DetermineColumnFormat.GetAllPossibleFormats("01/01/2020");
Assert.AreEqual(4, res2.Count());
var res3 = DetermineColumnFormat.GetAllPossibleFormats("30/1/2020");
Assert.AreEqual(1, res3.Count());
}
public void GetSampleValuesFileEmpty()
{
var setting = new CsvFile
{
FileName = Path.Combine(m_ApplicationDirectory, "CSVTestEmpty.txt"),
HasFieldHeader = true
};
using (var test = new CsvFileReader(setting))
{
test.Open(false, CancellationToken.None);
var samples = DetermineColumnFormat.GetSampleValues(test, 100, 0, 20, "NULL", CancellationToken.None);
Assert.AreEqual(0, samples.Count());
}
}
public void GetAllPossibleFormatsD()
{
var res = DetermineColumnFormat.GetAllPossibleFormats("30-Oct-18 04:26:28 AM");
var found = false;
foreach (var item in res)
{
if (item.DateFormat.Equals("dd/MMM/yy HH:mm:ss tt", System.StringComparison.InvariantCulture) && item.DateSeparator == "-")
{
found = true;
break;
}
}
Assert.IsTrue(found);
}
public async Task FillGuessColumnFormatReaderAsync_Parameter()
{
var fillGuessSettings = new FillGuessSettings
{
DetectNumbers = true,
DetectDateTime = true,
DetectPercentage = true,
DetectBoolean = true,
DetectGUID = true,
IgnoreIdColumns = true
};
var setting = new CsvFile
{
FileName = UnitTestInitializeCsv.GetTestPath("AllFormatsColon.txt"),
HasFieldHeader = true,
ByteOrderMark = true,
FileFormat = { FieldDelimiter = "," },
};
try
{
// ReSharper disable once AssignNullToNotNullAttribute
await DetermineColumnFormat.FillGuessColumnFormatReaderAsync(null, true, true, fillGuessSettings,
UnitTestInitializeCsv.Token);
}
catch (ArgumentNullException)
{
}
catch (Exception ex)
{
Assert.Fail("Wrong or exception thrown exception is : " + ex.GetType().Name);
}
try
{
// ReSharper disable once AssignNullToNotNullAttribute
await setting.FillGuessColumnFormatReaderAsync(true, true, null, UnitTestInitializeCsv.Token);
}
catch (ArgumentNullException)
{
}
catch (Exception ex)
{
Assert.Fail("Wrong or exception thrown exception is : " + ex.GetType().Name);
}
}
public async Task DetermineColumnFormatGetSampleValuesNoColumns()
{
using (var dt = new DataTable())
{
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
using (var reader = new DataTableWrapper(dt))
{
var temp = await DetermineColumnFormat
.GetSampleValuesAsync(reader, 0, new[] { 0 }, 20, null,
processDisplay.CancellationToken).ConfigureAwait(false);
Assert.AreEqual(0, temp.Count);
}
}
}
}
public async Task GetSampleValuesAsync()
{
using (var dataTable = UnitTestStatic.GetDataTable(150, false))
{
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
using (var reader = new DataTableWrapper(dataTable))
{
var res = await DetermineColumnFormat
.GetSampleValuesAsync(reader, 100, new[] { 0, 1 }, 20, null, processDisplay.CancellationToken)
.ConfigureAwait(false);
Assert.AreEqual(20, res[0].Values.Count);
}
}
}
}
public void GetSampleValuesByColIndex()
{
var setting = new CsvFile
{
FileName = Path.Combine(m_ApplicationDirectory, "BasicCSV.txt"),
HasFieldHeader = true
};
using (var test = new CsvFileReader(setting))
{
test.Open(false, CancellationToken.None);
var samples = DetermineColumnFormat.GetSampleValues(test, 1000, 0, 20, "NULL", CancellationToken.None);
Assert.AreEqual(7, samples.Count());
Assert.IsTrue(samples.Contains("1"));
Assert.IsTrue(samples.Contains("4"));
}
}
public async Task GetSourceColumnInformationTestAsync_Parameter()
{
try
{
// ReSharper disable once AssignNullToNotNullAttribute
await DetermineColumnFormat.GetWriterColumnInformationAsync("Nonsense SQL", 60, null, new List <IColumn>(),
UnitTestInitializeCsv.Token);
Assert.Fail("Expected Exception not thrown");
}
catch (ArgumentNullException)
{
// add good
}
catch (Exception ex)
{
Assert.Fail("Wrong Exception Type: " + ex.GetType());
}
}