public async Task GetDataTableAsync3()
{
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
var test3 = new CsvFile(UnitTestInitializeCsv.GetTestPath("WithEoFChar.txt"))
{
FileFormat = { FieldDelimiter = "Tab" }
};
test3.ColumnCollection.Add(new Column("Memo")
{
Ignore = true
});
using (var test = new CsvFileReader(test3, processDisplay))
{
await test.OpenAsync(processDisplay.CancellationToken);
var dt = await test.GetDataTableAsync(-1, true, true, true, true, true, null,
processDisplay.CancellationToken);
// 10 columns 1 ignored one added for Start line one for Error Field one for Record No one
// for Line end
Assert.AreEqual((10 - 1) + 4, dt.Columns.Count);
Assert.AreEqual(19, dt.Rows.Count);
}
}
}
public async Task FileCopy()
{
var dest = UnitTestInitializeCsv.GetTestPath("xyz.txt");
try
{
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
processDisplay.Maximum = -100;
Assert.IsFalse(FileSystemUtils.FileExists(dest));
await FileSystemUtils.FileCopy(UnitTestInitializeCsv.GetTestPath("AllFormats.txt"), dest, false,
processDisplay);
Assert.IsTrue(FileSystemUtils.FileExists(dest));
Assert.AreEqual(-100, processDisplay.Maximum);
// Copy again, the old file should be overwritten
await FileSystemUtils.FileCopy(UnitTestInitializeCsv.GetTestPath("AlternateTextQualifiers.txt"), dest, true,
processDisplay);
Assert.IsTrue(FileSystemUtils.FileExists(dest));
Assert.AreEqual(new FileInfo(UnitTestInitializeCsv.GetTestPath("AlternateTextQualifiers.txt")).Length,
new FileInfo(dest).Length);
}
}
finally
{
FileSystemUtils.FileDelete(dest);
}
}
public async Task WriteDataTableAsync()
{
using (var dataTable = new DataTable {
TableName = "DataTable", Locale = CultureInfo.InvariantCulture
})
{
dataTable.Columns.Add("ID", typeof(int));
dataTable.Columns.Add("Text", typeof(string));
for (var i = 0; i < 100; i++)
{
var row = dataTable.NewRow();
row["ID"] = i;
row["Text"] = i.ToString(CultureInfo.CurrentCulture);
dataTable.Rows.Add(row);
}
var writeFile = new CsvFile {
ID = "Test.txt", FileName = UnitTestInitializeCsv.GetTestPath("Test.txt"), SqlStatement = "Hello"
};
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
var writer = new CsvFileWriter(writeFile, processDisplay);
using (var reader = new DataTableWrapper(dataTable))
{
// await reader.OpenAsync(processDisplay.CancellationToken);
Assert.AreEqual(100, await writer.WriteAsync(reader, processDisplay.CancellationToken));
}
}
Assert.IsTrue(File.Exists(writeFile.FileName));
}
}
public async Task ReadJSon1TypedAsync()
{
var setting =
new CsvFile(UnitTestInitializeCsv.GetTestPath("Larger.json"))
{
JsonFormat = true
};
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
using (var jfr = new JsonFileReader(setting, processDisplay))
{
await jfr.OpenAsync(processDisplay.CancellationToken);
await jfr.ReadAsync(processDisplay.CancellationToken);
Assert.AreEqual(new Guid("ef21069c-3d93-4e07-878d-00e820727f65"), jfr.GetGuid(0));
Assert.IsTrue((new DateTime(2020, 04, 03, 18, 45, 29, 573, DateTimeKind.Utc) -
jfr.GetDateTime(1).ToUniversalTime()).TotalSeconds < 2);
Assert.AreEqual((short)0, jfr.GetInt16(5));
Assert.AreEqual((int)0, jfr.GetInt32(5));
Assert.AreEqual((long)0, jfr.GetInt64(5));
var val = new object[jfr.FieldCount];
jfr.GetValues(val);
Assert.IsNull(val[2]);
}
}
}
public async Task GetSampleValuesAsyncTest()
{
using (var dt = UnitTestStatic.GetDataTable(1000))
{
using (var dummy = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
var reader = new DataTableWrapper(dt);
foreach (DataColumn col in dt.Columns)
{
var res = await DetermineColumnFormat.GetSampleValuesAsync(reader, 10,
new[] { col.Ordinal }, 10, "null",
dummy.CancellationToken);
if (col.ColumnName != "AllEmpty")
{
Assert.AreNotEqual(0, res[col.Ordinal].Values.Count, col.ColumnName);
}
else
{
Assert.AreEqual(0, res[col.Ordinal].Values.Count, col.ColumnName);
}
}
}
}
}
public async Task DetermineColumnFormatGetSampleValuesAsync2Async()
{
using (var dt = new DataTable())
{
dt.Columns.Add(new DataColumn {
ColumnName = "ID", DataType = typeof(string)
});
for (var i = 0; i < 150; i++)
{
var row = dt.NewRow();
if (i == 10 || i == 47)
{
row[0] = "NULL";
}
row[0] = (i / 3).ToString(CultureInfo.InvariantCulture);
dt.Rows.Add(row);
}
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
using (var reader = new DataTableWrapper(dt))
{
string treatAsNull = string.Empty;
var res = await DetermineColumnFormat.GetSampleValuesAsync(reader, 0, new[] { 0 }, 20,
treatAsNull, processDisplay.CancellationToken);
Assert.IsTrue(res[0].RecordsRead >= 20);
Assert.AreEqual(20, res[0].Values.Count());
}
}
}
}
public void SetProcessTest()
{
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
processDisplay.SetProcess("Test", -1, true);
}
}
public async Task ColumnErrorDictionaryTest1Async()
{
var setting = new CsvFile
{
FileName = UnitTestInitializeCsv.GetTestPath("Sessions.txt"),
HasFieldHeader = true,
ByteOrderMark = true,
FileFormat = { FieldDelimiter = "\t" }
};
setting.ColumnCollection.AddIfNew(new Column("Start Date")
{
Ignore = true
});
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
using (var reader = new CsvFileReader(setting, processDisplay))
{
await reader.OpenAsync(processDisplay.CancellationToken);
var test1 = new ColumnErrorDictionary(reader);
Assert.IsNotNull(test1);
// Message in ignored column
reader.HandleWarning(0, "Msg1");
reader.HandleError(1, "Msg2");
Assert.AreEqual("Msg2", test1.Display);
}
}
public async Task GetSourceColumnInformationTestAsync()
{
var setting = new CsvFile
{
ID = "ID122",
FileName = UnitTestInitializeCsv.GetTestPath("BasicCSV.txt"),
HasFieldHeader = true,
DisplayStartLineNo = false,
SqlStatement = "ID122",
FileFormat = { FieldDelimiter = "," }
};
using (var reader = new CsvFileReader(setting, null))
{
UnitTestInitializeCsv.MimicSQLReader.AddSetting(setting.ID,
await reader.GetDataTableAsync(0, false, setting.DisplayStartLineNo, setting.DisplayRecordNo,
setting.DisplayEndLineNo, false, null, UnitTestInitializeCsv.Token));
}
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
var res1 = await DetermineColumnFormat.GetWriterColumnInformationAsync(setting.SqlStatement, setting.Timeout,
setting.FileFormat.ValueFormatMutable, setting.ColumnCollection.ReadonlyCopy(),
processDisplay.CancellationToken);
Assert.AreEqual(6, res1.Count());
setting.SqlStatement = null;
var res2 = await DetermineColumnFormat.GetSqlColumnNamesAsync(setting.SqlStatement, setting.Timeout,
processDisplay.CancellationToken);
Assert.AreEqual(0, res2.Count());
}
}
public async Task StartAsyncTestAsync()
{
var myDataTable = new DataTable();
bool beginCalled = false;
bool finishedCalled = false;
bool warningCalled = false;
bool refeshCalled = false;
Task refeshFunc(FilterType FilterType, CancellationToken CancellationToken) => Task.Run(() => refeshCalled = true);
using (var tsde = new TwoStepDataTableLoader(dt => myDataTable = dt, () => myDataTable, refeshFunc, null, () => beginCalled = true, (x) => finishedCalled = true))
{
var csv = new CsvFile
{
FileName = UnitTestInitializeCsv.GetTestPath("BasicCSV.txt"),
FileFormat = { FieldDelimiter = ",", CommentLine = "#" }
};
using (var proc = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
await tsde.StartAsync(csv, true, TimeSpan.FromMilliseconds(20), proc, (sender, args) => { warningCalled = true; });
Assert.IsTrue(refeshCalled);
Assert.IsFalse(warningCalled);
Assert.IsTrue(beginCalled);
Assert.IsTrue(finishedCalled);
Assert.AreEqual(8, myDataTable.Columns.Count());
}
}
}
public async Task MiscTest()
{
using (var process = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
using (var reader = new CsvFileReader(UnitTestHelper.ReaderGetAllFormats(), process))
{
await reader.OpenAsync(process.CancellationToken);
var wrapper = new DataReaderWrapper(reader);
Assert.IsTrue(wrapper.HasRows);
try
{
var test = wrapper.GetEnumerator();
}
catch (NotImplementedException)
{
// ignore
}
try
{
wrapper.GetValues(new object[10]);
}
catch (NotImplementedException)
{
// ignore
}
}
}
public async Task OpenLogAsync()
{
var setting = new CsvFile(UnitTestInitializeCsv.GetTestPath("LogFile.json"))
{
JsonFormat = true
};
using (var dpd = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
using (var jfr = new JsonFileReader(setting, dpd))
{
await jfr.OpenAsync(dpd.CancellationToken);
await jfr.ReadAsync(dpd.CancellationToken);
Assert.AreEqual("level", jfr.GetColumn(1).Name);
Assert.AreEqual("Error", jfr.GetValue(1));
await jfr.ReadAsync(dpd.CancellationToken);
Assert.AreEqual("Reading EdgeAPI vw_rpt_transcript", jfr.GetValue(2));
await jfr.ReadAsync(dpd.CancellationToken);
Assert.AreEqual("System.Data.DataException", jfr.GetValue(4));
}
}
public async Task ReadJSon3Async()
{
var setting = new CsvFile(UnitTestInitializeCsv.GetTestPath("Jason3.json"))
{
JsonFormat = true
};
using (var dpd = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
using (var jfr = new JsonFileReader(setting, dpd))
{
await jfr.OpenAsync(dpd.CancellationToken);
Assert.AreEqual(2, jfr.FieldCount);
await jfr.ReadAsync(dpd.CancellationToken);
Assert.AreEqual(28L, jfr.GetValue(0));
await jfr.ReadAsync(dpd.CancellationToken);
Assert.AreEqual(56L, jfr.GetValue(0));
while (await jfr.ReadAsync(dpd.CancellationToken))
{
}
Assert.AreEqual(5, jfr.RecordNumber);
}
}
public async Task ReadJSon2Async()
{
var setting = new CsvFile(UnitTestInitializeCsv.GetTestPath("Jason2.json"))
{
JsonFormat = true
};
using (var dpd = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
using (var jfr = new JsonFileReader(setting, dpd))
{
await jfr.OpenAsync(dpd.CancellationToken);
Assert.AreEqual(7, jfr.FieldCount);
await jfr.ReadAsync(dpd.CancellationToken);
await jfr.ReadAsync(dpd.CancellationToken);
Assert.AreEqual("Loading defaults C:\\Users\\rnoldner\\AppData\\Roaming\\CSVFileValidator\\Setting.xml",
jfr.GetValue(6));
while (await jfr.ReadAsync(dpd.CancellationToken))
{
}
Assert.AreEqual(29, jfr.RecordNumber);
}
}
public async Task DetermineColumnFormatFillGuessColumnFormatWriterAsync()
{
var setting = new CsvFile
{
ID = "DetermineColumnFormatFillGuessColumnFormatWriter",
FileName = UnitTestInitializeCsv.GetTestPath("BasicCSV.txt"),
DisplayStartLineNo = false,
HasFieldHeader = true,
FileFormat = { FieldDelimiter = "," }
};
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
using (var reader = new CsvFileReader(setting, null))
{
await reader.OpenAsync(processDisplay.CancellationToken);
UnitTestInitializeCsv.MimicSQLReader.AddSetting(setting.ID,
await reader.GetDataTableAsync(0, false, setting.DisplayStartLineNo, setting.DisplayRecordNo,
setting.DisplayEndLineNo, false, null, UnitTestInitializeCsv.Token));
}
var writer = new CsvFile {
SqlStatement = setting.ID
};
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
await writer.FillGuessColumnFormatWriterAsync(true, processDisplay);
Assert.AreEqual(6, writer.ColumnCollection.Count);
}
}
public async Task GetSqlColumnNamesAsyncParameter()
{
using (var dummy = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
var backup = FunctionalDI.SQLDataReader;
try
{
FunctionalDI.SQLDataReader = null;
await DetermineColumnFormat.GetSqlColumnNamesAsync("Nonsense SQL", 60, dummy.CancellationToken);
Assert.Fail("Expected Exception not thrown");
}
catch (FileWriterException)
{
// add good
}
catch (Exception ex)
{
Assert.Fail("Wrong Exception Type: " + ex.GetType());
}
finally
{
FunctionalDI.SQLDataReader = backup;
}
}
}
public void SetProcessTest1()
{
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
processDisplay.Maximum = 5;
processDisplay.SetProcess("Test", 100, true);
}
}
public void SetProcessTest2()
{
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
processDisplay.Maximum = 5;
processDisplay.SetProcess(null, new ProgressEventArgs("Hallo", 2, false));
}
}
public void GetFileWriter()
{
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
var res = FunctionalDI.GetFileWriter(m_CsvFile, processDisplay);
Assert.IsInstanceOfType(res, typeof(IFileWriter));
}
}
public async Task AllFormatsPipeReaderAsync()
{
var setting = new CsvFile(UnitTestInitializeCsv.GetTestPath("RealignColumn.txt"))
{
HasFieldHeader = true,
FileFormat = { FieldDelimiter = "\t" },
TryToSolveMoreColumns = true,
AllowRowCombining = true,
SkipEmptyLines = false
};
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
using (var test = new CsvFileReader(setting, processDisplay))
{
await test.OpenAsync(processDisplay.CancellationToken);
// first five rows are good.
await test.ReadAsync(processDisplay.CancellationToken); // Line 2
await test.ReadAsync(processDisplay.CancellationToken); // Line 3
await test.ReadAsync(processDisplay.CancellationToken); // Line 4
await test.ReadAsync(processDisplay.CancellationToken); // Line 5
await test.ReadAsync(processDisplay.CancellationToken); // Line 6
// Issue row Column 3 = Text|6
await test.ReadAsync(processDisplay.CancellationToken); // Line 6
Assert.AreEqual("Text\tF", test.GetValue(3));
await test.ReadAsync(processDisplay.CancellationToken); // Line 7
await test.ReadAsync(processDisplay.CancellationToken); // Line 8
await test.ReadAsync(processDisplay.CancellationToken); // Line 9
await test.ReadAsync(processDisplay.CancellationToken); // Line 10
await test.ReadAsync(processDisplay.CancellationToken); // Line 11
Assert.AreEqual("Memo: A long text, \t multiple words 11", test.GetValue(5));
await test.ReadAsync(processDisplay.CancellationToken); // Line 12
await test.ReadAsync(processDisplay.CancellationToken); // Line 13
await test.ReadAsync(processDisplay.CancellationToken); // Line 14
await test.ReadAsync(processDisplay.CancellationToken); // Line 15
await test.ReadAsync(processDisplay.CancellationToken); // Line 16
await test.ReadAsync(processDisplay.CancellationToken); // Line 17
Assert.AreEqual("Memo: A long text\nmultiple words 17", test.GetValue(5));
}
}
public void GetFileWriter()
{
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
m_StructuredFile.SqlStatement = "dummy";
var res = FunctionalDI.GetFileWriter(m_StructuredFile, processDisplay);
Assert.IsInstanceOfType(res, typeof(IFileWriter));
}
}
public async Task NewCsvFileGuessAllTestEmptyAsync()
{
using (var display = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
var det = await CsvHelper.GetDetectionResultFromFile(UnitTestInitializeCsv.GetTestPath("CSVTestEmpty.txt"), display);
Assert.AreEqual(0, det.SkipRows);
}
}
public async Task NewCsvFileGuessAllSmallFile()
{
using (var display = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
var det = await CsvHelper.GetDetectionResultFromFile(UnitTestInitializeCsv.GetTestPath("employee.txt"), display, true);
Assert.AreEqual(RecordDelimiterType.CRLF, det.NewLine);
}
}
public void GetFileReader()
{
m_CsvFile.FileName = UnitTestInitializeCsv.GetTestPath("BasicCSV.txt");
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
using (var res = FunctionalDI.GetFileReader(m_CsvFile, TimeZoneInfo.Local.Id, processDisplay))
{
Assert.IsInstanceOfType(res, typeof(IFileReader));
}
}
public async Task GetCsvFileSettingFromExtensionAsync()
{
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
var tuple = await CsvHelper.AnalyseFileAsync(UnitTestInitializeCsv.GetTestPath("BasicCSV.txt" + CsvFile.cCsvSettingExtension), true, true, true,
true, true, true, true, new FillGuessSettings(), processDisplay);
Assert.AreEqual(1200, tuple.CodePageId);
}
}
public async Task NewCsvFileGuessAllHeadingsAsync()
{
using (var display = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
var det = await CsvHelper.GetDetectionResultFromFile(UnitTestInitializeCsv.GetTestPath("BasicCSV.txt"), display);
Assert.AreEqual(0, det.SkipRows);
Assert.AreEqual(",".WrittenPunctuationToChar(), det.FieldDelimiter.WrittenPunctuationToChar());
Assert.AreEqual(1200, det.CodePageId); // UTF16_LE
}
}
public async System.Threading.Tasks.Task GetCsvFileSetting()
{
using (IProcessDisplay process = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
var result = await CsvHelper.AnalyseFileAsync(UnitTestInitializeCsv.GetTestPath("BasicCSV.txt"), true, true, true, true, true, true, false,
new FillGuessSettings(), process);
Assert.IsTrue(result.HasFieldHeader);
Assert.AreEqual(1200, result.CodePageId);
}
}
public async Task DepthTest()
{
using (var process = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
using (var reader = new CsvFileReader(UnitTestHelper.ReaderGetAllFormats(), process))
{
await reader.OpenAsync(process.CancellationToken);
var wrapper = new DataReaderWrapper(reader);
Assert.AreEqual(9, wrapper.Depth);
}
}
public void CustomProcessDisplayTest()
{
var called = false;
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
processDisplay.Progress += (sender, args) => { called = true; };
processDisplay.SetProcess("Test", -1, true);
Assert.IsTrue(called);
}
}
public void SetMaximum()
{
using (var processDisplay = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
processDisplay.Maximum = 666;
Assert.AreEqual(666, processDisplay.Maximum);
processDisplay.Maximum = -1;
Assert.AreEqual(-1, processDisplay.Maximum);
}
}
