public void OpenReadTestGZipSmallRead()
{
using (var res = new ImprovedStream(new SourceAccess(UnitTestInitializeCsv.GetTestPath("BasicCsV.txt.gz"), true)))
{
Assert.IsNotNull(res);
var result1 = new byte[2048];
using (var reader = new BinaryReader(res))
{
reader.Read(result1, 0, result1.Length);
}
// should return to teh start
res.Seek(0, SeekOrigin.Begin);
var result2 = new byte[2048];
using (var reader = new BinaryReader(res))
{
reader.Read(result2, 0, result2.Length);
}
Assert.AreEqual(result1[0], result2[0]);
Assert.AreEqual(result1[1], result2[1]);
Assert.AreEqual(result1[2], result2[2]);
Assert.AreEqual(result1[3], result2[3]);
Assert.AreEqual(result1[4], result2[4]);
Assert.AreEqual(result1[5], result2[5]);
}
}
public void ImprovedTextReaderPositionStoreTestFromMiddle()
{
using (var impStream = new ImprovedStream(new SourceAccess(UnitTestInitializeCsv.GetTestPath("txTranscripts.txt"), true)))
{
using (var test = new ImprovedTextReader(impStream, 65001, 1))
{
test.ToBeginning();
Assert.AreEqual(2, test.LineNumber);
Assert.AreEqual(
@"GCS_004805_Osipova 023c25d3-3420-449c-a75b-0d74d29ddc38 Completed 04/02/2008 00:00:00 04/02/2008 00:00:00 04/02/2008 00:00:00",
test.ReadLine());
var lastLine1 = string.Empty;
string lastLine2;
for (var i = 0; i < 5; i++)
{
lastLine1 = test.ReadLine();
}
var store = new ImprovedTextReaderPositionStore(test);
var readLine = false;
while (!store.AllRead())
{
lastLine2 = test.ReadLine();
// since there are buffers its we will not end up with the excact same line, but we need
// to at least have read the line.
if (lastLine2 == lastLine1)
{
readLine = true;
}
}
Assert.IsTrue(readLine);
}
}
}
private void WriteFile(string fileName, string password, string internalName, bool remove = false)
{
var fullname = UnitTestInitializeCsv.GetTestPath(fileName);
var encoding = EncodingHelper.GetEncoding(65001, true);
const string c_Line1 = "This is a test of compressed data written to a file";
const string c_Line2 = "Yet another line to be written";
const string c_Line3 = "A text with non ASCII characters: Raphael Nöldner";
var sourceAccsss = new SourceAccess(fullname, false);
if (string.IsNullOrEmpty(password))
{
sourceAccsss.EncryptedPassphrase = password;
}
if (!string.IsNullOrEmpty(internalName))
{
sourceAccsss.IdentifierInContainer = internalName;
}
using (var improvedStream = new ImprovedStream(sourceAccsss))
{
using (var writer = new StreamWriter(improvedStream, encoding, 8192))
{
writer.WriteLine(c_Line1);
writer.WriteLine(c_Line2);
writer.WriteLine(c_Line3);
writer.WriteLine();
writer.WriteLine(c_Line1);
}
improvedStream.Close();
}
Assert.IsTrue(FileSystemUtils.FileExists(fullname), "Check if File is created" + fileName);
sourceAccsss = new SourceAccess(fullname, true);
if (string.IsNullOrEmpty(password))
{
sourceAccsss.EncryptedPassphrase = password;
}
using (var improvedStream = new ImprovedStream(sourceAccsss))
{
using (var textReader = new StreamReader(improvedStream, encoding, true))
{
Assert.AreEqual(c_Line1, textReader.ReadLine(), "Line 1 : " + fileName);
Assert.AreEqual(c_Line2, textReader.ReadLine(), "Line 2 : " + fileName);
Assert.AreEqual(c_Line3, textReader.ReadLine(), "Line 3 : " + fileName);
Assert.AreEqual(string.Empty, textReader.ReadLine(), "Line 4 : " + fileName);
Assert.AreEqual(c_Line1, textReader.ReadLine(), "Line 5 : " + fileName);
}
improvedStream.Close();
}
if (remove)
{
FileSystemUtils.FileDelete(fullname);
}
}
public void OpenReadTestRegular()
{
using (var res = new ImprovedStream(new SourceAccess(UnitTestInitializeCsv.GetTestPath("BasicCsV.txt"), true)))
{
Assert.IsNotNull(res);
}
}
public void OpenReadTestGZipLargeRead()
{
using (var res = new ImprovedStream(new SourceAccess(UnitTestInitializeCsv.GetTestPath("Larger.json.gz"), true)))
{
Assert.IsNotNull(res);
var result1 = new byte[10000];
// read a potion that is larger than the buffered stream
using (var reader = new BinaryReader(res))
{
reader.Read(result1, 0, result1.Length);
}
// should return to the start
res.Seek(0, SeekOrigin.Begin);
var result2 = new byte[10000];
using (var reader = new BinaryReader(res))
{
reader.Read(result2, 0, result2.Length);
}
Assert.AreEqual(result1[0], result2[0]);
Assert.AreEqual(result1[1], result2[1]);
Assert.AreEqual(result1[2], result2[2]);
Assert.AreEqual(result1[3], result2[3]);
Assert.AreEqual(result1[4], result2[4]);
Assert.AreEqual(result1[5], result2[5]);
}
}
public void OpenReadTestgZip()
{
using (var res = ImprovedStream.OpenRead(Path.Combine(m_ApplicationDirectory, "BasicCsV.txt.gz")))
{
Assert.IsNotNull(res);
Assert.IsNotNull(res.Stream);
}
}
public void OpenReadTestPGP()
{
PGPKeyStorageTestHelper.SetApplicationSetting();
using (var res = ImprovedStream.OpenRead(Path.Combine(m_ApplicationDirectory, "BasicCsV.pgp"), () => { return("UGotMe".Encrypt()); }))
{
Assert.IsNotNull(res);
Assert.IsNotNull(res.Stream);
}
}
public void OpenReadTestSetting()
{
CsvFile setting = new CsvFile();
setting.FileName = Path.Combine(m_ApplicationDirectory, "BasicCsV.txt");
using (var res = ImprovedStream.OpenRead(setting))
{
Assert.IsNotNull(res);
Assert.IsNotNull(res.Stream);
}
}
public void OpenReadTestSetting()
{
var setting = new CsvFile {
FileName = UnitTestInitializeCsv.GetTestPath("BasicCsV.txt")
};
using (var res = new ImprovedStream(new SourceAccess(setting, true)))
{
Assert.IsNotNull(res);
}
}
public void ImprovedTextReaderTestGz()
{
using (var impStream = new ImprovedStream(new SourceAccess(UnitTestInitializeCsv.GetTestPath("BasicCsV.txt.gz"))))
{
using (var test = new ImprovedTextReader(impStream, 12000))
{
test.ToBeginning();
Assert.AreEqual(1, test.LineNumber);
Assert.AreEqual("ID,LangCodeID,ExamDate,Score,Proficiency,IsNativeLang", test.ReadLine());
Assert.AreEqual("1,German,20/01/2010,276,0.94,Y", test.ReadLine());
}
}
}
public void GuessHasHeader()
{
using (var stream = new ImprovedStream(new SourceAccess(UnitTestInitializeCsv.GetTestPath("BasicCSV.txt"))))
using (var reader = new ImprovedTextReader(stream))
{
reader.ToBeginning();
Assert.IsTrue(reader.GuessHasHeader("#", ",", UnitTestInitializeCsv.Token).Item1);
}
using (var stream =
new ImprovedStream(new SourceAccess(UnitTestInitializeCsv.GetTestPath("HandlingDuplicateColumnNames.txt"))))
using (var reader = new ImprovedTextReader(stream))
{
reader.ToBeginning();
Assert.IsFalse(reader.GuessHasHeader("#", ",", UnitTestInitializeCsv.Token).Item1);
}
}
public async System.Threading.Tasks.Task AnalyseStreamAsyncFile()
{
var stream = FileSystemUtils.OpenRead(UnitTestInitializeCsv.GetTestPath("BasicCSV.txt"));
ICollection <IColumn> determinedColumns;
// Not closing the stream
using (IProcessDisplay process = new CustomProcessDisplay(UnitTestInitializeCsv.Token))
{
using (var impStream = new ImprovedStream(stream))
{
var result = await impStream.GetDetectionResult("stream", process, false, true, true, true, true, true, false);
impStream.Seek(0, System.IO.SeekOrigin.Begin);
using (var reader = new CsvFileReader(impStream, result.CodePageId, result.SkipRows, result.HasFieldHeader,
new ColumnCollection(), TrimmingOption.Unquoted, result.FieldDelimiter, result.FieldQualifier,
result.EscapeCharacter, 0, false, false, result.CommentLine, 0, true, string.Empty, string.Empty,
string.Empty, true, false, true, false, false, false, false, false, false, true, true))
{
await reader.OpenAsync(process.CancellationToken);
var(info, columns) = await reader.FillGuessColumnFormatReaderAsyncReader(new FillGuessSettings(),
new ColumnCollection(), false, true, "null", process.CancellationToken);
determinedColumns = columns.ToList();
Assert.AreEqual(6, determinedColumns.Count(), "Recognized columns");
Assert.AreEqual(6, info.Count, "Information Lines");
}
impStream.Seek(0, SeekOrigin.Begin);
using (var reader = new CsvFileReader(impStream, result.CodePageId, result.SkipRows, result.HasFieldHeader,
determinedColumns, TrimmingOption.Unquoted, result.FieldDelimiter, result.FieldQualifier,
result.EscapeCharacter, 0, false, false, result.CommentLine, 0, true, string.Empty, string.Empty,
string.Empty, true, false, true, false, false, false, false, false, false, true, true))
{
await reader.OpenAsync(process.CancellationToken);
Assert.AreEqual(6, reader.FieldCount);
}
}
}
}
public void ImprovedTextReaderPositionStoreTest()
{
using (var impStream = new ImprovedStream(new SourceAccess(UnitTestInitializeCsv.GetTestPath("txTranscripts.txt"), true)))
{
using (var test = new ImprovedTextReader(impStream))
{
test.ToBeginning();
var store = new ImprovedTextReaderPositionStore(test);
Assert.AreEqual(1, test.LineNumber);
Assert.AreEqual(
"#UserID CurriculumID TranscriptStatus RequestDateTime RegistrationDateTime CompletionDateTime",
test.ReadLine());
var lastLine = string.Empty;
while (!store.AllRead())
{
lastLine = test.ReadLine();
}
Assert.AreEqual(
@"GCS_002846_Benavides A23c25d3-3420-449c-a75b-0d74d29ddc38 Completed 13/03/2008 00:00:00 13/03/2008 00:00:00 13/03/2008 00:00:00",
lastLine);
}
}
}
public void OpenReadTestZipSmallRead()
{
var sourceAccess = new SourceAccess(UnitTestInitializeCsv.GetTestPath("AllFormatsPipe.zip"), true);
// opeing without IdentifierInContainer should return teh first file entry
using (var res = new ImprovedStream(sourceAccess))
{
Assert.AreEqual("AllFormatsPipe.txt", sourceAccess.IdentifierInContainer);
Assert.IsNotNull(res);
var result1 = new byte[2048];
using (var reader = new BinaryReader(res))
{
reader.Read(result1, 0, result1.Length);
}
// should return to teh start
res.Seek(0, SeekOrigin.Begin);
var result2 = new byte[2048];
using (var reader = new BinaryReader(res))
{
reader.Read(result2, 0, result2.Length);
}
Assert.AreEqual(result1[0], result2[0]);
Assert.AreEqual(result1[1], result2[1]);
Assert.AreEqual(result1[2], result2[2]);
Assert.AreEqual(result1[3], result2[3]);
Assert.AreEqual(result1[4], result2[4]);
Assert.AreEqual(result1[5], result2[5]);
}
// now sourceAccess.IdentifierInContainer is set,
using (var res = new ImprovedStream(sourceAccess))
{
Assert.IsNotNull(res);
}
}
public void ToBeginningTest()
{
// use a file with a BOM
using (var impStream = new ImprovedStream(new SourceAccess(UnitTestInitializeCsv.GetTestPath("txTranscripts.txt"))))
{
using (var test = new ImprovedTextReader(impStream))
{
test.ToBeginning();
Assert.AreEqual(1, test.LineNumber);
Assert.AreEqual(
"#UserID CurriculumID TranscriptStatus RequestDateTime RegistrationDateTime CompletionDateTime",
test.ReadLine());
for (var i = 0; i < 200; i++)
{
_ = test.ReadLine();
}
test.ToBeginning();
Assert.AreEqual(1, test.LineNumber);
Assert.AreEqual(
"#UserID CurriculumID TranscriptStatus RequestDateTime RegistrationDateTime CompletionDateTime",
test.ReadLine());
for (var i = 0; i < 300; i++)
{
test.ReadLine();
}
test.ToBeginning();
Assert.AreEqual(1, test.LineNumber);
Assert.AreEqual(
"#UserID CurriculumID TranscriptStatus RequestDateTime RegistrationDateTime CompletionDateTime",
test.ReadLine());
}
}
}
public void DisposeTest()
{
var res = new ImprovedStream(new SourceAccess(UnitTestInitializeCsv.GetTestPath("BasicCsV.txt"), true));
res.Dispose();
}
public void DisposeTest()
{
var res = ImprovedStream.OpenRead(Path.Combine(m_ApplicationDirectory, "BasicCsV.txt"));
res.Dispose();
}
public async Task GuessJsonFileAsync()
{
using (var stream = new ImprovedStream(new SourceAccess(UnitTestInitializeCsv.GetTestPath("Jason1.json"))))
Assert.IsTrue(await stream.IsJsonReadable(Encoding.UTF8, UnitTestInitializeCsv.Token));
}
public async Task BOMTest()
{
// create files
var fn = new[]
{
new Tuple <string, int, byte[]>("GB18030", 54936,
new[] { (byte)0x84, (byte)0x31, (byte)0x95, (byte)0x33 }),
new Tuple <string, int, byte[]>("UTF-7_2", 65000,
new[] { (byte)0x2B, (byte)0x2F, (byte)0x76, (byte)0x39 }),
new Tuple <string, int, byte[]>("UTF-7_3", 65000,
new[] { (byte)0x2B, (byte)0x2F, (byte)0x76, (byte)0x2B }),
new Tuple <string, int, byte[]>("UTF-7_4", 65000,
new[] { (byte)0x2B, (byte)0x2F, (byte)0x76, (byte)0x2F }),
new Tuple <string, int, byte[]>("UTF-16 (BE)", 1201,
new[] { (byte)0xFE, (byte)0xFF }),
new Tuple <string, int, byte[]>("UTF8", 65001,
new[] { (byte)0xEF, (byte)0xBB, (byte)0xBF }),
new Tuple <string, int, byte[]>("UTF-16 (LE)", 1200,
new[] { (byte)0xFF, (byte)0xFE }),
new Tuple <string, int, byte[]>("UTF-32 (BE)", 12001,
new[] { (byte)0, (byte)0, (byte)0xFE, (byte)255 }),
new Tuple <string, int, byte[]>("UTF-32 (LE)", 12000,
new[] { (byte)0xFF, (byte)0xFE, (byte)0, (byte)0 }),
new Tuple <string, int, byte[]>("UTF-7_1", 65000,
new[] { (byte)0x2B, (byte)0x2F, (byte)0x76, (byte)0x38 })
};
//var Text = Encoding.ASCII.GetBytes("This is a test\\r\nLine2");
var line1 = "This is a test - First Line!";
var line2 = "Another line...";
foreach (var type in fn)
{
var fileName = UnitTestInitializeCsv.GetTestPath("Test_" + type.Item1 + ".txt");
using (var fs = new FileStream(fileName, FileMode.Create, FileAccess.Write))
{
// write the BOM
fs.Write(type.Item3, 0, type.Item3.Length);
using (var fs2 = new StreamWriter(fs, Encoding.GetEncoding(type.Item2)))
{
await fs2.WriteLineAsync(line1);
await fs2.WriteAsync(line2);
}
}
using (var impStream = new ImprovedStream(new SourceAccess(fileName)))
{
using (var test = new ImprovedTextReader(impStream, type.Item2))
{
test.ToBeginning();
Assert.AreEqual(1, test.LineNumber);
Assert.AreEqual(line1, test.ReadLine(), $"Issue reading Line1 {type.Item1}");
Assert.AreEqual(2, test.LineNumber);
Assert.AreEqual(line2, test.ReadLine(), $"Issue reading Line2 {type.Item1}");
test.ToBeginning();
Assert.AreEqual(1, test.LineNumber);
Assert.AreEqual(line1, test.ReadLine(), $"Issue reading after reset {type.Item1}");
}
}
File.Delete(fileName);
}
}
