private void Parse(CsvSettings csvSettings, DiagnosticTimer watch, string text, IntPtr bufferId)
{
var data = csvSettings.Parse(text);
watch.Checkpoint("Parse");
var columnTypes = new CsvColumnTypes(data, csvSettings);
try
{
Main.DataStorage.SaveData(bufferId, data, columnTypes);
}
catch (Exception ex)
{
this.ErrorMessage("Error when saving data to database:\n" + ex.Message);
return;
}
watch.Checkpoint("Saved to DB");
this.UiThread(() => txbQuery.Text = "SELECT * FROM THIS");
Execute(bufferId, watch);
var diagnostic = watch.LastCheckpoint("Resize");
Trace.TraceInformation(diagnostic);
if (Main.Settings.DebugMode)
{
this.Message(diagnostic);
}
}
public void CanParseMultilineFields()
{
var settings = new CsvSettings {
Separator = ',', UseQuotes = true
};
List <string[]> data;
using (var sr = new StreamReader(@"TestFiles\multline.csv")) data = settings.Parse(sr).ToList();
this.AssertDataEqual(_expectedData, data);
}
public void CanParse()
{
var csvText = "\"header1\",\"header2\"\n"
+ "\"text\",\"more text\"\n"
+ "\"here, might fail\",\":/\"";
var set = new CsvSettings {
Separator = ',', UseQuotes = true
};
// Act
var data = set.Parse(csvText);
// Assert
Assert.AreEqual(3, data.Count);
Assert.AreEqual(2, data[2].Length);
}
public void CanParseDifferent(string newline, char separator, bool quoted)
{
var indata = new List <string[]>
{
new[] { "A number", "another number here", "#¤%i" },
new[] { "1", "2g\"m", "3" },
new[] { "3", "12", "1,3\"" },
new[] { "4", "2", "3" }
};
var csvText = string.Join(newline, indata.Select(x => string.Join(separator.ToString(), x.Select(l => quoted ? $"\"{l.Replace("\"", "\"\"")}\"" : l))));
var set = new CsvSettings {
Separator = separator, UseQuotes = quoted, HasHeader = false
};
// Act
var data = set.Parse(csvText);
// Assert
this.AssertDataEqual(indata, data);
}
private void Parse(CsvSettings csvSettings, DiagnosticTimer watch, TextReader text, IntPtr bufferId)
{
IEnumerable <string[]> data;
try
{
data = csvSettings.Parse(text);
}
catch (Exception e)
{
this.ErrorMessage("Error when parsing text:\n" + e.Message);
return;
}
watch.Checkpoint("Parse");
var count = 0;
try
{
var first = true;
const int partitionSize = DataSettings.ChunkSize;
foreach (var chunk in data.Partition(partitionSize))
{
if (first)
{
var wholeChunk = chunk.ToList();
var columnTypes = new CsvColumnTypes(wholeChunk, csvSettings);
Main.DataStorage.SaveData(bufferId, wholeChunk, columnTypes);
first = false;
watch.Checkpoint("Saved first chunk to DB");
}
else
{
count += partitionSize;
var msg = $"Read lines: {count}";
this.UiThread(() => this.txbQuery.Text = msg);
Main.DataStorage.SaveMore(bufferId, chunk);
}
}
}
catch (Exception ex)
{
this.ErrorMessage("Error when saving data to database:\n" + ex.Message);
return;
}
watch.Checkpoint("Saved to DB");
var selectQuery = "SELECT * FROM THIS";
if (count > 10000)
{
selectQuery = Main.DataStorage.CreateLimitedSelect(10000);
}
this.UiThread(() => this.txbQuery.Text = selectQuery);
this.Execute(bufferId, watch);
var diagnostic = watch.LastCheckpoint("Resize");
Trace.TraceInformation(diagnostic);
if (Main.Settings.DebugMode)
{
this.Message(diagnostic);
}
}