public void ReaderTest()
{
/* CSV Contents:
* foo,bar, whitespace ," quotes "" , and commas "
* "quoted, with
* newline"
*
* ,
* ,,
* blarg
* blarg,"quoted, with
* newline"
* "quoted, with
* newline",blarg
* blarg,"quoted, with
* newline",blarg
*/
CommaSeparatedValueReader reader = new CommaSeparatedValueReader();
string filename = Path.Combine(InputDirectory, WellFormedCsvFile);
var expected = new List <string[]>(new string[][] {
new string[] { "foo", "bar", " whitespace ", @" quotes "" , and commas " },
new string[] { "quoted, with\r\nnewline" },
new string[] { "" }, //empty line
new string[] { "", "" }, //single ,
new string[] { "", "", "" }, //,,
new string[] { "blarg" },
new string[] { "blarg", "quoted, with\r\nnewline" },
new string[] { "quoted, with\r\nnewline", "blarg" },
new string[] { "blarg", "quoted, with\r\nnewline", "blarg" }
});
reader.LoadFile(filename);
int i = 0;
foreach (var row in reader)
{
IList <string> expectedRow = expected[i] as IList <string>;
Assert.AreEqual(expectedRow.Count, row.Count,
string.Format("Number of actual fields differs from number of expected at line {0} in file {1}", i, filename));
for (int j = 0; j < row.Count; j++)
{
Assert.AreEqual(expectedRow[j], row[j]);
}
i++;
}
}
public void ReaderTest()
{
/* CSV Contents:
foo,bar, whitespace ," quotes "" , and commas "
"quoted, with
newline"
,
,,
blarg
blarg,"quoted, with
newline"
"quoted, with
newline",blarg
blarg,"quoted, with
newline",blarg
*/
CommaSeparatedValueReader reader = new CommaSeparatedValueReader();
string filename = Path.Combine(InputDirectory, WellFormedCsvFile);
var expected = new List<string[]>(new string[][] {
new string[] { "foo", "bar", " whitespace ", @" quotes "" , and commas " },
new string[] { "quoted, with\r\nnewline" },
new string[] { "" }, //empty line
new string[] { "", "" }, //single ,
new string[] { "", "", "" }, //,,
new string[] { "blarg" },
new string[] { "blarg", "quoted, with\r\nnewline" },
new string[] { "quoted, with\r\nnewline", "blarg" },
new string[] { "blarg", "quoted, with\r\nnewline", "blarg" }
});
reader.LoadFile(filename);
int i = 0;
foreach (var row in reader)
{
IList<string> expectedRow = expected[i] as IList<string>;
Assert.AreEqual(expectedRow.Count, row.Count,
string.Format("Number of actual fields differs from number of expected at line {0} in file {1}", i, filename));
for (int j = 0; j < row.Count; j++)
{
Assert.AreEqual(expectedRow[j], row[j]);
}
i++;
}
}
static void Main(string[] args)
{
var csvReader = new CommaSeparatedValueReader("data.csv");
var customerRepository = new CustomerRepository(csvReader);
var customerBusinessRules = new CustomerService(customerRepository);
var frequencyList = customerBusinessRules.GetFrequencyList();
var sortedAddressList = customerBusinessRules.GetSortedAddressList();
File.WriteAllLines("frequencyList.txt", frequencyList);
File.WriteAllLines("addressList.txt", sortedAddressList);
Console.WriteLine("Display Frequency List");
Console.WriteLine(File.ReadAllText("frequencyList.txt"));
Console.WriteLine("Display Address List Sorted");
Console.WriteLine(File.ReadAllText("addressList.txt"));
Console.ReadKey();
}
/// <summary>
/// Gets the filtered LogicalRecordNumbers, and the requested variables, and builds a table of their crossjoin
/// </summary>
/// <param name="tableName"></param>
/// <returns></returns>
public bool CheckBuildVariableTable(string tableName)
{
try
{
using (var conn = DbClient.GetConnection())
{
if (DataClient.HasTable(conn, DbClient, tableName))
{
if (!ReusePreviousJobTable)
{
DbClient.GetCommand(string.Format("DROP TABLE IF EXISTS \"{0}\";", tableName), conn).ExecuteNonQuery();
}
else
{
_log.DebugFormat("Table {0} was already built", tableName);
return true;
}
}
//gets a list of the LRUs we want
HashSet<string> requestedLRNs = GetFilteredLRUs(conn);
//gets all the variable mapping information we need
DataTable reqVariablesDT = GetRequestedVariables(conn);
if ((reqVariablesDT == null) || (reqVariablesDT.Rows.Count == 0))
{
_log.Info("Zero variables found: I couldn't understand those variables, can you check them and try again?");
//_log.Warn("I didn't understand those variables, can you check them and try again?");
return false;
}
DataTable newTable = new DataTable();
newTable.Columns.Add("LOGRECNO", typeof(string));
Dictionary<string, DataRow> rowsByLRN = new Dictionary<string, DataRow>(requestedLRNs.Count);
foreach (var id in requestedLRNs)
{
var row = newTable.NewRow();
row[0] = id;
newTable.Rows.Add(row);
rowsByLRN[id] = row;
}
bool shouldNotParseErrorValues = PreserveJam;
_log.Debug("Importing Columns");
int varNum = 0;
foreach (DataRow variableRow in reqVariablesDT.Rows)
{
if (this.IsCancelled()) { return false; }
varNum++;
var sequenceNo = Utilities.GetAs<int>(variableRow["SEQNO"] as string, -1);
var seqFile = Directory.GetFiles(this.GetAggregateDataPath(), "e*" + sequenceNo.ToString("0000") + "000.txt"); //0001000
if ((seqFile == null) || (seqFile.Length == 0))
{
_log.DebugFormat("Couldn't find sequence file {0}", sequenceNo);
continue;
}
var errorFile = Directory.GetFiles(this.GetAggregateDataPath(), "m*" + sequenceNo.ToString("0000") + "000.txt"); //0001000
if ((errorFile == null) || (errorFile.Length == 0))
{
_log.DebugFormat("Couldn't find error margin file {0}", sequenceNo);
}
//These next two if statement checks should probably be removed once
//#19869 is resolved
//Until that case is resolved, we can't guarantee rows in reqVariablesDT will be
//unique so they should stay.
//TODO: alternate column naming?
string newColumnName = variableRow["COLNAME"] as string;
if (newTable.Columns.Contains(newColumnName))
{
newColumnName = newColumnName + varNum;
}
newTable.Columns.Add(newColumnName, typeof(double));
//this really ought to be unique.
string newErrorMarginColumnName = "m" + newColumnName;
if (newTable.Columns.Contains(newErrorMarginColumnName))
{
newErrorMarginColumnName = newErrorMarginColumnName + varNum;
}
if (shouldNotParseErrorValues)
{
newTable.Columns.Add(newErrorMarginColumnName, typeof(string));
}
else
{
newTable.Columns.Add(newErrorMarginColumnName, typeof(double));
}
_log.DebugFormat("Importing {0}...", newColumnName);
int columnIDX = Utilities.GetAs<int>(variableRow["COLNO"] as string, -1);
CommaSeparatedValueReader reader = new CommaSeparatedValueReader(seqFile[0], false);
foreach (List<string> values in reader)
{
string lrn = values[5];
if (!requestedLRNs.Contains(lrn))
continue;
if (columnIDX < values.Count)
{
double val = Utilities.GetAs<double>(values[columnIDX], double.NaN);
if (!double.IsNaN(val))
{
rowsByLRN[lrn][newColumnName] = val;
}
}
if (this.IsCancelled()) { break; }
}
reader.Close();
//these error files better have the exact same format!
reader = new CommaSeparatedValueReader(errorFile[0], false);
foreach (List<string> values in reader)
{
string lrn = values[5];
if (!requestedLRNs.Contains(lrn))
continue;
if (columnIDX < values.Count)
{
if (shouldNotParseErrorValues)
{
rowsByLRN[lrn][newErrorMarginColumnName] = values[columnIDX];
}
else
{
double val = Utilities.GetAs<double>(values[columnIDX], double.NaN);
if (!double.IsNaN(val))
{
rowsByLRN[lrn][newErrorMarginColumnName] = val;
}
rowsByLRN[lrn][newErrorMarginColumnName] = val;
}
}
if (this.IsCancelled()) { break; }
}
reader.Close();
}
if (this.IsCancelled()) { return false; }
_log.DebugFormat("Creating Table {0}", tableName);
string createTableSQL = SqliteDataClient.GenerateTableSQLFromTable(tableName, newTable, "LOGRECNO");
DbClient.GetCommand(createTableSQL, conn).ExecuteNonQuery();
if (this.IsCancelled()) { return false; }
_log.DebugFormat("Saving Table {0}...", tableName);
var dba = DataClient.GetMagicAdapter(conn, DbClient, string.Format("SELECT * FROM \"{0}\"", tableName));
dba.Update(newTable);
_log.Debug("Done!");
_log.Debug("Import complete!");
//TODO: save the table to the database
}
return true;
}
catch (Exception ex)
{
_log.Error("Error while building table", ex);
}
return false;
}
