private BiztalkParser GetBiztalkParser()
{
IRowParser rowParser = Substitute.For <IRowParser>();
rowParser
.ParseXmlForRows(Arg.Any <XElement>())
.Returns(c =>
{
if (_errorWhenProcessing)
{
throw new Exception("Test needed exception");
}
var xElement = c.ArgAt <XElement>(0);
var count = int.Parse(xElement.Element("RowsCount").Value);
var returns = new List <BiztalkMessage>();
for (int i = 0; i < count; i++)
{
returns.Add(new BiztalkMessage());
}
return(returns);
});
var parser = new BiztalkParser(rowParser);
return(parser);
}
public LogProcessor(string filename)
{
_rowParser = new RowParser();
_filename = filename;
_bytes = 0;
_rowNum = 0;
}
public CsvParser(TextReader textReader, IRowParser rowParser)
{
if (textReader == null)
{
throw new ArgumentNullException(nameof(textReader));
}
if (rowParser == null)
{
throw new ArgumentNullException(nameof(rowParser));
}
_textReader = textReader;
_rowParser = rowParser;
}
/// <summary>
/// Parses a csv file containing the MNIST data set, returning arrays for inputs and outputs in the format required by the svm.
/// </summary>
/// <param name="path">string</param>
/// <param name="count">int - max number of rows to read</param>
/// <param name="inputs">output variable for double[][] values (inputs)</param>
/// <param name="outputs">output variable for int[] values (labels)</param>
/// <param name="rowParser">IRowParser - indicates how to process the csv data</param>
/// <returns>int - number of rows read</returns>
private static int ReadData(string path, int count, out double[][] inputs, out int[] outputs, IRowParser rowParser)
{
Parser parser = new Parser(rowParser);
// Read the training data CSV file and get a resulting array of doubles and output labels.
List<MLData> rows = Utility.ShowProgressFor<List<MLData>>(() => parser.Parse(path, count), "Reading data");
// Convert the rows into arrays for processing.
inputs = rows.Select(t => t.Data.ToArray()).ToArray();
outputs = rows.Select(t => t.Label).ToArray();
Console.WriteLine(rows.Count + " rows processed.");
return rows.Count;
}
public RowParserFixture()
{
IMetadataParser metadataParser = Substitute.For <IMetadataParser>();
//metadataParser.ParseToXmlElement(Arg.Any<Metadata>()).Returns(c =>
//{
// var root = new XElement("MetaData");
// root.Add(new XElement("Dest", _destination));
// root.Add(new XElement("DateFrom", DateTime.Now.ToString("s")));
// root.Add(new XElement("DateTo", DateTime.Now.ToString("s")));
// root.Add(new XElement("GenerateDate", _generateDate));
// root.Add(new XElement("RowCount", 2));
// root.Add(new XElement("Source", "0003NPIK"));
// root.Add(new XElement("PackageId", _id));
// return root;
//});
metadataParser.ParseToXmlElement(Arg.Any <DestinationMetadata>()).Returns(c =>
{
var root = new XElement("MetaData");
root.Add(new XElement("Dest", _destination));
root.Add(new XElement("GenerateDate", _generateDate.ToString("s")));
root.Add(new XElement("Source", "0003NPIK"));
root.Add(new XElement("PackageId", _id));
return(root);
});
metadataParser.ParseFromXElementToMetadata(Arg.Any <XElement>()).Returns(c =>
{
if (c == null || !c.Args().Any() || (c.ArgAt <XElement>(0) == null))
{
throw new EmptyMetdaDataXmlProvieded();
}
Metadata metadata = new Metadata();
metadata.Destinations = new[] { _destination };
metadata.CorrelationId = _id;
metadata.Source = "0003NPIK";
return(metadata);
});
metadataParser.ParseFromXElementToDestinationMetadata(Arg.Any <XElement>()).Returns(c =>
{
DestinationMetadata metadata = new DestinationMetadata();
metadata.CorrelationId = _id;
metadata.Source = "0003NPIK";
return(metadata);
});
_rowParser = new BiztalkRowParser(metadataParser);
}
public Parser(IRowParser rowParser)
{
_rowParser = rowParser;
}
public CellByteStreamReader(IRowParser rowParser)
{
this.rowParser = rowParser;
}
private static int ReadData(string path, int count, out double[][] inputs, out int [] outputs, IRowParser rowParser)
{
Parser parser = new Parser(rowParser);
List <MLData> rows = Utility.ShowProgressFor <List <MLData> >(() => parser.Parse(path, count), "Reading Data");
inputs = rows.Select(t => t.Data.ToArray()).ToArray();
outputs = rows.Select(t => t.Label).ToArray();
Console.WriteLine(rows.Count + "Rows have Processed.");
return(rows.Count);
}
public Parser(IRowParser rowParser)
{
_rowParser = rowParser;
}
static void Main(string[] args)
{
string leftFile, rightFile;
string configFile = null;
//leftFile = args[0];
//rightFile = args[1];
leftFile = "input1.csv";
rightFile = "input2.csv";
writer = new StreamWriter("output.txt");
if (args.Length > 2)
{
configFile = args[2];
}
Configs = LoadConfiguration(configFile);
Formatters = LoadFormatters();
Parser = new CSVParser();
Configs.SetFormatter(1, NO_SPACE);
StreamReader srLeft = new StreamReader(leftFile);
StreamReader srRight = new StreamReader(rightFile);
Dictionary <string, List <string> > rowsLeft = new Dictionary <string, List <string> >();
Dictionary <string, List <string> > rowsRight = new Dictionary <string, List <string> >();
while (!srLeft.EndOfStream && !srRight.EndOfStream)
{
string strLeft = srLeft.ReadLine();
string strRight = srRight.ReadLine();
List <string> listLeft = Parser.ReadLine(strLeft);
List <string> listRight = Parser.ReadLine(strRight);
Clean(listLeft);
Clean(listRight);
string keyLeft = GetKey(listLeft);
string keyRight = GetKey(listLeft);
List <Diff> diffs;
if (keyLeft == keyRight)
{
diffs = Compare(listLeft, listRight, keyLeft);
}
else
{
List <string> tmpList = GetRow(rowsLeft, keyRight);
if (tmpList != null)
{
diffs = Compare(tmpList, listRight, keyRight);
}
else
{
rowsRight.Add(keyRight, listRight);
}
tmpList = GetRow(rowsRight, keyLeft);
if (tmpList != null)
{
diffs = Compare(tmpList, listLeft, keyLeft);
}
else
{
rowsRight.Add(keyLeft, listLeft);
}
}
}
ReadRemaining("Remaining Left------", srLeft);
ReadRemaining("Remaining Right-----", srRight);
srLeft.Close();
srRight.Close();
writer.Close();
}
/// <summary>
/// Parses a csv file containing the MNIST data set, returning arrays for inputs and outputs in the format required by the svm.
/// </summary>
/// <param name="path">string</param>
/// <param name="count">int - max number of rows to read</param>
/// <param name="inputs">output variable for double[][] values (inputs)</param>
/// <param name="outputs">output variable for int[] values (labels)</param>
/// <param name="rowParser">IRowParser - indicates how to process the csv data</param>
/// <returns>int - number of rows read</returns>
private static int ReadData(string path, int count, out double[][] inputs, out int[] outputs, IRowParser rowParser)
{
Parser parser = new Parser(rowParser);
// Read the training data CSV file and get a resulting array of doubles and output labels.
List <MLData> rows = Utility.ShowProgressFor <List <MLData> >(() => parser.Parse(path, count), "Reading data");
// Convert the rows into arrays for processing.
inputs = rows.Select(t => t.Data.ToArray()).ToArray();
outputs = rows.Select(t => t.Label).ToArray();
Console.WriteLine(rows.Count + " rows processed.");
return(rows.Count);
}
public FileReader(string pathToFile, IRowParser <T> rowParser)
{
_path = pathToFile;
_rowParser = rowParser;
}
public BiztalkParser(IRowParser rowParser)
{
_rowParser = rowParser;
}
