public void CompareCsvs(Csv csv1, Csv csv2, int countStartEachThread = 0, int countFinishEachThread = 2)
{
int count = 0;
//compare of the line size of csvs and get the csv with largest line count
if (csv1.sizeOfCsv() > csv2.sizeOfCsv())
{
count = csv1.sizeOfCsv();
}
else
{
count = csv2.sizeOfCsv();
}
// Parallel.For(0, countFinishEachThread, i =>
// {
//we have taken the line size of the largest csv file and compare that many times
for (int i = countStartEachThread; i <= countFinishEachThread; i++)
{
if (csv1.sizeOfCsv() == 0 || csv2.sizeOfCsv() == 0)
{
Console.WriteLine("CSV1 or CSV2 has no data, cannot compare");
break;
}
Console.ForegroundColor = ConsoleColor.Green;
int csv2Index = csv2.GetList().IndexOf(csv2.GetLine(i));
int csv1Index = csv1.GetList().IndexOf(csv1.GetLine(i));
//Check if composite field and payload matches
if (csv1.GetLine(i).CompositeField == csv2.GetLine(i).CompositeField&& csv1.GetLine(i).PayLoad == csv2.GetLine(i).PayLoad)
{
Console.WriteLine("({0},{1}) Lines match for CSV1 and CSV2", csv1Index, csv2Index, Console.ForegroundColor);
if (csv1.GetLine(i).Price != csv2.GetLine(i).Price)
{
Program p = new Program();
FailureLevel f = p.PricingDiscrepancy(csv1.GetLine(i).Price, csv2.GetLine(i).Price);
Console.WriteLine("Pricing discrepancy flagged as: " + f);
}
}
Console.ForegroundColor = ConsoleColor.Red;
//Check if composite field maches and payload does NOT match
if (csv1.GetLine(i).CompositeField == csv2.GetLine(i).CompositeField&& csv1.GetLine(i).PayLoad != csv2.GetLine(i).PayLoad)
{
Console.WriteLine("({0},{1}) Composite Key for CSV1 and CSV2 match BUT payload does NOT match CSV1 ({2}) and CSV2 ({3})", csv1Index, csv2Index, csv1.GetLine(i).PayLoad, csv2.GetLine(i).PayLoad, Console.ForegroundColor);
}
//composite fields do NOT match
if (csv1.GetLine(i).CompositeField != csv2.GetLine(i).CompositeField)
{
try
{
if (i == count - 1)
{
Console.WriteLine("({0},{1}) CSV1 ({2}) and CSV2 ({3}) composite keys do not match", csv1Index, csv2Index, csv1.GetLine(i).CompositeField, csv2.GetLine(i).CompositeField);
}
//to avoid comparing i+1=null at the end of the for loop
if (i != count - 1)
{
//check for test 8
if (csv1.GetLine(i).CompositeField == csv2.GetLine(i + 1).CompositeField&& csv1.GetLine(i + 1).CompositeField == csv2.GetLine(i).CompositeField)
{
int csv2IndexPlusOne = csv1Index + 1;
int csv1IndexPlusOne = csv1Index + 1;
if (csv1.GetLine(i).PayLoad == csv2.GetLine(i + 1).PayLoad)
{
Console.WriteLine("({0},{1}) Old and New programs printed csv files with incorrect index. CSV 1 index: {0} and CSV 2 index: {1}", csv1Index, csv2IndexPlusOne);
}
//compare payloads match
if (csv1.GetLine(i + 1).PayLoad == csv2.GetLine(i).PayLoad)
{
Console.WriteLine("({0},{1}) Old and New programs printed csv files with incorrect index. CSV 1 index: {0} and CSV 2 index: {1}", csv1IndexPlusOne, csv2Index);
}
i++;
continue;
}
bool foundMismatchingLine = false;
//find multiple missing lines
if (csv1.GetLine(i).CompositeField != csv2.GetLine(i + 1).CompositeField&& csv1.GetLine(i + 1).CompositeField != csv2.GetLine(i).CompositeField)
{
//find multple lines missing in csv1
int countOfMissingLines = 0;
//check diagonal records for a matching composite value for csv1
Thread tcsv1 = new Thread(new ThreadStart(() => ThreadedSearchOfMismtachingLinesCsv1(countOfMissingLines, count, csv1, csv2, foundMismatchingLine, i, csv1Index, csv2Index)));
tcsv1.Start();
//check diagonal records for a matching composite value for csv2
Thread tcsv2 = new Thread(new ThreadStart(() => ThreadedSearchOfMismatchingLinesCsv2(countOfMissingLines, count, csv1, csv2, foundMismatchingLine, i, csv1Index, csv2Index)));
tcsv2.Start();
//find multple lines missing in csv2
countOfMissingLines = 0;
Console.ForegroundColor = ConsoleColor.Red;
if (!foundMismatchingLine)
{
Console.WriteLine("({0},{1}) CSV1 ({2}) and CSV2 ({3}) composite keys do not match", csv1Index, csv2Index, csv1.GetLine(i).CompositeField, csv2.GetLine(i).CompositeField);
}
foundMismatchingLine = false;
}
//csv1 composite key on currentline matches csv2 composite in currentline+1
if (csv1.GetLine(i).CompositeField == csv2.GetLine(i + 1).CompositeField)
{
Console.WriteLine("({0},{1}) CSV1 ({2}) and CSV2 ({3}) composite keys do not match", csv1Index, csv2Index, csv1.GetLine(i).CompositeField, csv2.GetLine(i).CompositeField);
var matchingCsv2Line = csv2Index + 1;
Console.ForegroundColor = ConsoleColor.Blue;
//declare csv1 line at i is missing
Console.WriteLine("CSV1 missing at line: " + csv1Index);
Console.WriteLine("({0},{1}) CSV1 composite key at {0} matches the CSV2 composite key in next line at {1}", csv1Index, matchingCsv2Line);
Console.WriteLine("Adding + 1 to csv1 index to compare remaining composite key lines");
//check payload matches
if (csv1.GetLine(i).PayLoad == csv2.GetLine(i + 1).PayLoad)
{
Console.WriteLine("({0},{1}) Payload matches between csv1 currentline and csv2 currentline+1", csv1Index, matchingCsv2Line);
}
//established that we are missing a line in csv1, so shift the index so we can continue comparing the remaining lines
var item = csv1.GetLine(i);
var oldIndex = csv1Index;
csv1.GetList().Insert(oldIndex + 1, item);
}
//check if csv2 is missing line, composite keys at csv1.currentline=1 matches csv2.currentline
if (csv1.GetLine(i + 1).CompositeField == csv2.GetLine(i).CompositeField)
{
Console.WriteLine("({0},{1}) CSV1 ({2}) and CSV2 ({3}) composite keys do not match", csv1Index, csv2Index, csv1.GetLine(i).CompositeField, csv2.GetLine(i).CompositeField);
Console.ForegroundColor = ConsoleColor.Blue;
var matchingCsv1Line = csv1Index + 1;
//declare csv2 line at i is missing
Console.WriteLine("CSV2 missing at line: " + csv2Index);
Console.WriteLine("({0},{1}) CSV2 composite key matches the CSV1 composite key in next line", matchingCsv1Line, csv2Index);
Console.WriteLine("Adding + 1 to CSV2 index to compare remaining composite key lines");
//check payload matches
if (csv1.GetLine(i + 1).PayLoad == csv2.GetLine(i).PayLoad)
{
Console.WriteLine("({0},{1}) Payload matches between csv1 currentline+1 and csv2 currentline", matchingCsv1Line, csv2Index);
}
//established that we are missing a line in csv2, so shift the index so we can continue comparing the remaining lines
var item = csv2.GetLine(i);
var oldIndex = csv2Index;
csv2.GetList().Insert(oldIndex + 1, item);
}
}
}
catch (Exception e)
{
Console.WriteLine("Exception occured during comparison of CSV1 and CSV2: " + e);
break;
}
}
}
}
public static void ThreadedSearchOfMismatchingLinesCsv2(int countOfMissingLines, int count, Csv csv1, Csv csv2, bool foundMismatchingLine, int i, int csv1Index, int csv2Index)
{
try
{
for (int j = i; j < count - 1; j++)
{
countOfMissingLines++;
if (csv1.GetLine(j + 1).CompositeField == csv2.GetLine(i).CompositeField)
{
Console.ForegroundColor = ConsoleColor.Blue;
Console.WriteLine("CSV2 is missing this many lines: {0}", countOfMissingLines);
foundMismatchingLine = true;
//check payload matches
if (csv1.GetLine(j + 1).PayLoad == csv2.GetLine(i).PayLoad)
{
Console.ForegroundColor = ConsoleColor.Blue;
Console.WriteLine("({0},{1}) Payload matches between CSV2 currentline: {1} and CSV1 currentline: {0}", j + 1, i);
}
else
{
Console.ForegroundColor = ConsoleColor.Blue;
Console.WriteLine("({0},{1}) Payload does NOT matches between CSV1 and CSV2}", j + 1, i);
}
Console.ForegroundColor = ConsoleColor.Blue;
Console.WriteLine("({0},{1}) Composite Key matches between CSV1 currentline: {2}, CSV2 currentline: {3}", csv1.GetLine(j + 1).CompositeField, csv2.GetLine(i).CompositeField, j + 1, i);
//established that we are missing lines in csv2, so shift the index so we can continue comparing the remaining lines
var item = csv2.GetLine(i);
var oldIndex = csv2Index;
csv2.GetList().Insert(oldIndex + (j + 1), item);
}
}
}
catch (Exception e)
{
}
}
static void Main(string[] args)
{
/*
* composite key is unique, order is relevant
* composite key is unique, order is not relevant
* composite key is not unique, order is relevant
* composite key is not unique, order is not relevant
*
*
*
*
*
* ASSUMPTION SET 1: (Started Dev)
* composite key is unique, order is relevant, not currently order, order first?
* we can read the csv files line by line if they are already ordered
* if not, order files first
* possible hashtable use, if we are using multithreading
* handle mismatching with tolenrence of 1 line, so if line missing compare with next line in other csv file and continue comparing thereafter
* 1. if composite keys match, check payload matches and report result
* 2. if composite keys do not match, report that composite keys do not match and the line of mistmatch
* 3. if we notice that either csv 1 or csv 2 are missing a single line (i.e. composite key matches in the next line) then report a missing line in csv and continue comparing
* 4. does not handle multiple line missing
*
*
*
* ASSUMPTION SET 2:
* composite key is unique, order is NOT relevant
* then we can find matching composite key in other csv file and compare payload
* and report
* 1. Payload of matching composite keys are the same or different
* 2. If there are no matches at all for composite key on each csv file
*
*
*
*
* ASSUMPTION SET 3:
* composite key is NOT unique, order is relevant
* strictly fail on line mismatch on composite key, to avoid false positive (comparing successfully two lines we shouldnt have)
* and report
* 1. Payload of matching composite keys are the same on the same line
*
*
*
* ASSUMPTION SET 4:
* composite key is NOT unique, order is NOT relevant,
* 1. find the first matching composite key in csv files and report match, any odd ones left report on which csv (can get a bit complicated if price weight is different in multiple matches)
* 2. report on any non matching composite keys at all
* 3. report on matching keys
*
*
*
*
*
*
*
*
* TODO : import from csv can convert to csv object
* TODO : write report to text file
* TODO : Make code that missing lines multithreaded
*
*
*
*/
DateTime now = DateTime.Now;
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine("PROGRAM START TIME: {0}", now);
Console.WriteLine("--------TEST 1---------");
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
Csv test1_csv1 = new Csv();
test1_csv1.AddLine("ABC", "GBP,233232,London", 123.56);
test1_csv1.AddLine("DDS", "GBP,233232,London", 123.56);
test1_csv1.AddLine("SAA", "GBP,233232,London", 123.56);
Csv test1_csv2 = new Csv();
test1_csv2.AddLine("DDS", "GBP,233232,London", 123.56);
test1_csv2.AddLine("SAA", "GBP,233232,London", 123.56);
CompareCsvFiles test1 = new CompareCsvFiles();
test1.CompareCsvs(test1_csv1, test1_csv2, 0, 2);
stopwatch.Stop();
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine("Time elapsed: {0}", stopwatch.Elapsed);
stopwatch.Restart();
stopwatch.Start();
Console.WriteLine("--------TEST 2---------");
//entire csvs match
Csv test2_csv1 = new Csv();
test2_csv1.AddLine("ABC", "GBP,233232,London", 123.56);
test2_csv1.AddLine("DDS", "GBP,233232,London", 123.56);
test2_csv1.AddLine("SAA", "GBP,233232,London", 123.56);
Csv test2_csv2 = new Csv();
test2_csv2.AddLine("ABC", "GBP,233232,London", 123.56);
test2_csv2.AddLine("DDS", "GBP,233232,London", 123.56);
test2_csv2.AddLine("SAA", "GBP,233232,London", 123.56);
CompareCsvFiles test2 = new CompareCsvFiles();
test2.CompareCsvs(test2_csv1, test2_csv2, 0, 2);
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine("Time elapsed: {0}", stopwatch.Elapsed);
stopwatch.Restart();
stopwatch.Start();
Console.WriteLine("--------TEST 3---------");
//payload does not match but composite key matches
Csv test3_csv1 = new Csv();
test3_csv1.AddLine("ABC", "GBP,233232,London", 123.56);
test3_csv1.AddLine("DDS", "YYY,233232,London", 123.56);
test3_csv1.AddLine("SAA", "GBP,233232,London", 123.56);
Csv test3_csv2 = new Csv();
test3_csv2.AddLine("ABC", "GBP,233232,London", 123.56);
test3_csv2.AddLine("DDS", "ZZZ,233232,London", 123.56);
test3_csv2.AddLine("SAA", "GBP,233232,London", 123.56);
CompareCsvFiles test3 = new CompareCsvFiles();
test3.CompareCsvs(test3_csv1, test3_csv2, 0, 2);
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine("Time elapsed: {0}", stopwatch.Elapsed);
Console.WriteLine("--------TEST 4---------");
//not matching at all on every line of csvs
stopwatch.Restart();
stopwatch.Start();
Csv test4_csv1 = new Csv();
test4_csv1.AddLine("ABC", "GBP,233232,London", 123.56);
test4_csv1.AddLine("DDS", "YYY,233232,London", 123.56);
test4_csv1.AddLine("SAA", "GBP,233232,London", 123.56);
Csv test4_csv2 = new Csv();
test4_csv2.AddLine("FFF", "GBP,233232,London", 123.56);
test4_csv2.AddLine("SSS", "ZZZ,233232,London", 123.56);
test4_csv2.AddLine("CCC", "GBP,233232,London", 123.56);
CompareCsvFiles test4 = new CompareCsvFiles();
test4.CompareCsvs(test4_csv1, test4_csv2);
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine("Time elapsed: {0}", stopwatch.Elapsed);
Console.WriteLine("--------TEST 5---------");
stopwatch.Restart();
stopwatch.Start();
Csv test5_csv1 = new Csv();
test5_csv1.AddLine("DDS", "GBP,233232,London", 123.56);
test5_csv1.AddLine("SAA", "GBP,233232,London", 123.56);
Csv test5_csv2 = new Csv();
test5_csv2.AddLine("ABC", "GBP,233232,London", 123.56);
test5_csv2.AddLine("DDS", "GBP,233232,London", 123.56);
test5_csv2.AddLine("SAA", "GBP,233232,London", 123.56);
CompareCsvFiles test5 = new CompareCsvFiles();
test5.CompareCsvs(test5_csv1, test5_csv2);
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine("Time elapsed: {0}", stopwatch.Elapsed);
Console.WriteLine("--------TEST 6---------");
//some matching first and then some lines that are not matching
stopwatch.Restart();
stopwatch.Start();
Csv test6_csv1 = new Csv();
test6_csv1.AddLine("ABC", "GBP,233232,London", 123.56);
test6_csv1.AddLine("SAA", "GBP,233232,London", 123.56);
test6_csv1.AddLine("AAA", "GBP,233232,London", 123.56);
test6_csv1.AddLine("BBB", "GBP,233232,London", 123.56);
Csv test6_csv2 = new Csv();
test6_csv2.AddLine("ABC", "GBP,233232,London", 123.56);
test6_csv2.AddLine("SAA", "GBP,233232,London", 123.56);
test6_csv2.AddLine("XXX", "GBP,233232,London", 123.56);
test6_csv2.AddLine("YYY", "GBP,233232,London", 123.56);
CompareCsvFiles test6 = new CompareCsvFiles();
test6.CompareCsvs(test6_csv1, test6_csv2, 0, 3);
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine("Time elapsed: {0}", stopwatch.Elapsed);
Console.WriteLine("--------TEST 7---------");
//missing line after a successful matching line
stopwatch.Restart();
stopwatch.Start();
Csv test7_csv1 = new Csv();
test7_csv1.AddLine("ABC", "GBP,233232,London", 123.56);
test7_csv1.AddLine("SAA", "GBP,233232,London", 123.56);
test7_csv1.AddLine("GBP", "GBP,233232,London", 123.56);
test7_csv1.AddLine("TDD", "GBP,233232,London", 123.56);
Csv test7_csv2 = new Csv();
test7_csv2.AddLine("ABC", "GBP,233232,London", 123.56);
test7_csv2.AddLine("DDS", "GBP,233232,London", 123.56);
test7_csv2.AddLine("SAA", "GBP,233232,London", 123.56);
test7_csv2.AddLine("GBP", "GBP,233232,London", 123.56);
CompareCsvFiles test7 = new CompareCsvFiles();
test7.CompareCsvs(test7_csv1, test7_csv2, 0, 3);
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine("Time elapsed: {0}", stopwatch.Elapsed);
Console.WriteLine("--------TEST 8---------");
//program 1 and program 2 spits out output to csv files in different orders
//i.e. index 1 in csv 1 is index 2 in csv 2
//and index 2 in csv 1 is index 1 in csv 2
stopwatch.Restart();
stopwatch.Start();
Csv test8_csv1 = new Csv();
test8_csv1.AddLine("ABC", "GBP,233232,London", 123.56);
test8_csv1.AddLine("SAA", "GBP,233232,London", 123.56);
test8_csv1.AddLine("DDS", "GBP,233232,London", 123.56);
test8_csv1.AddLine("TDD", "GBP,233232,London", 123.56);
Csv test8_csv2 = new Csv();
test8_csv2.AddLine("ABC", "GBP,233232,London", 123.56);
test8_csv2.AddLine("DDS", "GBP,233232,London", 123.56);
test8_csv2.AddLine("SAA", "GBP,233232,London", 123.56);
test8_csv2.AddLine("TDD", "GBP,233232,London", 123.56);
CompareCsvFiles test8 = new CompareCsvFiles();
test8.CompareCsvs(test8_csv1, test8_csv2, 0, 3);
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine("Time elapsed: {0}", stopwatch.Elapsed);
Console.WriteLine("--------TEST 9---------");
//no records in csvs
stopwatch.Restart();
stopwatch.Start();
Csv test9_csv1 = new Csv();
Csv test9_csv2 = new Csv();
CompareCsvFiles test9 = new CompareCsvFiles();
//test9.CompareCsvs(test9_csv1, test9_csv2);
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine("Time elapsed: {0}", stopwatch.Elapsed);
Console.WriteLine("--------TEST 10---------");
Csv test10_csv1 = new Csv();
Csv test10_csv2 = new Csv();
/*
*
* for (int i = 0; i < 100000; i++)
* {
* string random = Convert.ToBase64String(Guid.NewGuid().ToByteArray()).Substring(0, 8);
*
* test10_csv1.AddLine(random, "GBP,233232,London", 123.56);
* test10_csv2.AddLine(random, "GBP,233232,London", 123.56);
*
*
* }
*
*/
test10_csv1.AddLine("AAA", "GBP,233232,London", 123.561);
test10_csv1.AddLine("BBB", "GBP,233232,London", 123.58);
test10_csv1.AddLine("CCC", "GBP,233232,London", 123.24);
test10_csv1.AddLine("DDD", "GBP,233232,London", 123.56);
test10_csv1.AddLine("EEE", "GBP,233232,London", 123.56);
test10_csv1.AddLine("FFF", "GBP,233232,London", 123.56);
test10_csv1.AddLine("GGG", "GBP,233232,London", 123.56);
test10_csv1.AddLine("HHH", "GBP,233232,London", 123.56);
test10_csv1.AddLine("III", "GBP,233232,London", 123.56);
test10_csv1.AddLine("JJJ", "GBP,233232,London", 123.56);
test10_csv2.AddLine("AAA", "GBP,233232,London", 123.56);
test10_csv2.AddLine("BBB", "GBP,233232,London", 123.56);
test10_csv2.AddLine("CCC", "GBP,233232,London", 123.56);
test10_csv2.AddLine("HHH", "GBP,233232,London", 123.56);
test10_csv2.AddLine("III", "GBP,233232,London", 123.56);
test10_csv2.AddLine("JJJ", "GBP,233232,London", 123.56);
test10_csv2.AddLine("KKK", "GBP,233232,London", 123.56);
test10_csv2.AddLine("LLL", "GBP,233232,London", 123.56);
test10_csv2.AddLine("MMM", "GBP,233232,London", 123.56);
test10_csv2.AddLine("NNN", "GBP,233232,London", 123.56);
Thread test10Compare1 = new Thread(new ThreadStart(() => CompareCsvsMultiThreaded(test10_csv1, test10_csv2, 0, 9)));
test10Compare1.Start();
/*
* stopwatch.Start();
*
* CompareCsvFiles test10 = new CompareCsvFiles();
*
*
* test10.CompareCsvs(test10_csv1, test10_csv2, 0, 100009);
*
* Console.WriteLine("Time elapsed: {0}", stopwatch.Elapsed);
*/
//Parallel.Invoke(() => CompareCsvsMultiThreaded(test10_csv1, test10_csv2, 0, 100009));
//bug: diagnal testing of mismatch only within thread, needs to be across all lines
//bug:
/*
*
* Thread test10Compare1 = new Thread(new ThreadStart(() => CompareCsvsMultiThreaded(test10_csv1, test10_csv2, 0, 10000)));
* test10Compare1.Start();
*
* Thread test10Compare2 = new Thread(new ThreadStart(() => CompareCsvsMultiThreaded(test10_csv1, test10_csv2, 10001, 20000)));
* test10Compare2.Start();
*
*
* Thread test10Compare3 = new Thread(new ThreadStart(() => CompareCsvsMultiThreaded(test10_csv1, test10_csv2, 20001, 30000)));
* test10Compare3.Start();
*
*
* Thread test10Compare4 = new Thread(new ThreadStart(() => CompareCsvsMultiThreaded(test10_csv1, test10_csv2, 30001, 40000)));
* test10Compare4.Start();
*
* Thread test10Compare5 = new Thread(new ThreadStart(() => CompareCsvsMultiThreaded(test10_csv1, test10_csv2, 40001, 50000)));
* test10Compare5.Start();
*
* Thread test10Compare6 = new Thread(new ThreadStart(() => CompareCsvsMultiThreaded(test10_csv1, test10_csv2, 50001, 60000)));
* test10Compare6.Start();
*
* Thread test10Compare7 = new Thread(new ThreadStart(() => CompareCsvsMultiThreaded(test10_csv1, test10_csv2, 60001, 70000)));
* test10Compare7.Start();
*
* Thread test10Compare8 = new Thread(new ThreadStart(() => CompareCsvsMultiThreaded(test10_csv1, test10_csv2, 70001, 80000)));
* test10Compare8.Start();
*
* Thread test10Compare9 = new Thread(new ThreadStart(() => CompareCsvsMultiThreaded(test10_csv1, test10_csv2, 80001, 90000)));
* test10Compare9.Start();
*
* Thread test10Compare10 = new Thread(new ThreadStart(() => CompareCsvsMultiThreaded(test10_csv1, test10_csv2, 90001, 100009)));
* test10Compare10.Start();
*/
Console.ForegroundColor = ConsoleColor.White;
}
