public static void CombineMisMatchedCsvFiles(string[] filePaths, string destinationFile, char splitter = ',', bool Union = true)
{
HashSet <string> combinedheaders = new HashSet <string>();
int i;
// aggregate headers
for (i = 0; i < filePaths.Length; i++)
{
string file = filePaths[i];
//if (Union)
combinedheaders.UnionWith(File.ReadLines(file).First().Split(splitter));
//else
//combinedheaders.Intersect(File.ReadLines(file).First().Split(splitter));
}
if (combinedheaders.Contains(""))
{
combinedheaders.Remove("");
}
var hdict = combinedheaders.ToDictionary(y => y, y => new List <object>());
string[] combinedHeadersArray = combinedheaders.ToArray();
for (i = 0; i < filePaths.Length; i++)
{
var fileheaders = File.ReadLines(filePaths[i]).First().Split(splitter).Where(x => x != "").ToArray();
var notfiledheaders = combinedheaders.Except(fileheaders);
var fi = new FileInfo(filePaths[i]);
if (fi.Length == 0)
{
throw new Exception($"File {fi.Name} is empty");
}
File.ReadLines(filePaths[i]).Skip(1).Select(line => line.Split(splitter)).ForEach(spline =>
{
for (int j = 0; j < fileheaders.Length; j++)
{
hdict[fileheaders[j]].Add(spline[j]);
}
foreach (string header in notfiledheaders)
{
hdict[header].Add(null);
}
});
}
System.Data.DataTable dt = hdict.ToDataTable();
dt.SaveToCSV(destinationFile);
}
