internal static void DeduplicateFiles()
{
foreach (var f in UsedFiles)
{
var n = f.FileName;
var removed = 0;
var scanned = new OrderedHashSet <string>();
var first = true;
using (var reader = new StreamReader(n))
{
string line;
while ((line = reader.ReadLine()) != null)
{
if (first)
{
first = false;
if (!line.Equals(CsvContainer.GetFileTypeHeader(f.FileType)))
{
scanned.Add(CsvContainer.GetFileTypeHeader(f.FileType));
}
}
try
{
scanned.Add(line);
}
catch
{
removed++;
}
}
}
using (var writer = new StreamWriter(f.FileName, false))
{
foreach (var line in scanned)
{
writer.WriteLine(line);
}
}
if (removed > 0)
{
Console.WriteLine($"Removed {removed} duplicate lines from {f.FileName}");
}
}
}
internal static void AddUsedFile(CsvContainer file)
{
UsedFiles.Add(file);
}