/// <summary>
/// Look up the column index of a given column by name.
/// Will only work if the file had a header row.
/// Column names are case insensitive.
/// Will throw if the column name wasn't found.
/// </summary>
/// <param name="columnNameOrIndex">Column name for which to find column index, or already an integer index</param>
/// <returns>Index of column in TSV. Throws if column isn't found or no header row was read.</returns>
public static int ColumnIndex(this ITabularReader reader, string columnNameOrIndex)
{
int columnIndex;
if (reader.TryGetColumnIndex(columnNameOrIndex, out columnIndex))
{
return(columnIndex);
}
throw new ColumnNotFoundException(String.Format("Column Name \"{0}\" not found in file.\nKnown Columns: \"{1}\"", columnNameOrIndex, String.Join(", ", reader.Columns)));
}
private static void OnlyLatest(string inputFolderPath, string outputFilePath, string idColumnIdentifier)
{
String8Block block = new String8Block();
Dictionary <String8, Tuple <string, int> > latestFileAndRowByID = new Dictionary <String8, Tuple <string, int> >();
IReadOnlyList <string> writerColumns = null;
// Walk the input files to figure out the latest copy of each ID
Trace.WriteLine($"Identifying latest {idColumnIdentifier} in all files in {inputFolderPath}...");
int rowCountRead = 0;
foreach (string inputFilePath in Directory.GetFiles(inputFolderPath))
{
using (ITabularReader reader = TabularFactory.BuildReader(inputFilePath))
{
int idColumnIndex = reader.ColumnIndex(idColumnIdentifier);
while (reader.NextRow())
{
rowCountRead++;
String8 id = reader.Current(idColumnIndex).ToString8();
id.ToUpperInvariant();
// Record the file and row containing this ID, overwriting previous entries
latestFileAndRowByID[block.GetCopy(id)] = new Tuple <string, int>(inputFilePath, reader.RowCountRead);
}
// Capture the columns from the last CSV to write
writerColumns = reader.Columns;
}
}
Trace.WriteLine($"Scan Complete. {rowCountRead:n0} rows read; {latestFileAndRowByID.Count:n0} distinct IDs found.");
using (ITabularWriter writer = TabularFactory.BuildWriter(outputFilePath))
{
writer.SetColumns(writerColumns);
int[] writerColumnIndexInReader = new int[writerColumns.Count];
foreach (string inputFilePath in Directory.GetFiles(inputFolderPath))
{
using (ITabularReader reader = TabularFactory.BuildReader(inputFilePath))
{
// Look up each output column's position in the input file
for (int i = 0; i < writerColumns.Count; ++i)
{
reader.TryGetColumnIndex(writerColumns[i], out writerColumnIndexInReader[i]);
}
int idColumnIndex = reader.ColumnIndex(idColumnIdentifier);
while (reader.NextRow())
{
String8 id = reader.Current(idColumnIndex).ToString8();
id.ToUpperInvariant();
// Copy this row to the output file, *if* it's the latest for this ID
Tuple <string, int> latestForID = latestFileAndRowByID[id];
if (latestForID.Item1 == inputFilePath && latestForID.Item2 == reader.RowCountRead)
{
for (int i = 0; i < writerColumns.Count; ++i)
{
int readerColumnIndex = writerColumnIndexInReader[i];
if (readerColumnIndex >= 0 && readerColumnIndex < reader.CurrentRowColumns)
{
writer.Write(reader.Current(i).ToString8());
}
else
{
writer.Write(String8.Empty);
}
}
writer.NextRow();
}
}
}
}
WriteSizeSummary(null, writer);
}
}