/// <summary> /// Look up the column index of a given column by name. /// Will only work if the file had a header row. /// Column names are case insensitive. /// Will throw if the column name wasn't found. /// </summary> /// <param name="columnNameOrIndex">Column name for which to find column index, or already an integer index</param> /// <returns>Index of column in TSV. Throws if column isn't found or no header row was read.</returns> public static int ColumnIndex(this ITabularReader reader, string columnNameOrIndex) { int columnIndex; if (reader.TryGetColumnIndex(columnNameOrIndex, out columnIndex)) { return(columnIndex); } throw new ColumnNotFoundException(String.Format("Column Name \"{0}\" not found in file.\nKnown Columns: \"{1}\"", columnNameOrIndex, String.Join(", ", reader.Columns))); }
private static void OnlyLatest(string inputFolderPath, string outputFilePath, string idColumnIdentifier) { String8Block block = new String8Block(); Dictionary <String8, Tuple <string, int> > latestFileAndRowByID = new Dictionary <String8, Tuple <string, int> >(); IReadOnlyList <string> writerColumns = null; // Walk the input files to figure out the latest copy of each ID Trace.WriteLine($"Identifying latest {idColumnIdentifier} in all files in {inputFolderPath}..."); int rowCountRead = 0; foreach (string inputFilePath in Directory.GetFiles(inputFolderPath)) { using (ITabularReader reader = TabularFactory.BuildReader(inputFilePath)) { int idColumnIndex = reader.ColumnIndex(idColumnIdentifier); while (reader.NextRow()) { rowCountRead++; String8 id = reader.Current(idColumnIndex).ToString8(); id.ToUpperInvariant(); // Record the file and row containing this ID, overwriting previous entries latestFileAndRowByID[block.GetCopy(id)] = new Tuple <string, int>(inputFilePath, reader.RowCountRead); } // Capture the columns from the last CSV to write writerColumns = reader.Columns; } } Trace.WriteLine($"Scan Complete. {rowCountRead:n0} rows read; {latestFileAndRowByID.Count:n0} distinct IDs found."); using (ITabularWriter writer = TabularFactory.BuildWriter(outputFilePath)) { writer.SetColumns(writerColumns); int[] writerColumnIndexInReader = new int[writerColumns.Count]; foreach (string inputFilePath in Directory.GetFiles(inputFolderPath)) { using (ITabularReader reader = TabularFactory.BuildReader(inputFilePath)) { // Look up each output column's position in the input file for (int i = 0; i < writerColumns.Count; ++i) { reader.TryGetColumnIndex(writerColumns[i], out writerColumnIndexInReader[i]); } int idColumnIndex = reader.ColumnIndex(idColumnIdentifier); while (reader.NextRow()) { String8 id = reader.Current(idColumnIndex).ToString8(); id.ToUpperInvariant(); // Copy this row to the output file, *if* it's the latest for this ID Tuple <string, int> latestForID = latestFileAndRowByID[id]; if (latestForID.Item1 == inputFilePath && latestForID.Item2 == reader.RowCountRead) { for (int i = 0; i < writerColumns.Count; ++i) { int readerColumnIndex = writerColumnIndexInReader[i]; if (readerColumnIndex >= 0 && readerColumnIndex < reader.CurrentRowColumns) { writer.Write(reader.Current(i).ToString8()); } else { writer.Write(String8.Empty); } } writer.NextRow(); } } } } WriteSizeSummary(null, writer); } }