private static IEnumerable <KeyValuePair <string, List <InputRow> > > CreateKeyListAsStringAndRowListQuery(
Predicate <int> splitIndexFilter, List <KeyValuePair <string, string> > cidGroupAndInputFileNameList, string pTargetHeader, List <string> keyNameList, ParallelOptions parallelOptions)
{
//read from all the files at once, return all the lines with the same key. Assume (but check) all have same keys in same order
List <IEnumerable <InputRow> > nextSetOfLinesQueryList =
cidGroupAndInputFileNameList.Select(cidGroupAndInputFileName => CreateRowListQuery(splitIndexFilter, cidGroupAndInputFileName, pTargetHeader, keyNameList)).ToList();
IEnumerable <InputRow> mergedInputRows = SpecialFunctions.EnumerateSortedMerge(nextSetOfLinesQueryList, inputRow => inputRow.KeyList.StringJoin("\t"));
IEnumerable <List <InputRow> > groupedInputRowList = SpecialFunctions.EnumerateSortedGroups(mergedInputRows, inputRow => inputRow.KeyList.StringJoin("\t"));
foreach (List <InputRow> groupedInputRow in groupedInputRowList)
{
string key = groupedInputRow[0].KeyList.StringJoin("\t");
var nextItem = new KeyValuePair <string, List <InputRow> >(key, groupedInputRow);
yield return(nextItem);
}
}
private static IEnumerable <List <InputRow> > NextSetOfLinesQueryList(Predicate <int> splitIndexFilter, KeyValuePair <string, string> cidGroupAndInputFileName, string pTargetHeader, List <string> keyNameList)
{
var inputRowQuery = CreateRowListQuery(splitIndexFilter, cidGroupAndInputFileName, pTargetHeader, keyNameList);
return(SpecialFunctions.EnumerateSortedGroups(inputRowQuery, inputRow => inputRow.KeyList.StringJoin("\t")));
}
