private static IEnumerable <KeyValuePair <string, List <InputRow> > > CreateKeyListAsStringAndRowListQuery( Predicate <int> splitIndexFilter, List <KeyValuePair <string, string> > cidGroupAndInputFileNameList, string pTargetHeader, List <string> keyNameList, ParallelOptions parallelOptions) { //read from all the files at once, return all the lines with the same key. Assume (but check) all have same keys in same order List <IEnumerable <InputRow> > nextSetOfLinesQueryList = cidGroupAndInputFileNameList.Select(cidGroupAndInputFileName => CreateRowListQuery(splitIndexFilter, cidGroupAndInputFileName, pTargetHeader, keyNameList)).ToList(); IEnumerable <InputRow> mergedInputRows = SpecialFunctions.EnumerateSortedMerge(nextSetOfLinesQueryList, inputRow => inputRow.KeyList.StringJoin("\t")); IEnumerable <List <InputRow> > groupedInputRowList = SpecialFunctions.EnumerateSortedGroups(mergedInputRows, inputRow => inputRow.KeyList.StringJoin("\t")); foreach (List <InputRow> groupedInputRow in groupedInputRowList) { string key = groupedInputRow[0].KeyList.StringJoin("\t"); var nextItem = new KeyValuePair <string, List <InputRow> >(key, groupedInputRow); yield return(nextItem); } }
private static IEnumerable <List <InputRow> > NextSetOfLinesQueryList(Predicate <int> splitIndexFilter, KeyValuePair <string, string> cidGroupAndInputFileName, string pTargetHeader, List <string> keyNameList) { var inputRowQuery = CreateRowListQuery(splitIndexFilter, cidGroupAndInputFileName, pTargetHeader, keyNameList); return(SpecialFunctions.EnumerateSortedGroups(inputRowQuery, inputRow => inputRow.KeyList.StringJoin("\t"))); }