public void MultiComputationsPivot() { #region multicomputations pivot var inputList = Enumerable.Range(0, 10).Select(i => new { Key = i / 5, Value = i }).ToList(); var outputList = new List <string>(); StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream => { rootStream .CrossApplyEnumerable("list elements", config => config) .Pivot("simple count", i => i.Key, i => new { Id = i.Key, Count = AggregationOperators.Count(), Sum = AggregationOperators.Sum(i.Value), First = AggregationOperators.First(i.Value), Last = AggregationOperators.Last(i.Value), Max = AggregationOperators.Max(i.Value), Min = AggregationOperators.Min(i.Value), Avg = AggregationOperators.Avg(i.Value), }) .Select("transform into string", SimpleSerializeToString) .ThroughAction("collect values", outputList.Add); }).Wait(); var expectedList = new[] { "FirstValue:{ Key = 0, Value = 0 },Key:0,Aggregation:{ Id = 0, Count = 5, Sum = 10, First = 0, Last = 4, Max = 4, Min = 0, Avg = 2 }", "FirstValue:{ Key = 1, Value = 5 },Key:1,Aggregation:{ Id = 0, Count = 5, Sum = 35, First = 5, Last = 9, Max = 9, Min = 5, Avg = 7 }" }.ToList(); CollectionAssert.AreEquivalent(expectedList, outputList); #endregion }
public void DefineProcess(IStream <SimpleConfig> rootStream) { var outputFileS = rootStream.Select("open output file", i => new StreamWriter(i.OutputFilePath)); rootStream .CrossApplyTextFile("read input file", new FlatFileDefinition <SimpleInputFileRow>() .MapColumnToProperty("#", i => i.Id) .MapColumnToProperty("Label", i => i.Name) .MapColumnToProperty("Value", i => i.ValueType) .MapColumnToProperty("Category", i => i.CategoryCode) .IsColumnSeparated('\t'), i => i.InputFilePath) .ToAction("Write input file to console", i => Console.WriteLine($"{i.Id}->{i.Name}->{i.CategoryCode}->{i.ValueType}")) .Pivot("group and count", i => i.CategoryCode, i => new { Count = AggregationOperators.Count(), CountA = AggregationOperators.Count().For(i.ValueType == "a"), CountB = AggregationOperators.Count().For(i.ValueType == "b"), }) .Select("create output row", i => new CategoryStatisticFileRow { CategoryCode = i.Key, CountA = i.Aggregation.CountA, CountB = i.Aggregation.CountB, Count = i.Aggregation.Count }) .Sort("sort output values", i => new { i.CategoryCode }) .ToTextFile("write to text file", outputFileS, new FlatFileDefinition <CategoryStatisticFileRow>()); }
public void MultiColumnsPivot() { #region multicolumns pivot var inputList = Enumerable.Range(0, 10).Select(i => new { Key = i / 5, Value = i, Col = i % 2 }).ToList(); var outputList = new List <string>(); StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream => { rootStream .CrossApplyEnumerable("list elements", config => config) .Pivot("simple count", i => i.Key, i => new { Id = i.Key, EvensCount = AggregationOperators.Count().For(i.Col == 0), OddsCount = AggregationOperators.Count().For(i.Col == 1), }) .Select("transform into string", SimpleSerializeToString) .ThroughAction("collect values", outputList.Add); }).Wait(); var expectedList = new[] { "FirstValue:{ Key = 0, Value = 0, Col = 0 },Key:0,Aggregation:{ Id = 0, EvensCount = 3, OddsCount = 2 }", "FirstValue:{ Key = 1, Value = 5, Col = 1 },Key:1,Aggregation:{ Id = 0, EvensCount = 2, OddsCount = 3 }" }.ToList(); CollectionAssert.AreEquivalent(expectedList, outputList); #endregion }
public void SimplePivot() { var inputList = Enumerable.Range(0, 10).Select(i => new { Key = i / 5, Value = i }).OrderBy(i => i.Key).ToList(); var outputList = new List <string>(); #region simple pivot StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream => { rootStream .CrossApplyEnumerable("list elements", config => config) .EnsureSorted("ensure sorted", i => i.Key) .Pivot("simple count", i => new { Id = i.Key, Count = AggregationOperators.Count() }) .Select("transform into string", SimpleSerializeToString) .ThroughAction("collect values", outputList.Add); }).Wait(); var expectedList = new[] { "FirstValue:{ Key = 0, Value = 0 },Key:0,Aggregation:{ Id = 0, Count = 5 }", "FirstValue:{ Key = 1, Value = 5 },Key:1,Aggregation:{ Id = 0, Count = 5 }" }.ToList(); CollectionAssert.AreEquivalent(expectedList, outputList); #endregion }
static void Main(string[] args) { StreamProcessRunner.CreateAndExecuteAsync( new { InputFilePath = Path.Combine(args[0], "simpleinputfile.csv"), OutputFilePath = Path.Combine(args[0], "simpleoutputfile.csv") }, rootStream => { var outputFileS = rootStream.Select("open output file", i => (Stream)File.OpenWrite(i.OutputFilePath)); rootStream .CrossApplyTextFile("read input file", FlatFileDefinition.Create(i => new { Id = i.ToColumn <int>("#"), Name = i.ToColumn <string>("Label"), ValueType = i.ToColumn <string>("Value"), CategoryCode = i.ToColumn <string>("Category") }).IsColumnSeparated('\t'), i => i.InputFilePath) .ThroughAction("Write input file to console", i => Console.WriteLine($"{i.Id}->{i.Name}->{i.CategoryCode}->{i.ValueType}")) .Pivot("group and count", i => i.CategoryCode, i => new { Count = AggregationOperators.Count(), CountA = AggregationOperators.Count().For(i.ValueType == "a"), CountB = AggregationOperators.Count().For(i.ValueType == "b"), }) .Select("create output row", i => new { CategoryCode = i.Key, i.Aggregation.Count, i.Aggregation.CountA, i.Aggregation.CountB }) .Sort("sort output values", i => new { i.CategoryCode }) .ThroughTextFile("write to text file", outputFileS, FlatFileDefinition.Create(i => new { CategoryCode = i.ToColumn <string>("MyCategoryCode"), Count = i.ToColumn <int>("Count"), CountA = i.ToColumn <int>("CountA"), CountB = i.ToColumn <int>("CountB") })); }).Wait(); Console.WriteLine("Press a key..."); Console.ReadKey(); }
private static void DefineProcess102(ISingleStream <string> contextStream) { contextStream .CrossApply("create values from enumeration", ctx => Enumerable.Range(1, 100) .Select(i => new { Id = i, OutputId = i % 3, Label = $"{ctx}{i}", Description = (i % 5 == 0) ? null : $"Description {i}" })) .Pivot("pivot values", i => i.OutputId, i => new { TheFirst = AggregationOperators.First(i.Label), Count = AggregationOperators.Count(), Count0 = AggregationOperators.Count().For(i.OutputId == 0), Count1 = AggregationOperators.Count().For(i.OutputId == 1), Count2 = AggregationOperators.Count().For(i.OutputId == 2) }) .Do("print file name to console", i => Console.WriteLine($"{i.Key}: Label={i.Aggregation.TheFirst} Count={i.Aggregation.Count}, Count0={i.Aggregation.Count0}, Count1={i.Aggregation.Count1}, Count2={i.Aggregation.Count2}")); }
public void MultiColumnsMultiComputationsPivot() { var inputList = Enumerable.Range(0, 10).Select(i => new { Key = i / 5, Value = i, Col = i % 2 }).OrderBy(i => i.Key).ToList(); var outputList = new List <string>(); #region multicolumns multicomputations pivot StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream => { rootStream .CrossApplyEnumerable("list elements", config => config) .EnsureSorted("ensure sorted", i => i.Key) .Pivot("simple count", i => new { Id = i.Key, EvensCount = AggregationOperators.Count().For(i.Col == 0), EvensSum = AggregationOperators.Sum(i.Value).For(i.Col == 0), EvensFirst = AggregationOperators.First(i.Value).For(i.Col == 0), EvensLast = AggregationOperators.Last(i.Value).For(i.Col == 0), EvensMax = AggregationOperators.Max(i.Value).For(i.Col == 0), EvensMin = AggregationOperators.Min(i.Value).For(i.Col == 0), EvensAvg = AggregationOperators.Avg(i.Value).For(i.Col == 0), OddsCount = AggregationOperators.Count().For(i.Col == 1), OddsSum = AggregationOperators.Sum(i.Value).For(i.Col == 1), OddsFirst = AggregationOperators.First(i.Value).For(i.Col == 1), OddsLast = AggregationOperators.Last(i.Value).For(i.Col == 1), OddsMax = AggregationOperators.Max(i.Value).For(i.Col == 1), OddsMin = AggregationOperators.Min(i.Value).For(i.Col == 1), OddsAvg = AggregationOperators.Avg(i.Value).For(i.Col == 1), }) .Select("transform into string", SimpleSerializeToString) .ThroughAction("collect values", outputList.Add); }).Wait(); var expectedList = new[] { "FirstValue:{ Key = 0, Value = 0, Col = 0 },Key:0,Aggregation:{ Id = 0, EvensCount = 3, EvensSum = 6, EvensFirst = 0, EvensLast = 4, EvensMax = 4, EvensMin = 0, EvensAvg = 2, OddsCount = 2, OddsSum = 4, OddsFirst = 1, OddsLast = 3, OddsMax = 3, OddsMin = 1, OddsAvg = 2 }", "FirstValue:{ Key = 1, Value = 5, Col = 1 },Key:1,Aggregation:{ Id = 0, EvensCount = 2, EvensSum = 14, EvensFirst = 6, EvensLast = 8, EvensMax = 8, EvensMin = 6, EvensAvg = 7, OddsCount = 3, OddsSum = 21, OddsFirst = 5, OddsLast = 9, OddsMax = 9, OddsMin = 5, OddsAvg = 7 }" }.ToList(); CollectionAssert.AreEquivalent(expectedList, outputList); #endregion }
public void EmptyPivot() { #region simple pivot var inputList = new int[] { }.Select(i => new { Key = i / 5, Value = i }).ToList(); var outputList = new List <string>(); StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream => { rootStream .CrossApplyEnumerable("list elements", config => config) .Pivot("simple count", i => i.Key, i => new { Id = i.Key, Count = AggregationOperators.Count() }) .Select("transform into string", SimpleSerializeToString) .ThroughAction("collect values", outputList.Add); }).Wait(); var expectedList = new string[] { }.ToList(); CollectionAssert.AreEquivalent(expectedList, outputList); #endregion }
public void DefineProcess(IStream <MyConfig> rootStream) { var outputFileResourceS = rootStream.Select("open output file", i => new StreamWriter(i.DestinationFilePath)); var outputCategoryResourceS = rootStream.Select("open output category file", i => new StreamWriter(i.CategoryDestinationFilePath)); var parsedLineS = rootStream .CrossApplyFolderFiles("get folder files", i => i.InputFolderPath, i => i.InputFilesSearchPattern) .CrossApplyTextFile("parse input file", new InputFileRowMapper(), (i, p) => { p.FileName = i; return(p); }); var parsedTypeLineS = rootStream .Select("get input file type path", i => i.TypeFilePath) .CrossApplyTextFile("parse type input file", new TypeFileRowMapper()); var joinedLineS = parsedLineS .Lookup("join types to file", parsedTypeLineS, i => i.TypeId, i => i.Id, (l, r) => new { l.Id, r.Name, l.FileName, r.Category }); var categoryStatistics = joinedLineS .Pivot("create statistic for categories", i => i.Category, i => new { Count = AggregationOperators.Count(), Total = AggregationOperators.Sum(i.Id) }) .Select("create output category data", i => new OutputCategoryRow { Category = i.Key, AmountOfEntries = i.Aggregation.Count, TotalAmount = i.Aggregation.Total }) .ToTextFile("write category statistics to file", outputCategoryResourceS, new OutputCategoryRowMapper()); joinedLineS.Select("create output data", i => new OutputFileRow { Id = i.Id, Name = i.Name, FileName = i.FileName }) .ToTextFile("write to output file", outputFileResourceS, new OutputFileRowMapper()) .ToAction("write to console", i => Console.WriteLine($"{i.FileName}:{i.Id}-{i.Name}")); }