public void MultiComputationsPivot()
{
#region multicomputations pivot
var inputList = Enumerable.Range(0, 10).Select(i => new { Key = i / 5, Value = i }).ToList();
var outputList = new List <string>();
StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream =>
{
rootStream
.CrossApplyEnumerable("list elements", config => config)
.Pivot("simple count", i => i.Key, i => new
{
Id = i.Key,
Count = AggregationOperators.Count(),
Sum = AggregationOperators.Sum(i.Value),
First = AggregationOperators.First(i.Value),
Last = AggregationOperators.Last(i.Value),
Max = AggregationOperators.Max(i.Value),
Min = AggregationOperators.Min(i.Value),
Avg = AggregationOperators.Avg(i.Value),
})
.Select("transform into string", SimpleSerializeToString)
.ThroughAction("collect values", outputList.Add);
}).Wait();
var expectedList = new[] {
"FirstValue:{ Key = 0, Value = 0 },Key:0,Aggregation:{ Id = 0, Count = 5, Sum = 10, First = 0, Last = 4, Max = 4, Min = 0, Avg = 2 }",
"FirstValue:{ Key = 1, Value = 5 },Key:1,Aggregation:{ Id = 0, Count = 5, Sum = 35, First = 5, Last = 9, Max = 9, Min = 5, Avg = 7 }"
}.ToList();
CollectionAssert.AreEquivalent(expectedList, outputList);
#endregion
}
public void DefineProcess(IStream <SimpleConfig> rootStream)
{
var outputFileS = rootStream.Select("open output file", i => new StreamWriter(i.OutputFilePath));
rootStream
.CrossApplyTextFile("read input file",
new FlatFileDefinition <SimpleInputFileRow>()
.MapColumnToProperty("#", i => i.Id)
.MapColumnToProperty("Label", i => i.Name)
.MapColumnToProperty("Value", i => i.ValueType)
.MapColumnToProperty("Category", i => i.CategoryCode)
.IsColumnSeparated('\t'),
i => i.InputFilePath)
.ToAction("Write input file to console", i => Console.WriteLine($"{i.Id}->{i.Name}->{i.CategoryCode}->{i.ValueType}"))
.Pivot("group and count", i => i.CategoryCode, i => new
{
Count = AggregationOperators.Count(),
CountA = AggregationOperators.Count().For(i.ValueType == "a"),
CountB = AggregationOperators.Count().For(i.ValueType == "b"),
})
.Select("create output row", i => new CategoryStatisticFileRow
{
CategoryCode = i.Key,
CountA = i.Aggregation.CountA,
CountB = i.Aggregation.CountB,
Count = i.Aggregation.Count
})
.Sort("sort output values", i => new { i.CategoryCode })
.ToTextFile("write to text file", outputFileS, new FlatFileDefinition <CategoryStatisticFileRow>());
}
public void MultiColumnsPivot()
{
#region multicolumns pivot
var inputList = Enumerable.Range(0, 10).Select(i => new { Key = i / 5, Value = i, Col = i % 2 }).ToList();
var outputList = new List <string>();
StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream =>
{
rootStream
.CrossApplyEnumerable("list elements", config => config)
.Pivot("simple count", i => i.Key, i => new
{
Id = i.Key,
EvensCount = AggregationOperators.Count().For(i.Col == 0),
OddsCount = AggregationOperators.Count().For(i.Col == 1),
})
.Select("transform into string", SimpleSerializeToString)
.ThroughAction("collect values", outputList.Add);
}).Wait();
var expectedList = new[] {
"FirstValue:{ Key = 0, Value = 0, Col = 0 },Key:0,Aggregation:{ Id = 0, EvensCount = 3, OddsCount = 2 }",
"FirstValue:{ Key = 1, Value = 5, Col = 1 },Key:1,Aggregation:{ Id = 0, EvensCount = 2, OddsCount = 3 }"
}.ToList();
CollectionAssert.AreEquivalent(expectedList, outputList);
#endregion
}
public void SimplePivot()
{
var inputList = Enumerable.Range(0, 10).Select(i => new { Key = i / 5, Value = i }).OrderBy(i => i.Key).ToList();
var outputList = new List <string>();
#region simple pivot
StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream =>
{
rootStream
.CrossApplyEnumerable("list elements", config => config)
.EnsureSorted("ensure sorted", i => i.Key)
.Pivot("simple count", i => new
{
Id = i.Key,
Count = AggregationOperators.Count()
})
.Select("transform into string", SimpleSerializeToString)
.ThroughAction("collect values", outputList.Add);
}).Wait();
var expectedList = new[] {
"FirstValue:{ Key = 0, Value = 0 },Key:0,Aggregation:{ Id = 0, Count = 5 }",
"FirstValue:{ Key = 1, Value = 5 },Key:1,Aggregation:{ Id = 0, Count = 5 }"
}.ToList();
CollectionAssert.AreEquivalent(expectedList, outputList);
#endregion
}
static void Main(string[] args)
{
StreamProcessRunner.CreateAndExecuteAsync(
new
{
InputFilePath = Path.Combine(args[0], "simpleinputfile.csv"),
OutputFilePath = Path.Combine(args[0], "simpleoutputfile.csv")
},
rootStream =>
{
var outputFileS = rootStream.Select("open output file", i => (Stream)File.OpenWrite(i.OutputFilePath));
rootStream
.CrossApplyTextFile("read input file",
FlatFileDefinition.Create(i => new
{
Id = i.ToColumn <int>("#"),
Name = i.ToColumn <string>("Label"),
ValueType = i.ToColumn <string>("Value"),
CategoryCode = i.ToColumn <string>("Category")
}).IsColumnSeparated('\t'),
i => i.InputFilePath)
.ThroughAction("Write input file to console",
i => Console.WriteLine($"{i.Id}->{i.Name}->{i.CategoryCode}->{i.ValueType}"))
.Pivot("group and count",
i => i.CategoryCode,
i => new
{
Count = AggregationOperators.Count(),
CountA = AggregationOperators.Count().For(i.ValueType == "a"),
CountB = AggregationOperators.Count().For(i.ValueType == "b"),
})
.Select("create output row",
i => new
{
CategoryCode = i.Key,
i.Aggregation.Count,
i.Aggregation.CountA,
i.Aggregation.CountB
})
.Sort("sort output values",
i => new { i.CategoryCode })
.ThroughTextFile("write to text file",
outputFileS,
FlatFileDefinition.Create(i => new
{
CategoryCode = i.ToColumn <string>("MyCategoryCode"),
Count = i.ToColumn <int>("Count"),
CountA = i.ToColumn <int>("CountA"),
CountB = i.ToColumn <int>("CountB")
}));
}).Wait();
Console.WriteLine("Press a key...");
Console.ReadKey();
}
private static void DefineProcess102(ISingleStream <string> contextStream)
{
contextStream
.CrossApply("create values from enumeration", ctx => Enumerable.Range(1, 100)
.Select(i => new
{
Id = i,
OutputId = i % 3,
Label = $"{ctx}{i}",
Description = (i % 5 == 0) ? null : $"Description {i}"
}))
.Pivot("pivot values", i => i.OutputId, i => new
{
TheFirst = AggregationOperators.First(i.Label),
Count = AggregationOperators.Count(),
Count0 = AggregationOperators.Count().For(i.OutputId == 0),
Count1 = AggregationOperators.Count().For(i.OutputId == 1),
Count2 = AggregationOperators.Count().For(i.OutputId == 2)
})
.Do("print file name to console", i => Console.WriteLine($"{i.Key}: Label={i.Aggregation.TheFirst} Count={i.Aggregation.Count}, Count0={i.Aggregation.Count0}, Count1={i.Aggregation.Count1}, Count2={i.Aggregation.Count2}"));
}
public void MultiColumnsMultiComputationsPivot()
{
var inputList = Enumerable.Range(0, 10).Select(i => new { Key = i / 5, Value = i, Col = i % 2 }).OrderBy(i => i.Key).ToList();
var outputList = new List <string>();
#region multicolumns multicomputations pivot
StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream =>
{
rootStream
.CrossApplyEnumerable("list elements", config => config)
.EnsureSorted("ensure sorted", i => i.Key)
.Pivot("simple count", i => new
{
Id = i.Key,
EvensCount = AggregationOperators.Count().For(i.Col == 0),
EvensSum = AggregationOperators.Sum(i.Value).For(i.Col == 0),
EvensFirst = AggregationOperators.First(i.Value).For(i.Col == 0),
EvensLast = AggregationOperators.Last(i.Value).For(i.Col == 0),
EvensMax = AggregationOperators.Max(i.Value).For(i.Col == 0),
EvensMin = AggregationOperators.Min(i.Value).For(i.Col == 0),
EvensAvg = AggregationOperators.Avg(i.Value).For(i.Col == 0),
OddsCount = AggregationOperators.Count().For(i.Col == 1),
OddsSum = AggregationOperators.Sum(i.Value).For(i.Col == 1),
OddsFirst = AggregationOperators.First(i.Value).For(i.Col == 1),
OddsLast = AggregationOperators.Last(i.Value).For(i.Col == 1),
OddsMax = AggregationOperators.Max(i.Value).For(i.Col == 1),
OddsMin = AggregationOperators.Min(i.Value).For(i.Col == 1),
OddsAvg = AggregationOperators.Avg(i.Value).For(i.Col == 1),
})
.Select("transform into string", SimpleSerializeToString)
.ThroughAction("collect values", outputList.Add);
}).Wait();
var expectedList = new[] {
"FirstValue:{ Key = 0, Value = 0, Col = 0 },Key:0,Aggregation:{ Id = 0, EvensCount = 3, EvensSum = 6, EvensFirst = 0, EvensLast = 4, EvensMax = 4, EvensMin = 0, EvensAvg = 2, OddsCount = 2, OddsSum = 4, OddsFirst = 1, OddsLast = 3, OddsMax = 3, OddsMin = 1, OddsAvg = 2 }",
"FirstValue:{ Key = 1, Value = 5, Col = 1 },Key:1,Aggregation:{ Id = 0, EvensCount = 2, EvensSum = 14, EvensFirst = 6, EvensLast = 8, EvensMax = 8, EvensMin = 6, EvensAvg = 7, OddsCount = 3, OddsSum = 21, OddsFirst = 5, OddsLast = 9, OddsMax = 9, OddsMin = 5, OddsAvg = 7 }"
}.ToList();
CollectionAssert.AreEquivalent(expectedList, outputList);
#endregion
}
public void EmptyPivot()
{
#region simple pivot
var inputList = new int[] { }.Select(i => new { Key = i / 5, Value = i }).ToList();
var outputList = new List <string>();
StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream =>
{
rootStream
.CrossApplyEnumerable("list elements", config => config)
.Pivot("simple count", i => i.Key, i => new
{
Id = i.Key,
Count = AggregationOperators.Count()
})
.Select("transform into string", SimpleSerializeToString)
.ThroughAction("collect values", outputList.Add);
}).Wait();
var expectedList = new string[] { }.ToList();
CollectionAssert.AreEquivalent(expectedList, outputList);
#endregion
}
public void DefineProcess(IStream <MyConfig> rootStream)
{
var outputFileResourceS = rootStream.Select("open output file", i => new StreamWriter(i.DestinationFilePath));
var outputCategoryResourceS = rootStream.Select("open output category file", i => new StreamWriter(i.CategoryDestinationFilePath));
var parsedLineS = rootStream
.CrossApplyFolderFiles("get folder files", i => i.InputFolderPath, i => i.InputFilesSearchPattern)
.CrossApplyTextFile("parse input file", new InputFileRowMapper(), (i, p) => { p.FileName = i; return(p); });
var parsedTypeLineS = rootStream
.Select("get input file type path", i => i.TypeFilePath)
.CrossApplyTextFile("parse type input file", new TypeFileRowMapper());
var joinedLineS = parsedLineS
.Lookup("join types to file", parsedTypeLineS, i => i.TypeId, i => i.Id, (l, r) => new { l.Id, r.Name, l.FileName, r.Category });
var categoryStatistics = joinedLineS
.Pivot("create statistic for categories", i => i.Category, i => new { Count = AggregationOperators.Count(), Total = AggregationOperators.Sum(i.Id) })
.Select("create output category data", i => new OutputCategoryRow {
Category = i.Key, AmountOfEntries = i.Aggregation.Count, TotalAmount = i.Aggregation.Total
})
.ToTextFile("write category statistics to file", outputCategoryResourceS, new OutputCategoryRowMapper());
joinedLineS.Select("create output data", i => new OutputFileRow {
Id = i.Id, Name = i.Name, FileName = i.FileName
})
.ToTextFile("write to output file", outputFileResourceS, new OutputFileRowMapper())
.ToAction("write to console", i => Console.WriteLine($"{i.FileName}:{i.Id}-{i.Name}"));
}
