public void SimpleSortedDistinct() { #region simple distinct on sorted var inputList = Enumerable.Range(0, 100).Select(i => i % 10).OrderBy(i => i).ToList(); var outputList = new List <int>(); StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream => { rootStream .CrossApplyEnumerable("list elements", config => config) .Sort("sort output", i => i) .Distinct("distinct") .ThroughAction("collect values", outputList.Add); }).Wait(); var expected = Enumerable.Range(0, 10).Select(i => i).ToList(); CollectionAssert.AreEquivalent(expected, outputList); #endregion }
public void ProduceSubValues() { #region produce sub values without pre/post process var inputList = Enumerable.Range(10, 10).Select(i => $"{i}").ToList(); var outputList = new List <string>(); StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream => { rootStream .CrossApplyEnumerable("list elements", i => i, true) .CrossApply <string, string>("produce sub values", (input, pushValue) => input.Select(i => $"{i}").ToList().ForEach(pushValue)) .ThroughAction("collect values", outputList.Add); }).Wait(); var expected = inputList.SelectMany(i => i.Select(j => $"{j}")).ToList(); var actual = outputList; CollectionAssert.AreEquivalent(expected, actual); #endregion }
public void ProduceSubValuesParallelized() { #region parallel produce sub values without pre/post process var inputList = Enumerable.Range(10, 10).Select(i => Enumerable.Range(i, 4).Select(j => $"{j}").ToList()).ToList(); var outputList = new List <string>(); StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream => { rootStream .CrossApplyEnumerable <List <List <string> >, List <string> >("list elements", config => config) .CrossApply <List <string>, string>("produce sub values", (input, pushValue) => input.ForEach(pushValue)) .ThroughAction("collect values", outputList.Add); }).Wait(); var expected = inputList.SelectMany(i => i).OrderBy(i => i).ToList(); var actual = outputList.OrderBy(i => i).ToList(); CollectionAssert.AreEquivalent(expected, actual); #endregion }
public void FailEnsureKeyedNotSortedWithoutException() { #region ensure keyed not sorted without exception var inputList = new[] { 2, 1, 3, 4, 5 }.ToList(); var outputList = new List <int>(); var task = StreamProcessRunner.CreateAndExecuteWithNoFaultAsync(inputList, rootStream => { rootStream .CrossApplyEnumerable("list elements", config => config) .EnsureKeyed("ensure keyed", i => i) .ThroughAction("collect values", outputList.Add); }); task.Wait(); Assert.IsTrue(task.Result.Failed); Assert.IsNotNull(task.Result.EndOfProcessTraceEvent.NodeName == "ensure keyed"); Assert.IsNotNull(task.Result.StreamStatisticErrors.FirstOrDefault(i => i.NodeName == "ensure keyed")); CollectionAssert.AreEquivalent(new[] { 2 }.ToList(), outputList); #endregion }
static void Main(string[] args) { var task = StreamProcessRunner.CreateAndExecuteAsync( // new ImportFilesConfig { InputFilesRootFolderPath = @"C:\Users\sroyer\Downloads\RBC" }, new ImportFilesConfig { InputFilesRootFolderPath = @"C:\Users\paill\Documents\GitHub\Etl.Net\src\Samples\TestFiles\RBC" } , ImportFiles.DefineProcess , traceStream => traceStream.ThroughAction("trace", i => System.Diagnostics.Debug.WriteLine(i)) ); try { task.Wait(); } catch (Exception ex) { var jex = ex.InnerException as Paillave.Etl.Core.JobExecutionException; var te = jex.TraceEvent; } }
static async Task Main9(string[] args) { var processRunner = StreamProcessRunner.Create <string>(DefineProcess2); processRunner.DebugNodeStream += (sender, e) => { /* PLACE A CONDITIONAL BREAKPOINT HERE FOR DEBUG ex: e.NodeName == "parse file" */ }; using (var dbCtx = new SimpleTutorialDbContext(args[1])) { var executionOptions = new ExecutionOptions <string> { Resolver = new SimpleDependencyResolver().Register <DbContext>(dbCtx), TraceProcessDefinition = (ts, cs) => ts.Do("Show trace on console", t => Console.WriteLine(t.ToString())), // TraceProcessDefinition = DefineTraceProcess, // UseDetailedTraces = true // activate only if per row traces are meant to be caught }; var res = await processRunner.ExecuteAsync(args[0], executionOptions); Console.Write(res.Failed ? "Failed" : "Succeeded"); } }
public void MultiColumnsMultiComputationsPivot() { var inputList = Enumerable.Range(0, 10).Select(i => new { Key = i / 5, Value = i, Col = i % 2 }).OrderBy(i => i.Key).ToList(); var outputList = new List <string>(); #region multicolumns multicomputations pivot StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream => { rootStream .CrossApplyEnumerable("list elements", config => config) .EnsureSorted("ensure sorted", i => i.Key) .Pivot("simple count", i => new { Id = i.Key, EvensCount = AggregationOperators.Count().For(i.Col == 0), EvensSum = AggregationOperators.Sum(i.Value).For(i.Col == 0), EvensFirst = AggregationOperators.First(i.Value).For(i.Col == 0), EvensLast = AggregationOperators.Last(i.Value).For(i.Col == 0), EvensMax = AggregationOperators.Max(i.Value).For(i.Col == 0), EvensMin = AggregationOperators.Min(i.Value).For(i.Col == 0), EvensAvg = AggregationOperators.Avg(i.Value).For(i.Col == 0), OddsCount = AggregationOperators.Count().For(i.Col == 1), OddsSum = AggregationOperators.Sum(i.Value).For(i.Col == 1), OddsFirst = AggregationOperators.First(i.Value).For(i.Col == 1), OddsLast = AggregationOperators.Last(i.Value).For(i.Col == 1), OddsMax = AggregationOperators.Max(i.Value).For(i.Col == 1), OddsMin = AggregationOperators.Min(i.Value).For(i.Col == 1), OddsAvg = AggregationOperators.Avg(i.Value).For(i.Col == 1), }) .Select("transform into string", SimpleSerializeToString) .ThroughAction("collect values", outputList.Add); }).Wait(); var expectedList = new[] { "FirstValue:{ Key = 0, Value = 0, Col = 0 },Key:0,Aggregation:{ Id = 0, EvensCount = 3, EvensSum = 6, EvensFirst = 0, EvensLast = 4, EvensMax = 4, EvensMin = 0, EvensAvg = 2, OddsCount = 2, OddsSum = 4, OddsFirst = 1, OddsLast = 3, OddsMax = 3, OddsMin = 1, OddsAvg = 2 }", "FirstValue:{ Key = 1, Value = 5, Col = 1 },Key:1,Aggregation:{ Id = 0, EvensCount = 2, EvensSum = 14, EvensFirst = 6, EvensLast = 8, EvensMax = 8, EvensMin = 6, EvensAvg = 7, OddsCount = 3, OddsSum = 21, OddsFirst = 5, OddsLast = 9, OddsMax = 9, OddsMin = 5, OddsAvg = 7 }" }.ToList(); CollectionAssert.AreEquivalent(expectedList, outputList); #endregion }
public void ProduceListWithOneItem() { #region produce list with one "resource" item var inputList = Enumerable.Range(0, 100).ToList(); var inputResource = 1; var outputList = new List <int>(); StreamProcessRunner.CreateAndExecuteAsync(new { InputResource = inputResource, InputList = inputList }, rootStream => { var resourceStream = rootStream.Select("get another resource", config => config.InputResource); rootStream .CrossApplyEnumerable("list elements", resourceStream, (config, res) => config.InputList.Select(i => i + res).ToList()) .ThroughAction("collect values", outputList.Add); }).Wait(); CollectionAssert.AreEquivalent(inputList.Select(i => i + inputResource).ToList(), outputList); #endregion }
public void ComputeAverage() { #region compute average var inputList = Enumerable.Range(0, 10).ToList(); var outputList = new List <string>(); StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream => { rootStream .CrossApplyEnumerable("list elements", config => config) .Select("produce business object", i => new { Value = i, Category = $"CAT{i % 2}" }) .Aggregate("compute sum per category", i => new { Nb = 0, Sum = 0 }, i => i.Category, (agg, elt) => new { Nb = agg.Nb + 1, Sum = agg.Sum + elt.Value }) .Select("format and compute result", i => $"{i.Key}:{i.Aggregation.Sum / i.Aggregation.Nb}") .ThroughAction("collect values", outputList.Add); }).Wait(); CollectionAssert.AreEquivalent(new[] { "CAT0:4", "CAT1:5" }.ToList(), outputList); #endregion }
public void GroupElements() { #region group elements var inputList = Enumerable.Range(0, 10).ToList(); var outputList = new List <string>(); StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream => { rootStream .CrossApplyEnumerable("list elements", config => config) .Select("produce business object", i => new { Value = i, Category = $"CAT{i % 2}" }) .Aggregate("compute sum per category", i => "", i => i.Category, (agg, elt) => $"{agg}{elt.Value}") .Select("format result", i => $"{i.Key}:{i.Aggregation}") .ThroughAction("collect values", outputList.Add); }).Wait(); CollectionAssert.AreEquivalent(new[] { "CAT0:02468", "CAT1:13579" }.ToList(), outputList); #endregion }
public void LeftJoinWithSomeMatches() { #region simple left join with couple of matches var inputLeftList = Enumerable.Range(0, 100).Select(i => new { Id = i, ForeignId = i / 10 }).ToList(); var inputRightList = new[] { 2, 5 }.Select(i => new { Id = i, Label = $"Label{i}" }).ToList(); var outputList = new List <string>(); StreamProcessRunner.CreateAndExecuteAsync(new { InputLeftList = inputLeftList, InputRightList = inputRightList }, rootStream => { var leftStream = rootStream .CrossApplyEnumerable("input left elements", config => config.InputLeftList) .EnsureSorted("ensure left is sorted", i => i.ForeignId); var rightStream = rootStream .CrossApplyEnumerable("input right elements", config => config.InputRightList) .EnsureKeyed("ensure right is keyed", i => i.Id); leftStream .LeftJoin("join left and right", rightStream, (left, right) => $"{left.Id}-{right?.Label}") .ThroughAction("collect values", outputList.Add); }).Wait(); var expected = Enumerable.Range(0, 100).Select(i => { if (new[] { 2, 5 }.Contains(i / 10)) { return($"{i}-Label{i / 10}"); } else { return($"{i}-"); } }).ToList(); CollectionAssert.AreEquivalent(expected, outputList); #endregion }
public void EmptyPivot() { #region simple pivot var inputList = new int[] { }.Select(i => new { Key = i / 5, Value = i }).ToList(); var outputList = new List <string>(); StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream => { rootStream .CrossApplyEnumerable("list elements", config => config) .Pivot("simple count", i => i.Key, i => new { Id = i.Key, Count = AggregationOperators.Count() }) .Select("transform into string", SimpleSerializeToString) .ThroughAction("collect values", outputList.Add); }).Wait(); var expectedList = new string[] { }.ToList(); CollectionAssert.AreEquivalent(expectedList, outputList); #endregion }
public void InlineMethodWay() { var config = new SimpleConfigStreamType { Divider = 0 }; var task = StreamProcessRunner.CreateAndExecuteAsync(config, SimpleJob.Job1); try { task.Wait(); Assert.Fail("the execution should not be successfull"); } catch (AggregateException ex) { JobExecutionException jobExecutionException = ex.InnerException as JobExecutionException; if (jobExecutionException == null) { throw; } Assert.IsInstanceOfType(jobExecutionException.InnerException, typeof(DivideByZeroException)); Assert.IsInstanceOfType((jobExecutionException.TraceEvent?.Content as UnhandledExceptionStreamTraceContent)?.Exception, typeof(DivideByZeroException)); } }
static void Main(string[] args) { var runner = new StreamProcessRunner <ComplexQuickstartJob, MyConfig>(); runner.GetDefinitionStructure().OpenEstimatedExecutionPlanVisNetwork(); StreamProcessDefinition <TraceEvent> traceStreamProcessDefinition = new StreamProcessDefinition <TraceEvent>(traceStream => traceStream.ToAction("logs to console", Console.WriteLine)); var testFilesDirectory = @"C:\Users\sroyer\Source\Repos\Etl.Net\src\Samples\TestFiles"; // var testFilesDirectory = @"C:\Users\paill\source\repos\Etl.Net\src\Samples\TestFiles"; var task = runner.ExecuteAsync(new MyConfig { InputFolderPath = Path.Combine(testFilesDirectory, @"."), InputFilesSearchPattern = "testin.*.csv", TypeFilePath = Path.Combine(testFilesDirectory, @"ref - Copy.csv"), DestinationFilePath = Path.Combine(testFilesDirectory, @"outfile.csv"), CategoryDestinationFilePath = Path.Combine(testFilesDirectory, @"categoryStats.csv") }, traceStreamProcessDefinition); task.Result.OpenActualExecutionPlanD3Sankey(); Console.WriteLine("Done"); Console.WriteLine("Press a key..."); Console.ReadKey(); }
public void ProduceSubValuesWithToApplyParallelized() { #region produce sub values without pre/post process with to apply parallelized var inputList = Enumerable.Range(10, 100).Select(i => $"{i}").ToList(); var outputList = new List <string>(); StreamProcessRunner.CreateAndExecuteAsync(new { InputList = inputList, Multiplier = 2 }, rootStream => { var toApplyStream = rootStream.Select("get multiplier", config => config.Multiplier); rootStream .CrossApplyEnumerable("list elements", config => config.InputList) .CrossApply <string, int, string>("produce sub values", toApplyStream, (input, toApply, pushValue) => input.Select(i => $"{i}*{toApply}").ToList().ForEach(pushValue)) .ThroughAction("collect values", outputList.Add); }).Wait(); var expected = inputList.SelectMany(i => i.Select(j => $"{j}*2")).OrderBy(i => i).ToList(); var actual = outputList.OrderBy(i => i).ToList(); CollectionAssert.AreEquivalent(expected, actual); #endregion }
public void SimpleLookup() { #region simple lookup var inputLeftList = Enumerable.Range(0, 100).Select(i => new { Id = i, ForeignId = i / 10 }).ToList(); var inputRightList = Enumerable.Range(0, 10).Select(i => new { Id = i, Label = $"Label{i}" }).ToList(); var outputList = new List <string>(); StreamProcessRunner.CreateAndExecuteAsync(new { InputLeftList = inputLeftList, InputRightList = inputRightList }, rootStream => { var leftStream = rootStream.CrossApplyEnumerable("input left elements", config => config.InputLeftList); var rightStream = rootStream.CrossApplyEnumerable("input right elements", config => config.InputRightList); leftStream .Lookup("join left and right", rightStream, i => i.ForeignId, i => i.Id, (left, right) => $"{left.Id}-{right.Label}") .ThroughAction("collect values", outputList.Add); }).Wait(); var expected = Enumerable.Range(0, 100).Select(i => $"{i}-Label{i / 10}").ToList(); CollectionAssert.AreEquivalent(expected, outputList); #endregion }
public void ProduceSubValuesWithToApplyPrePost() { #region produce sub values without pre/post process with to apply PrePost var inputList = Enumerable.Range(10, 100).Select(i => $"{i}").ToList(); var outputList = new List <string>(); StreamProcessRunner.CreateAndExecuteAsync(new { InputList = inputList, Multiplier = 2 }, rootStream => { var toApplyStream = rootStream.Select("get multiplier", config => config.Multiplier); rootStream .CrossApplyEnumerable("list elements", config => config.InputList, true) .CrossApply <string, int, int, int, string>("produce sub values", toApplyStream, (input, toApply, pushValue) => input.ToString().Select(i => int.Parse(i.ToString()) * toApply).ToList().ForEach(pushValue), (i, j) => int.Parse(i) + 2, (val, initVal, toApply) => $"{val / toApply}-{initVal}") .ThroughAction("collect values", outputList.Add); }).Wait(); var expected = inputList.SelectMany(i => (int.Parse(i) + 2).ToString().Select(j => int.Parse(j.ToString()) * 2).Select(j => $"{j / 2}-{i}").ToList()).ToList(); var actual = outputList; CollectionAssert.AreEquivalent(expected, actual); #endregion }
static async Task Main3(string[] args) { var anyArrayOfByte = new byte[] { }; var res = await StreamProcessRunner.CreateAndExecuteAsync <Stream>(new MemoryStream(anyArrayOfByte), DefineProcess20); }
static async Task Main(string[] args) { var processRunner = StreamProcessRunner.Create <string[]>(Import); var res = await processRunner.ExecuteAsync(args); }