public void SimpleSortedDistinct()
{
#region simple distinct on sorted
var inputList = Enumerable.Range(0, 100).Select(i => i % 10).OrderBy(i => i).ToList();
var outputList = new List <int>();
StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream =>
{
rootStream
.CrossApplyEnumerable("list elements", config => config)
.Sort("sort output", i => i)
.Distinct("distinct")
.ThroughAction("collect values", outputList.Add);
}).Wait();
var expected = Enumerable.Range(0, 10).Select(i => i).ToList();
CollectionAssert.AreEquivalent(expected, outputList);
#endregion
}
public void ProduceSubValues()
{
#region produce sub values without pre/post process
var inputList = Enumerable.Range(10, 10).Select(i => $"{i}").ToList();
var outputList = new List <string>();
StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream =>
{
rootStream
.CrossApplyEnumerable("list elements", i => i, true)
.CrossApply <string, string>("produce sub values", (input, pushValue) => input.Select(i => $"{i}").ToList().ForEach(pushValue))
.ThroughAction("collect values", outputList.Add);
}).Wait();
var expected = inputList.SelectMany(i => i.Select(j => $"{j}")).ToList();
var actual = outputList;
CollectionAssert.AreEquivalent(expected, actual);
#endregion
}
public void ProduceSubValuesParallelized()
{
#region parallel produce sub values without pre/post process
var inputList = Enumerable.Range(10, 10).Select(i => Enumerable.Range(i, 4).Select(j => $"{j}").ToList()).ToList();
var outputList = new List <string>();
StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream =>
{
rootStream
.CrossApplyEnumerable <List <List <string> >, List <string> >("list elements", config => config)
.CrossApply <List <string>, string>("produce sub values", (input, pushValue) => input.ForEach(pushValue))
.ThroughAction("collect values", outputList.Add);
}).Wait();
var expected = inputList.SelectMany(i => i).OrderBy(i => i).ToList();
var actual = outputList.OrderBy(i => i).ToList();
CollectionAssert.AreEquivalent(expected, actual);
#endregion
}
public void FailEnsureKeyedNotSortedWithoutException()
{
#region ensure keyed not sorted without exception
var inputList = new[] { 2, 1, 3, 4, 5 }.ToList();
var outputList = new List <int>();
var task = StreamProcessRunner.CreateAndExecuteWithNoFaultAsync(inputList, rootStream =>
{
rootStream
.CrossApplyEnumerable("list elements", config => config)
.EnsureKeyed("ensure keyed", i => i)
.ThroughAction("collect values", outputList.Add);
});
task.Wait();
Assert.IsTrue(task.Result.Failed);
Assert.IsNotNull(task.Result.EndOfProcessTraceEvent.NodeName == "ensure keyed");
Assert.IsNotNull(task.Result.StreamStatisticErrors.FirstOrDefault(i => i.NodeName == "ensure keyed"));
CollectionAssert.AreEquivalent(new[] { 2 }.ToList(), outputList);
#endregion
}
static void Main(string[] args)
{
var task = StreamProcessRunner.CreateAndExecuteAsync(
// new ImportFilesConfig { InputFilesRootFolderPath = @"C:\Users\sroyer\Downloads\RBC" },
new ImportFilesConfig {
InputFilesRootFolderPath = @"C:\Users\paill\Documents\GitHub\Etl.Net\src\Samples\TestFiles\RBC"
}
, ImportFiles.DefineProcess
, traceStream => traceStream.ThroughAction("trace", i => System.Diagnostics.Debug.WriteLine(i))
);
try
{
task.Wait();
}
catch (Exception ex)
{
var jex = ex.InnerException as Paillave.Etl.Core.JobExecutionException;
var te = jex.TraceEvent;
}
}
static async Task Main9(string[] args)
{
var processRunner = StreamProcessRunner.Create <string>(DefineProcess2);
processRunner.DebugNodeStream += (sender, e)
=>
{ /* PLACE A CONDITIONAL BREAKPOINT HERE FOR DEBUG ex: e.NodeName == "parse file" */ };
using (var dbCtx = new SimpleTutorialDbContext(args[1]))
{
var executionOptions = new ExecutionOptions <string>
{
Resolver = new SimpleDependencyResolver().Register <DbContext>(dbCtx),
TraceProcessDefinition = (ts, cs) => ts.Do("Show trace on console", t => Console.WriteLine(t.ToString())),
// TraceProcessDefinition = DefineTraceProcess,
// UseDetailedTraces = true // activate only if per row traces are meant to be caught
};
var res = await processRunner.ExecuteAsync(args[0], executionOptions);
Console.Write(res.Failed ? "Failed" : "Succeeded");
}
}
public void MultiColumnsMultiComputationsPivot()
{
var inputList = Enumerable.Range(0, 10).Select(i => new { Key = i / 5, Value = i, Col = i % 2 }).OrderBy(i => i.Key).ToList();
var outputList = new List <string>();
#region multicolumns multicomputations pivot
StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream =>
{
rootStream
.CrossApplyEnumerable("list elements", config => config)
.EnsureSorted("ensure sorted", i => i.Key)
.Pivot("simple count", i => new
{
Id = i.Key,
EvensCount = AggregationOperators.Count().For(i.Col == 0),
EvensSum = AggregationOperators.Sum(i.Value).For(i.Col == 0),
EvensFirst = AggregationOperators.First(i.Value).For(i.Col == 0),
EvensLast = AggregationOperators.Last(i.Value).For(i.Col == 0),
EvensMax = AggregationOperators.Max(i.Value).For(i.Col == 0),
EvensMin = AggregationOperators.Min(i.Value).For(i.Col == 0),
EvensAvg = AggregationOperators.Avg(i.Value).For(i.Col == 0),
OddsCount = AggregationOperators.Count().For(i.Col == 1),
OddsSum = AggregationOperators.Sum(i.Value).For(i.Col == 1),
OddsFirst = AggregationOperators.First(i.Value).For(i.Col == 1),
OddsLast = AggregationOperators.Last(i.Value).For(i.Col == 1),
OddsMax = AggregationOperators.Max(i.Value).For(i.Col == 1),
OddsMin = AggregationOperators.Min(i.Value).For(i.Col == 1),
OddsAvg = AggregationOperators.Avg(i.Value).For(i.Col == 1),
})
.Select("transform into string", SimpleSerializeToString)
.ThroughAction("collect values", outputList.Add);
}).Wait();
var expectedList = new[] {
"FirstValue:{ Key = 0, Value = 0, Col = 0 },Key:0,Aggregation:{ Id = 0, EvensCount = 3, EvensSum = 6, EvensFirst = 0, EvensLast = 4, EvensMax = 4, EvensMin = 0, EvensAvg = 2, OddsCount = 2, OddsSum = 4, OddsFirst = 1, OddsLast = 3, OddsMax = 3, OddsMin = 1, OddsAvg = 2 }",
"FirstValue:{ Key = 1, Value = 5, Col = 1 },Key:1,Aggregation:{ Id = 0, EvensCount = 2, EvensSum = 14, EvensFirst = 6, EvensLast = 8, EvensMax = 8, EvensMin = 6, EvensAvg = 7, OddsCount = 3, OddsSum = 21, OddsFirst = 5, OddsLast = 9, OddsMax = 9, OddsMin = 5, OddsAvg = 7 }"
}.ToList();
CollectionAssert.AreEquivalent(expectedList, outputList);
#endregion
}
public void ProduceListWithOneItem()
{
#region produce list with one "resource" item
var inputList = Enumerable.Range(0, 100).ToList();
var inputResource = 1;
var outputList = new List <int>();
StreamProcessRunner.CreateAndExecuteAsync(new
{
InputResource = inputResource,
InputList = inputList
}, rootStream =>
{
var resourceStream = rootStream.Select("get another resource", config => config.InputResource);
rootStream
.CrossApplyEnumerable("list elements", resourceStream, (config, res) => config.InputList.Select(i => i + res).ToList())
.ThroughAction("collect values", outputList.Add);
}).Wait();
CollectionAssert.AreEquivalent(inputList.Select(i => i + inputResource).ToList(), outputList);
#endregion
}
public void ComputeAverage()
{
#region compute average
var inputList = Enumerable.Range(0, 10).ToList();
var outputList = new List <string>();
StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream =>
{
rootStream
.CrossApplyEnumerable("list elements", config => config)
.Select("produce business object", i => new { Value = i, Category = $"CAT{i % 2}" })
.Aggregate("compute sum per category",
i => new { Nb = 0, Sum = 0 },
i => i.Category,
(agg, elt) => new { Nb = agg.Nb + 1, Sum = agg.Sum + elt.Value })
.Select("format and compute result", i => $"{i.Key}:{i.Aggregation.Sum / i.Aggregation.Nb}")
.ThroughAction("collect values", outputList.Add);
}).Wait();
CollectionAssert.AreEquivalent(new[] { "CAT0:4", "CAT1:5" }.ToList(), outputList);
#endregion
}
public void GroupElements()
{
#region group elements
var inputList = Enumerable.Range(0, 10).ToList();
var outputList = new List <string>();
StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream =>
{
rootStream
.CrossApplyEnumerable("list elements", config => config)
.Select("produce business object", i => new { Value = i, Category = $"CAT{i % 2}" })
.Aggregate("compute sum per category",
i => "",
i => i.Category,
(agg, elt) => $"{agg}{elt.Value}")
.Select("format result", i => $"{i.Key}:{i.Aggregation}")
.ThroughAction("collect values", outputList.Add);
}).Wait();
CollectionAssert.AreEquivalent(new[] { "CAT0:02468", "CAT1:13579" }.ToList(), outputList);
#endregion
}
public void LeftJoinWithSomeMatches()
{
#region simple left join with couple of matches
var inputLeftList = Enumerable.Range(0, 100).Select(i => new { Id = i, ForeignId = i / 10 }).ToList();
var inputRightList = new[] { 2, 5 }.Select(i => new { Id = i, Label = $"Label{i}" }).ToList();
var outputList = new List <string>();
StreamProcessRunner.CreateAndExecuteAsync(new
{
InputLeftList = inputLeftList,
InputRightList = inputRightList
}, rootStream =>
{
var leftStream = rootStream
.CrossApplyEnumerable("input left elements", config => config.InputLeftList)
.EnsureSorted("ensure left is sorted", i => i.ForeignId);
var rightStream = rootStream
.CrossApplyEnumerable("input right elements", config => config.InputRightList)
.EnsureKeyed("ensure right is keyed", i => i.Id);
leftStream
.LeftJoin("join left and right", rightStream, (left, right) => $"{left.Id}-{right?.Label}")
.ThroughAction("collect values", outputList.Add);
}).Wait();
var expected = Enumerable.Range(0, 100).Select(i =>
{
if (new[] { 2, 5 }.Contains(i / 10))
{
return($"{i}-Label{i / 10}");
}
else
{
return($"{i}-");
}
}).ToList();
CollectionAssert.AreEquivalent(expected, outputList);
#endregion
}
public void EmptyPivot()
{
#region simple pivot
var inputList = new int[] { }.Select(i => new { Key = i / 5, Value = i }).ToList();
var outputList = new List <string>();
StreamProcessRunner.CreateAndExecuteAsync(inputList, rootStream =>
{
rootStream
.CrossApplyEnumerable("list elements", config => config)
.Pivot("simple count", i => i.Key, i => new
{
Id = i.Key,
Count = AggregationOperators.Count()
})
.Select("transform into string", SimpleSerializeToString)
.ThroughAction("collect values", outputList.Add);
}).Wait();
var expectedList = new string[] { }.ToList();
CollectionAssert.AreEquivalent(expectedList, outputList);
#endregion
}
public void InlineMethodWay()
{
var config = new SimpleConfigStreamType {
Divider = 0
};
var task = StreamProcessRunner.CreateAndExecuteAsync(config, SimpleJob.Job1);
try
{
task.Wait();
Assert.Fail("the execution should not be successfull");
}
catch (AggregateException ex)
{
JobExecutionException jobExecutionException = ex.InnerException as JobExecutionException;
if (jobExecutionException == null)
{
throw;
}
Assert.IsInstanceOfType(jobExecutionException.InnerException, typeof(DivideByZeroException));
Assert.IsInstanceOfType((jobExecutionException.TraceEvent?.Content as UnhandledExceptionStreamTraceContent)?.Exception, typeof(DivideByZeroException));
}
}
static void Main(string[] args)
{
var runner = new StreamProcessRunner <ComplexQuickstartJob, MyConfig>();
runner.GetDefinitionStructure().OpenEstimatedExecutionPlanVisNetwork();
StreamProcessDefinition <TraceEvent> traceStreamProcessDefinition = new StreamProcessDefinition <TraceEvent>(traceStream => traceStream.ToAction("logs to console", Console.WriteLine));
var testFilesDirectory = @"C:\Users\sroyer\Source\Repos\Etl.Net\src\Samples\TestFiles";
// var testFilesDirectory = @"C:\Users\paill\source\repos\Etl.Net\src\Samples\TestFiles";
var task = runner.ExecuteAsync(new MyConfig
{
InputFolderPath = Path.Combine(testFilesDirectory, @"."),
InputFilesSearchPattern = "testin.*.csv",
TypeFilePath = Path.Combine(testFilesDirectory, @"ref - Copy.csv"),
DestinationFilePath = Path.Combine(testFilesDirectory, @"outfile.csv"),
CategoryDestinationFilePath = Path.Combine(testFilesDirectory, @"categoryStats.csv")
}, traceStreamProcessDefinition);
task.Result.OpenActualExecutionPlanD3Sankey();
Console.WriteLine("Done");
Console.WriteLine("Press a key...");
Console.ReadKey();
}
public void ProduceSubValuesWithToApplyParallelized()
{
#region produce sub values without pre/post process with to apply parallelized
var inputList = Enumerable.Range(10, 100).Select(i => $"{i}").ToList();
var outputList = new List <string>();
StreamProcessRunner.CreateAndExecuteAsync(new
{
InputList = inputList,
Multiplier = 2
}, rootStream =>
{
var toApplyStream = rootStream.Select("get multiplier", config => config.Multiplier);
rootStream
.CrossApplyEnumerable("list elements", config => config.InputList)
.CrossApply <string, int, string>("produce sub values", toApplyStream, (input, toApply, pushValue) => input.Select(i => $"{i}*{toApply}").ToList().ForEach(pushValue))
.ThroughAction("collect values", outputList.Add);
}).Wait();
var expected = inputList.SelectMany(i => i.Select(j => $"{j}*2")).OrderBy(i => i).ToList();
var actual = outputList.OrderBy(i => i).ToList();
CollectionAssert.AreEquivalent(expected, actual);
#endregion
}
public void SimpleLookup()
{
#region simple lookup
var inputLeftList = Enumerable.Range(0, 100).Select(i => new { Id = i, ForeignId = i / 10 }).ToList();
var inputRightList = Enumerable.Range(0, 10).Select(i => new { Id = i, Label = $"Label{i}" }).ToList();
var outputList = new List <string>();
StreamProcessRunner.CreateAndExecuteAsync(new
{
InputLeftList = inputLeftList,
InputRightList = inputRightList
}, rootStream =>
{
var leftStream = rootStream.CrossApplyEnumerable("input left elements", config => config.InputLeftList);
var rightStream = rootStream.CrossApplyEnumerable("input right elements", config => config.InputRightList);
leftStream
.Lookup("join left and right", rightStream, i => i.ForeignId, i => i.Id, (left, right) => $"{left.Id}-{right.Label}")
.ThroughAction("collect values", outputList.Add);
}).Wait();
var expected = Enumerable.Range(0, 100).Select(i => $"{i}-Label{i / 10}").ToList();
CollectionAssert.AreEquivalent(expected, outputList);
#endregion
}
public void ProduceSubValuesWithToApplyPrePost()
{
#region produce sub values without pre/post process with to apply PrePost
var inputList = Enumerable.Range(10, 100).Select(i => $"{i}").ToList();
var outputList = new List <string>();
StreamProcessRunner.CreateAndExecuteAsync(new
{
InputList = inputList,
Multiplier = 2
}, rootStream =>
{
var toApplyStream = rootStream.Select("get multiplier", config => config.Multiplier);
rootStream
.CrossApplyEnumerable("list elements", config => config.InputList, true)
.CrossApply <string, int, int, int, string>("produce sub values", toApplyStream, (input, toApply, pushValue) => input.ToString().Select(i => int.Parse(i.ToString()) * toApply).ToList().ForEach(pushValue), (i, j) => int.Parse(i) + 2, (val, initVal, toApply) => $"{val / toApply}-{initVal}")
.ThroughAction("collect values", outputList.Add);
}).Wait();
var expected = inputList.SelectMany(i => (int.Parse(i) + 2).ToString().Select(j => int.Parse(j.ToString()) * 2).Select(j => $"{j / 2}-{i}").ToList()).ToList();
var actual = outputList;
CollectionAssert.AreEquivalent(expected, actual);
#endregion
}
static async Task Main3(string[] args)
{
var anyArrayOfByte = new byte[] { };
var res = await StreamProcessRunner.CreateAndExecuteAsync <Stream>(new MemoryStream(anyArrayOfByte), DefineProcess20);
}
static async Task Main(string[] args)
{
var processRunner = StreamProcessRunner.Create <string[]>(Import);
var res = await processRunner.ExecuteAsync(args);
}