static void Main(string[] args)
{
var runner = StreamProcessRunner.Create <MyConfig>(ComplexQuickstartJob.DefineProcess);
Action <IStream <TraceEvent> > traceStreamProcessDefinition = traceStream => traceStream.Where("errors only", i => i.Content.Level == TraceLevel.Error).ThroughAction("logs to console", Console.WriteLine);
var testFilesDirectory = @"C:\Users\paill\Documents\GitHub\Etl.Net\src\Samples\TestFiles";
var config = new MyConfig
{
InputFolderPath = Path.Combine(testFilesDirectory, @"."),
InputFilesSearchPattern = "testin.*.csv",
TypeFilePath = Path.Combine(testFilesDirectory, @"ref - Copy.csv"),
DestinationFilePath = Path.Combine(testFilesDirectory, @"outfile.csv"),
CategoryDestinationFilePath = Path.Combine(testFilesDirectory, @"categoryStats.csv")
};
//var config2 = new MySimpleConfig
//{
// InputFilesPath = Path.Combine(testFilesDirectory, "testin.1.csv"),
// TypeFilePath = Path.Combine(testFilesDirectory, @"ref - Copy.csvu"),
//};
var task = runner.ExecuteAsync(config, traceStreamProcessDefinition, true);
task.Wait();
task = runner.ExecuteAsync(config, traceStreamProcessDefinition, true);
task.Wait();
Console.WriteLine("Done");
Console.WriteLine("Press a key...");
Console.ReadKey();
}
static async Task Main2(string[] args)
{
var processRunner = StreamProcessRunner.Create <string>(DefineProcess11);
var structure = processRunner.GetDefinitionStructure();
processRunner.DebugNodeStream += (sender, e) => { };
// System.IO.File.WriteAllText(
// "execplan.json",
// Newtonsoft.Json.JsonConvert.SerializeObject(structure, Newtonsoft.Json.Formatting.Indented)
// );
// var structure = processRunner.GetDefinitionStructure();
// structure.OpenEstimatedExecutionPlan();
using (var cnx = new SqlConnection(args[1]))
{
cnx.Open();
var executionOptions = new ExecutionOptions <string>
{
Resolver = new SimpleDependencyResolver().Register(cnx),
};
var res = await processRunner.ExecuteAsync(args[0], executionOptions);
// System.IO.File.WriteAllText(
// "execplan.json",
// Newtonsoft.Json.JsonConvert.SerializeObject(res.StreamStatisticCounters, Newtonsoft.Json.Formatting.Indented)
// );
// res.OpenActualExecutionPlan();
Console.Write(res.Failed ? "Failed" : "Succeeded");
if (res.Failed)
{
Console.Write($"{res.ErrorTraceEvent.NodeName}({res.ErrorTraceEvent.NodeTypeName}):{res.ErrorTraceEvent.Content.Message}");
}
}
}
/// <summary>
/// Example 2: Import and export files showing the estimated execution plan, the real time evolution of the process, and the actual execution plan
/// </summary>
/// <param name="args"></param>
/// <returns></returns>
static async Task ImportAndCreateFileAsync(string[] args)
{
var processRunner = StreamProcessRunner.Create <string[]>(TestImport2.Import);
var structure = processRunner.GetDefinitionStructure();
structure.OpenEstimatedExecutionPlan();
ITraceReporter traceReporter = new AdvancedConsoleExecutionDisplay();
var executionOptions = new ExecutionOptions <string[]>
{
Connectors = new FileValueConnectors()
.Register(new FileSystemFileValueProvider("PTF", "Portfolios", Path.Combine(Environment.CurrentDirectory, "InputFiles"), "*.Portfolios.csv"))
.Register(new FileSystemFileValueProvider("POS", "Positions", Path.Combine(Environment.CurrentDirectory, "InputFiles"), "*.Positions.csv"))
.Register(new FileSystemFileValueProcessor("OUT", "Result", Path.Combine(Environment.CurrentDirectory, "InputFiles"))),
Resolver = new SimpleDependencyResolver()
.Register(new DataAccess.TestDbContext()),
TraceProcessDefinition = traceReporter.TraceProcessDefinition,
};
traceReporter.Initialize(structure);
var res = await processRunner.ExecuteAsync(args, executionOptions);
res.OpenActualExecutionPlan();
}
public void StaticMethodWay()
{
var runner = StreamProcessRunner.Create <SimpleConfigStreamType>(SimpleJob.Job1);
var config = new SimpleConfigStreamType {
Divider = 10
};
var task = runner.ExecuteAsync(config);
task.Wait();
CollectionAssert.AreEquivalent(new[] { $"{100 / 10} times hello world!" }, config.Messages.ToArray());
}
static async Task Main(string[] args)
{
var processRunner = StreamProcessRunner.Create <string>(DefineProcess102);
var executionOptions = new ExecutionOptions <string>
{
// TraceProcessDefinition = DefineTraceProcess
};
var res = await processRunner.ExecuteAsync(args[0], executionOptions);
// var res2 = await processRunner.ExecuteAsync(args[0], executionOptions);
}
/// <summary>
/// Example 3: Import and export files using a config file to setup connectors
/// </summary>
/// <param name="args"></param>
/// <returns></returns>
static async Task ImportAndCreateFileWithConfigAsync(string[] args)
{
var processRunner = StreamProcessRunner.Create <string[]>(TestImport2.Import);
var executionOptions = new ExecutionOptions <string[]>
{
Connectors = CreateConfigurationFileValueConnectorParser()
.GetConnectors(File.ReadAllText(Path.Combine(Environment.CurrentDirectory, "connectorsConfig.json"))),
Resolver = new SimpleDependencyResolver()
.Register(new DataAccess.TestDbContext())
};
var res = await processRunner.ExecuteAsync(args, executionOptions);
}
public void StaticMethodWay()
{
var runner = StreamProcessRunner.Create <SimpleConfigStreamType>(SimpleJob.Job1);
var config = new SimpleConfigStreamType {
Divider = 0
};
var task = runner.ExecuteWithNoFaultAsync(config);
task.Wait();
Assert.IsTrue(task.Result.Failed, "the execution should not be successfull");
Assert.IsInstanceOfType((task.Result.EndOfProcessTraceEvent?.Content as UnhandledExceptionStreamTraceContent)?.Exception, typeof(DivideByZeroException));
}
/// <summary>
/// Example 1: simple importation of a set of files
/// </summary>
/// <param name="args"></param>
/// <returns></returns>
static async Task SimplyImportAsync(string[] args)
{
var processRunner = StreamProcessRunner.Create <string[]>(TestImport.Import);
// processRunner.GetDefinitionStructure().OpenEstimatedExecutionPlan();
var output = await processRunner.ExecuteAsync(args, new ExecutionOptions <string[]>
{
Connectors = new FileValueConnectors()
.Register(new FileSystemFileValueProvider("PTF", "Portfolios", Path.Combine(Environment.CurrentDirectory, "InputFiles"), "*.Portfolios.csv"))
.Register(new FileSystemFileValueProvider("POS", "Positions", Path.Combine(Environment.CurrentDirectory, "InputFiles"), "*.Positions.csv")),
Resolver = new SimpleDependencyResolver()
.Register(new DataAccess.TestDbContext()),
});
}
static async Task Main(string[] args)
{
var processRunner = StreamProcessRunner.Create <string>(DefineProcess1);
processRunner.DebugNodeStream += (sender, e) => { };
var res = await processRunner.ExecuteAsync(args[0]);
Console.Write(res.Failed ? "Failed" : "Succeeded");
if (res.Failed)
{
Console.Write($"{res.ErrorTraceEvent.NodeName}({res.ErrorTraceEvent.NodeTypeName}):{res.ErrorTraceEvent.Content.Message}");
}
}
static void Main(string[] args)
{
var testFilesDirectory = @"C:\Users\sroyer\Source\Repos\Etl.Net\src\Samples\TestFiles";
// var testFilesDirectory = @"C:\Users\paill\Documents\GitHub\Etl.Net\src\Samples\TestFiles";
StreamProcessRunner.Create <SimpleConfig>(ExcelQuickstartJob.DefineProcess).ExecuteAsync(new SimpleConfig
{
InputDirectory = testFilesDirectory,
OutputFile = @"C:\Users\sroyer\Source\Repos\Etl.Net\src\Samples\testoutput.xlsx"
}, null).Wait();
Console.WriteLine("Press a key...");
Console.ReadKey();
}
static async Task Main3(string[] args)
{
var processRunner = StreamProcessRunner.Create <string>(DefineProcess);
var executionOptions = new ExecutionOptions <string>
{
Resolver = new SimpleDependencyResolver()
.Register(new SomeExternalValue
{
AStringValue = "injected string",
AnIntValue = 658
})
};
var res = await processRunner.ExecuteAsync("transmitted parameter", executionOptions);
}
static void Main(string[] args)
{
StreamProcessRunner.Create <SimpleConfig>(FtpQuickstartJob.DefineProcess).ExecuteAsync(new SimpleConfig
{
ConnectionInfo = new FtpConnectionInfo
{
Login = "******",
Password = "******",
Server = "localhost"
},
Folder = "SubFolder"
}, null).Wait();
Console.WriteLine("Press a key...");
Console.ReadKey();
}
public void Test1()
{
var sw = new Stopwatch();
var streamProcessRunner = StreamProcessRunner.Create(
(ISingleStream <int> configStream) => configStream
.CrossApplyEnumerable <int, int>("test", i => Enumerable.Range(0, i))
.Observable.Count().Do(i => Console.WriteLine($"Count={i}")));
sw.Start();
Console.WriteLine("Starting...");
var tmp = streamProcessRunner.ExecuteAsync(2);
Console.WriteLine("Started");
tmp.Wait();
Console.WriteLine($"Done {!tmp.Result.Failed}");
}
static async Task Main9(string[] args)
{
var processRunner = StreamProcessRunner.Create <string>(DefineProcess2);
processRunner.DebugNodeStream += (sender, e)
=>
{ /* PLACE A CONDITIONAL BREAKPOINT HERE FOR DEBUG ex: e.NodeName == "parse file" */ };
using (var dbCtx = new SimpleTutorialDbContext(args[1]))
{
var executionOptions = new ExecutionOptions <string>
{
Resolver = new SimpleDependencyResolver().Register <DbContext>(dbCtx),
TraceProcessDefinition = (ts, cs) => ts.Do("Show trace on console", t => Console.WriteLine(t.ToString())),
// TraceProcessDefinition = DefineTraceProcess,
// UseDetailedTraces = true // activate only if per row traces are meant to be caught
};
var res = await processRunner.ExecuteAsync(args[0], executionOptions);
Console.Write(res.Failed ? "Failed" : "Succeeded");
}
}
static void Main(string[] args)
{
var runner = StreamProcessRunner.Create <MyConfig>(SubProcessQuickstartJob.DefineProcess);
runner.GetDefinitionStructure().OpenEstimatedExecutionPlanVisNetwork();
Action <IStream <TraceEvent> > traceStreamProcessDefinition = traceStream => traceStream.ThroughAction("logs to console", Console.WriteLine);
var testFilesDirectory = @"C:\Users\sroyer\Source\Repos\Etl.Net\src\Samples\TestFiles";
// var testFilesDirectory = @"C:\Users\paill\source\repos\Etl.Net\src\Samples\TestFiles";
var task = runner.ExecuteAsync(new MyConfig
{
InputFolderPath = Path.Combine(testFilesDirectory, @"."),
InputFilesSearchPattern = "testin.*.csv",
TypeFilePath = Path.Combine(testFilesDirectory, @"ref - Copy.csv"),
DestinationFilePath = Path.Combine(testFilesDirectory, @"outfile.csv"),
CategoryDestinationFolder = Path.Combine(testFilesDirectory, @"categoryStats")
}, traceStreamProcessDefinition);
task.Result.OpenActualExecutionPlanD3Sankey();
Console.WriteLine("Done");
Console.WriteLine("Press a key...");
Console.ReadKey();
}
public void StaticMethodWay()
{
var runner = StreamProcessRunner.Create <SimpleConfigStreamType>(SimpleJob.Job1);
var config = new SimpleConfigStreamType {
Divider = 0
};
var task = runner.ExecuteAsync(config);
try
{
task.Wait();
Assert.Fail("the execution should not be successfull");
}
catch (AggregateException ex)
{
JobExecutionException jobExecutionException = ex.InnerException as JobExecutionException;
if (jobExecutionException == null)
{
throw;
}
Assert.IsInstanceOfType(jobExecutionException.InnerException, typeof(DivideByZeroException));
Assert.IsInstanceOfType((jobExecutionException.TraceEvent?.Content as UnhandledExceptionStreamTraceContent)?.Exception, typeof(DivideByZeroException));
}
}
static async Task Main(string[] args)
{
var processRunner = StreamProcessRunner.Create <string[]>(Import);
var res = await processRunner.ExecuteAsync(args);
}