static async Task Covid19()
{
var ctx = new ETLContext();
ctx.ConfigureService(cfg =>
{
cfg.UseElastic <ElasticDbCovidSettings>(ctx.Configuration, s => s.OnCreate(settings => { settings.DefaultIndex("covid"); }), typeof(Covid19Schema));
});
var job = ctx.CreateJob();
var ff = ctx.GetService <TRest <AllLiveFranceData> >();
ff.RequestUri = "https://coronavirusapi-france.now.sh/AllDataByDate?date=2020-11-20";
var log = ctx.GetService <TLogRow <Covid19Schema> >();
log.AddInput(job, ff.OnOutput.SelectMany(x => x.AllFranceDataByDate));
log.Mode = TLogRowMode.Table;
log.ShowItemNumber = true;
var dboutuput = ctx.GetService <TOutputDb <Covid19Schema, ObjectId> >();
dboutuput.AddInput(job, ff.OnOutput.SelectMany(x => x.AllFranceDataByDate));
ff.AddToJob(job);
await job.Start();
while (!job.IsCompleted)
{
Thread.Sleep(200);
}
}
static async Task Weather()
{
var ctx = new ETLContext();
ctx.ConfigureService(cfg =>
{
/* MONGO */
//cfg.UseMongoDb<MongoDbBookstoreSettings>(ctx.Configuration, null, typeof(OpenWeatherMapSchema));
/* ELASTIC */
cfg.UseElastic <ElasticDbBookstoreSettings>(ctx.Configuration, s => s.OnCreate(settings => { settings.DefaultIndex("weather"); }), typeof(OpenWeatherMapSchema));
});
var job = ctx.CreateJob();
// var dboutuput = ctx.GetService<TOutputDb<OpenWeatherMapSchema, ObjectId>>();
var ff = ctx.GetService <TOpenWeather>();
ff.City = "delle";
ff.ApiKey = "d288da12b207992dd796241cf56014b1";
ff.OnError.Subscribe(err => { Console.Error.WriteLine($"{err.Message}\n{err.InnerException.Message}"); });
ff.AddToJob(job);
//dboutuput.AddInput(job, ff.OnOutput);
ff.OnOutput.Subscribe(new DebugObserver <OpenWeatherMapSchema>());
await job.Start();
while (!job.IsCompleted)
{
Thread.Sleep(200);
}
}
public void TestInitialize()
{
ctx = new ETLContext();
Assert.IsNotNull(ctx);
job = ctx.CreateJob();
Assert.IsNotNull(job);
scheduler = new TestScheduler();
job.OnStart.SubscribeOn(scheduler).Subscribe(j => Debug.WriteLine($"Job start at {j.Item2.ToShortTimeString()}"));
job.OnCompleted.SubscribeOn(scheduler).Subscribe(j => Debug.WriteLine($"Job completed in {j.Item1.ElapsedTime.Duration()}"));
}
static async Task ReadOfflineProgrammeTurf()
{
var ctx = new ETLContext();
ctx.ConfigureService(cfg => cfg.AddEntityFrameworkElastic());
var job = ctx.CreateJob();
var fileInput = ctx.GetService <TInputFile>();
fileInput.Filename = Path.Combine(ctx.ExecutionPath, "14112020.json");
fileInput.AddToJob(job);
var strToJson = ctx.GetService <TStringToJson <ProgrammeSchema> >();
strToJson.AddInput(job, fileInput.OnOutput);
strToJson.OnOutput.Subscribe(p =>
{
Console.WriteLine(p.Programme.Date.ToString());
});
strToJson.OnError.Subscribe(ex => Console.Error.WriteLine(ex.InnerException.Message));
await job.Start();
}
static async Task ReadOnlineProgrammeTurf()
{
var date = DateTime.Now.Subtract(TimeSpan.FromDays(1)).ToString("ddMMyyyy");
var ctx = new ETLContext();
ctx.ConfigureService(cfg =>
{
cfg.UseElastic <ElasticDbTurfSettings>(ctx.Configuration, s => s.OnCreate(settings => { settings.DefaultIndex("turf"); }), typeof(Programme));
});
var job = ctx.CreateJob();
job.CreateBag();
var forReunion = new TForEach <ProgrammeSchema, Reunion>();
var forCourse = new TForEach <(Reunion, int), Course>();
var action1 = new TAction <ProgrammeSchema>();
var restProgramme = ctx.GetService <TRest <ProgrammeSchema> >();
var log = ctx.GetService <TLogRow <Programme> >();
log.AddInput(job, restProgramme.OnOutput.Select(x => x.Programme));
log.Mode = TLogRowMode.Table;
log.ShowItemNumber = true;
restProgramme.RequestUri = $"https://online.turfinfo.api.pmu.fr/rest/client/1/programme/{date}?meteo=true&specialisation=INTERNET";
restProgramme.AddToJob(job);
//restProgramme.OnBeforeTransform
ProgrammeSchema programme = new ProgrammeSchema();
//programme.Programme.Reunions.WithIndex(e=>e.NumeroOfficiel/* (item, index) => (item,index,item.NumeroOfficiel)*/);
action1.AddInput(job, restProgramme.OnOutput);
action1.Set((Job job, ProgrammeSchema prg) =>
{
dynamic bag = job.Bag(job.Id.ToString());
bag.Programme = prg;
});
forReunion.AddIteration(job, restProgramme.OnOutput, (ProgrammeSchema prg) => prg.Programme.Reunions.Select(x => (x, x.NumeroOfficiel)));
forCourse.AddIteration(job, forReunion.OnOutput, (r) => r.Item1.Courses.Select(x => (x, r.Item2)));
var action2 = new TAction <(Course, int)>();
action2.AddInput(job, forCourse.OnOutput);
action2.Set((job, cc) =>
{
Console.WriteLine(cc.Item1.LibelleCourt);
var restParticipant = ctx.GetService <TRest <ListeParticipants> >();
restParticipant.AddToJob(job);
var r = cc.Item2;
var c = cc.Item1.Numero;
restParticipant.RequestUri = $"https://online.turfinfo.api.pmu.fr/rest/client/1/programme/{date}/R{r}/C{c}/participants?specialisation=INTERNET";
restParticipant.AddToJob(job);
});
var restParticipant = ctx.GetService <TRest <ListeParticipants> >();
// restParticipant.AddInput()
/* restProgramme.OnOutput.Subscribe(item =>
* {
* Console.WriteLine("receiving programme");
* programme = item;
* var row = item;
*
* foreach (var reunion in row.Programme.Reunions)
* {
* var r = reunion.NumeroOfficiel;
* foreach (var course in reunion.Courses)
* {
* var c = course.Numero;
* var restParticipant = ctx.GetService<TRest<ListeParticipants>>();
* restParticipant.RequestUri = $"https://online.turfinfo.api.pmu.fr/rest/client/1/programme/{date}/R{r}/C{c}/participants?specialisation=INTERNET";
*
*
* restParticipant.OnOutput.Subscribe(ps =>
* {
* foreach (var p in ps.Participants)
* {
* Console.WriteLine($"{course.Libelle} {p.Numero}->{p.Nom}");
*
* }
* course.Participants = ps.Participants;
* });
* restParticipant.OnError.Subscribe(ex => Console.Error.WriteLine(ex.InnerException));
* }
* }
*
*
* });*/
restProgramme.OnCompleted.Subscribe(j =>
{
Console.WriteLine("programme received");
});
restProgramme.OnError.Subscribe(ex =>
{
Console.Error.WriteLine(ex.InnerException.Message);
});
bool usedb = false;
if (usedb)
{
var dbout = ctx.GetService <TOutputDb <Programme, ObjectId> >();
dbout.AddInput(job, restProgramme.OnOutput.Select(x => x.Programme));
}
job.OnCompleted.Subscribe(job =>
{
Console.WriteLine("Job is completed");
});
await job.Start();
while (!job.IsCompleted)
{
Thread.Sleep(200);
}
}