private static void DefineProcess1(ISingleStream <string> contextStream)
{
contextStream
.CrossApplyFolderFiles("get files", "*.txt")
.CrossApply("parse file", BloombergValuesProvider.Create(i => new
{
CountryFullName = i.ToColumn("COUNTRY_FULL_NAME"),
IndustrySector = i.ToColumn("INDUSTRY_SECTOR"),
IndustrySectorNum = i.ToColumn("INDUSTRY_SECTOR_NUM"),
EqyShOut = i.ToNumberColumn <double?>("EQY_SH_OUT", "."),
EqyShOutActual = i.ToNumberColumn <double?>("EQY_SH_OUT_ACTUAL", "."),
EqyFloat = i.ToNumberColumn <double?>("EQY_FLOAT", "."),
CrncyAdjMktCap = i.ToNumberColumn <double?>("CRNCY_ADJ_MKT_CAP", "."),
HisVolOnPx1 = i.ToNumberColumn <double?>("HIS_VOL_ON_PX:1", "."),
HisVolOnPx2 = i.ToNumberColumn <double?>("HIS_VOL_ON_PX:2", "."),
MhisCloseOnPx3 = i.ToNumberColumn <double?>("MHIS_CLOSE_ON_PX:3", "."),
MhisCloseOnPx4 = i.ToNumberColumn <double?>("MHIS_CLOSE_ON_PX:4", "."),
CountryIso = i.ToColumn("COUNTRY_ISO"),
DvdExDt = i.ToOptionalDateColumn("DVD_EX_DT", "MM/dd/yyyy"),
DvdFreq = i.ToColumn("DVD_FREQ"),
EqyDvdRightExDtCurr = i.ToColumn("EQY_DVD_RIGHT_EX_DT_CURR"),
SecurityTyp = i.ToColumn("SECURITY_TYP"),
PeRatio = i.ToNumberColumn <double?>("PE_RATIO", "."),
EqyDvdYld12m = i.ToNumberColumn <double?>("EQY_DVD_YLD_12M", "."),
}))
.Do("write to console", i =>
{
});
}
private static void DefineProcess(ISingleStream <string> contextStream)
{
contextStream
.ResolveAndSelect("get some values", o => o
.Resolve <SomeExternalValue>()
.Select((context, injected) => $"{context}-{injected.AStringValue}:{injected.AnIntValue}"));
}
private static void DefineProcess20(ISingleStream <Stream> contextStream)
{
// System.Text.Json.JsonSerializer
// Newtonsoft.Json.JsonSerializer
contextStream
.Select("Create file value", i => FileValue.Create(i, i is FileStream fileStream ? fileStream.Name : "fileName.csv", "from stream"))
.CrossApply <IFileValue, Person>("parse file", (fileValue, dependencyResolver, cancellationToken, push) =>
{
var parsedFile = Newtonsoft.Json.Linq.JObject.Parse(new StreamReader(fileValue.GetContent()).ReadToEnd());
var jPeople = parsedFile["people"] as JArray;
foreach (var jPerson in jPeople)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
push(new Person
{
FirstName = (string)jPerson["TheFirstName"],
LastName = (string)jPerson["TheLastName"],
Email = (string)jPerson["TheEmail"],
});
}
})
.Do("write to console", i => Console.WriteLine(i.Email));
}
private static void DefineProcess(ISingleStream <string> contextStream)
{
contextStream
.CrossApplyFolderFiles("list all required files", "*.zip", true)
.CrossApplyZipFiles("extract files from zip", "*.csv")
.CrossApplyTextFile("parse file", FlatFileDefinition.Create(i => new Person
{
Email = i.ToColumn("email"),
FirstName = i.ToColumn("first name"),
LastName = i.ToColumn("last name"),
DateOfBirth = i.ToDateColumn("date of birth", "yyyy-MM-dd"),
Reputation = i.ToNumberColumn <int?>("reputation", ".")
}).IsColumnSeparated(','))
.Distinct("exclude duplicates based on the Email", i => i.Email)
.SqlServerSave("upsert using Email as key and ignore the Id", o => o
.ToTable("dbo.Person")
.SeekOn(p => p.Email)
.DoNotSave(p => p.Id))
.Select("define row to report", i => new { i.Email, i.Id })
.ToTextFileValue("write summary to file", "report.csv", FlatFileDefinition.Create(i => new
{
Email = i.ToColumn("Email"),
Id = i.ToNumberColumn <int>("new or existing Id", ".")
}).IsColumnSeparated(','))
.WriteToFile("save log file", i => i.Name);
}
public static void DefineProcess(ISingleStream <MyConfig> rootStream)
{
var outputFileResourceS = rootStream.Select("open output file", i => (Stream)File.OpenWrite(i.DestinationFilePath));
var outputCategoryResourceS = rootStream.Select("open output category file", i => (Stream)File.OpenWrite(i.CategoryDestinationFilePath));
var parsedLineS = rootStream
.CrossApplyFolderFiles("get folder files", i => i.InputFolderPath, i => i.InputFilesSearchPattern, (f, r) => f.Name)
.CrossApplyTextFile("parse input file", new InputFileRowMapper(), (i, p) => { p.FileName = i; return(p); });
var parsedTypeLineS = rootStream
.Select("get input file type path", i => i.TypeFilePath)
.CrossApplyTextFile("parse type input file", new TypeFileRowMapper());
var joinedLineS = parsedLineS
.Lookup("join types to file", parsedTypeLineS, i => i.TypeId, i => i.Id, (l, r) => new { l.Id, r.Name, l.FileName, r.Category });
var categoryStatistics = joinedLineS
.Pivot("create statistic for categories", i => i.Category, i => new { Count = AggregationOperators.Count(), Total = AggregationOperators.Sum(i.Id) })
.Select("create output category data", i => new OutputCategoryRow {
Category = i.Key, AmountOfEntries = i.Aggregation.Count, TotalAmount = i.Aggregation.Total
})
.ThroughTextFile("write category statistics to file", outputCategoryResourceS, new OutputCategoryRowMapper());
joinedLineS.Select("create output data", i => new OutputFileRow {
Id = i.Id, Name = i.Name, FileName = i.FileName
})
.ThroughTextFile("write to output file", outputFileResourceS, new OutputFileRowMapper())
.ThroughAction("write to console", i => Console.WriteLine($"{i.FileName}:{i.Id}-{i.Name}"));
}
public static void DefineProcess(ISingleStream <object> rootStream)
{
rootStream
.CrossApplyEnumerable("create some values", (input) => Enumerable.Range(0, 10).Select(i => new MyInputType {
Id = i, Value = (i % 3 == 0) ? i : (int?)null
}))
.Select("set null value to the previous not null value", new MySelectProcessor());
}
public static ISingleStream <Correlated <T> > FixNull <T>(this ISingleStream <Correlated <T> > stream, string name, Func <Fixer <T>, Fixer <T> > fixer)
{
return(new FixCorrelatedSingleStreamNode <T>(name, new FixCorrelatedSingleArgs <T>
{
Fixer = fixer(new Fixer <T>()),
Stream = stream
}).Output);
}
public static ISingleStream <Correlated <TIn> > DoCorrelatedAndResolve <TIn>(this ISingleStream <Correlated <TIn> > stream, string name, Func <DoWithResolutionProcessorBuilder <Correlated <TIn>, TIn>, IDoProcessor <Correlated <TIn> > > o)
{
return(new DoStreamNode <Correlated <TIn>, ISingleStream <Correlated <TIn> > >(name, new DoArgs <Correlated <TIn>, ISingleStream <Correlated <TIn> > >
{
Processor = o(new DoWithResolutionProcessorBuilder <Correlated <TIn>, TIn>(i => i.Row)),
Stream = stream
}).Output);
}
public static ISingleStream <TIn> Do <TIn>(this ISingleStream <TIn> stream, string name, Action <TIn> processRow)
{
return(new DoStreamNode <TIn, ISingleStream <TIn> >(name, new DoArgs <TIn, ISingleStream <TIn> >
{
Processor = new SimpleDoProcessor <TIn, TIn>(i => i, processRow),
Stream = stream
}).Output);
}
public static ISingleStream <Correlated <TIn> > DoCorrelated <TIn>(this ISingleStream <Correlated <TIn> > stream, string name, Action <TIn> processRow)
{
return(new DoStreamNode <Correlated <TIn>, ISingleStream <Correlated <TIn> > >(name, new DoArgs <Correlated <TIn>, ISingleStream <Correlated <TIn> > >
{
Processor = new SimpleDoProcessor <Correlated <TIn>, TIn>(i => i.Row, processRow),
Stream = stream
}).Output);
}
public static ISingleStream <TOut> Select <TIn, TOut>(this ISingleStream <TIn> stream, string name, Func <TIn, TOut> resultSelector, bool excludeNull = false)
{
return(new SelectSingleStreamNode <TIn, TOut>(name, new SelectSingleArgs <TIn, TOut>
{
Stream = stream,
Processor = new SimpleSelectProcessor <TIn, TOut>(resultSelector),
ExcludeNull = excludeNull
}).Output);
}
public static ISingleStream <TOut> Select <TIn, TOut, TCtx>(this ISingleStream <TIn> stream, string name, TCtx initialContext, Func <TIn, int, TCtx, Action <TCtx>, TOut> resultSelector, bool excludeNull = false)
{
return(new SelectSingleWithIndexStreamNode <TIn, TOut>(name, new SelectSingleWithIndexArgs <TIn, TOut>
{
Stream = stream,
Processor = new ContextSelectWithIndexProcessor <TIn, TOut, TCtx>(resultSelector, initialContext),
ExcludeNull = excludeNull
}).Output);
}
public static ISingleStream <TOut> Select <TIn, TOut>(this ISingleStream <TIn> stream, string name, ISelectWithIndexProcessor <TIn, TOut> processor, bool excludeNull = false)
{
return(new SelectSingleWithIndexStreamNode <TIn, TOut>(name, new SelectSingleWithIndexArgs <TIn, TOut>
{
Stream = stream,
Processor = processor,
ExcludeNull = excludeNull
}).Output);
}
public static ISingleStream <TOut> ResolveAndSelect <TIn, TService, TOut>(this ISingleStream <TIn> stream, string name, Func <Resolver <TIn>, Selector <TIn, TService, TOut> > selection, bool withNoDispose = false) where TService : class
{
return(new ResolveAndSelectSingleStreamNode <TIn, TService, TOut>(name, new ResolveAndSelectSingleArgs <TIn, TService, TOut>
{
Stream = stream,
Selection = selection,
WithNoDispose = withNoDispose
}).Output);
}
public static ISingleStream <Correlated <TOut> > Select <TIn, TOut>(this ISingleStream <Correlated <TIn> > stream, string name, ISelectProcessor <Correlated <TIn>, Correlated <TOut> > processor, bool withNoDispose = false)
{
return(new SelectSingleStreamNode <Correlated <TIn>, Correlated <TOut> >(name, new SelectSingleArgs <Correlated <TIn>, Correlated <TOut> >
{
Stream = stream,
Processor = processor,
WithNoDispose = withNoDispose
}).Output);
}
public static ISingleStream <TOut> Select <TIn, TOut, TCtx>(this ISingleStream <TIn> stream, string name, TCtx initialContext, Func <TIn, TCtx, Action <TCtx>, TOut> resultSelector, bool withNoDispose = false)
{
return(new SelectSingleStreamNode <TIn, TOut>(name, new SelectSingleArgs <TIn, TOut>
{
Stream = stream,
Processor = new ContextSelectProcessor <TIn, TOut, TCtx>(resultSelector, initialContext),
WithNoDispose = withNoDispose
}).Output);
}
public static ISingleStream <TOut> Select <TIn, TOut>(this ISingleStream <TIn> stream, string name, Func <TIn, TOut> resultSelector, bool withNoDispose = false)
{
return(new SelectSingleStreamNode <TIn, TOut>(name, new SelectSingleArgs <TIn, TOut>
{
Stream = stream,
Processor = new SimpleSelectProcessor <TIn, TOut>(resultSelector),
WithNoDispose = withNoDispose
}).Output);
}
private static void DefineProcess9(ISingleStream <string> contextStream)
{
contextStream
.CrossApplySqlServerQuery("get people", o => o.FromQuery("select * from dbo.Person as p").WithMapping(i => new
{
Name = i.ToColumn("FirstName"),
Birthday = i.ToDateColumn("DateOfBirth")
}));
}
private static void DefineProcess71(ISingleStream <string> contextStream)
{
// var tmp = 0;
// contextStream
// .Select("zzer", i => 1 / tmp);
contextStream
.CrossApply("create values from enumeration", ctx => Enumerable
.Range(1, 100)
.Select(i => new { Id = i, OutputId = i % 10, label = $"Label{i}" }));
}
public static void DefineProcess(ISingleStream <object> rootStream)
{
rootStream
.CrossApplyEnumerable("create some values", (input) => Enumerable.Range(0, 10))
.Aggregate("aggregate values for average computation",
i => new { sum = 0, nb = 0 },
i => i % 3,
(previousAggr, value) => new { sum = previousAggr.sum + value, nb = previousAggr.nb + 1 })
.Select("compute average", i => new { i.Key, Avg = i.Aggregation.sum / i.Aggregation.nb });
}
public static void Import(ISingleStream <string[]> contextStream)
{
var portfolioFileStream = contextStream.CrossApply("Get Files", new FtpFileValueProvider("SRC", "Solar exports", "files from ftp", new FtpAdapterConnectionParameters {
Server = "localhost",
Login = "******",
Password = "******"
}, new FtpAdapterProviderParameters {
}))
.Do("print files to console", i => Console.WriteLine(i.Name));
}
public static ISingleStream <TOut> Select <TIn1, TIn2, TOut>(this ISingleStream <TIn1> stream, string name, ISingleStream <TIn2> streamToApply, Func <TIn1, TIn2, TOut> resultSelector, bool excludeNull = false)
{
return(new ApplySingleStreamNode <TIn1, TIn2, TOut>(name, new ApplySingleArgs <TIn1, TIn2, TOut>
{
MainStream = stream,
StreamToApply = streamToApply,
Selector = resultSelector,
ExcludeNull = excludeNull
}).Output);
}
private static void DefineProcess11(ISingleStream <string> contextStream)
{
contextStream
.CrossApply("build criteria", i => new[] {
new { Reputation = 345, NewReputation = 346 },
new { Reputation = 45, NewReputation = 201 }
})
.ToSqlCommand("update reputation", "update p set Reputation = @NewReputation from dbo.Person as p where p.Reputation = @Reputation")
.ToSqlCommand("update reputation like before", "update p set Reputation = @Reputation from dbo.Person as p where p.Reputation = @NewReputation");
}
private static void DefineProcess10(ISingleStream <string> contextStream)
{
contextStream
.Select("build criteria", i => new { Reputation = 345 })
.CrossApplySqlServerQuery("get people", o => o.FromQuery("select * from dbo.Person as p where p.Reputation = @Reputation").WithMapping(i => new
{
Name = i.ToColumn("FirstName"),
Birthday = i.ToDateColumn("DateOfBirth")
}));
}
public static ISingleStream <TOut> Select <TIn1, TIn2, TOut>(this ISingleStream <TIn1> stream, string name, ISingleStream <TIn2> streamToApply, Func <TIn1, TIn2, TOut> resultSelector, bool withNoDispose = false)
{
return(new ApplySingleStreamNode <TIn1, TIn2, TOut>(name, new ApplySingleArgs <TIn1, TIn2, TOut>
{
MainStream = stream,
StreamToApply = streamToApply,
Selector = resultSelector,
WithNoDispose = withNoDispose
}).Output);
}
public static ISingleStream <Correlated <TOut> > Select <TIn, TOut>(this ISingleStream <Correlated <TIn> > stream, string name, Func <TIn, TOut> resultSelector, bool withNoDispose = false)
{
return(new SelectSingleStreamNode <Correlated <TIn>, Correlated <TOut> >(name, new SelectSingleArgs <Correlated <TIn>, Correlated <TOut> >
{
Stream = stream,
Processor = new SimpleSelectProcessor <Correlated <TIn>, Correlated <TOut> >(i => new Correlated <TOut> {
Row = resultSelector(i.Row), CorrelationKeys = i.CorrelationKeys
}),
WithNoDispose = withNoDispose
}).Output);
}
public static void DefineSimpleProcess(ISingleStream <MySimpleConfig> rootStream)
{
var parsedLineS = rootStream
.CrossApplyTextFile("parse input file", new InputFileRowMapper(), i => i.InputFilesPath);
var parsedTypeLineS = rootStream
.CrossApplyTextFile("parse type input file", new TypeFileRowMapper(), i => i.TypeFilePath);
var joinedLineS = parsedLineS
.Lookup("join types to file", parsedTypeLineS, i => i.TypeId, i => i.Id, (l, r) => new { l.Id, r.Name, l.FileName, r.Category });
}
public static void DefineProcess(ISingleStream <object> rootStream)
{
rootStream
.CrossApplyEnumerable("create some values", (input) => Enumerable.Range(0, 10).Select(i => new { Id = i, Value = (i % 3 == 0) ? i : (int?)null }))
.Select("set null value to the previous not null value", 0, (i, ctx, setCtx) =>
{
var v = i.Value ?? ctx;
setCtx(v);
return(new { i.Id, Value = v });
});
}
private static void DefineProcess7(ISingleStream <string> stream)
{
stream
.CrossApply("produce a list of dummy values for ", _ => Enumerable.Range(0, 100).Select(idx => new { Index = idx, Name = $"Index {idx}", CategoryId = idx % 3 }))
.GroupBy("process per group", i => i.CategoryId)
.Select("create file", rows => FileValueWriter
.Create($"otherFileExport{rows.FirstValue.CategoryId}.txt")
.WriteLine($"here is the list of indexes in the category {rows.FirstValue.CategoryId}")
.Write(String.Join(", ", rows.Aggregation.Select(row => row.Name).ToList())))
.WriteToFile("write to folder", i => i.Name);
}
public static void DefineProcess(ISingleStream <ImportFilesConfig> config)
{
config
.CrossApplyFolderFiles("get all Nav files", i => i.InputFilesRootFolderPath, "*NAVPUBLTEXTRACT*.csv", true)
.CrossApplyTextFile("parse nav file", new RbcNavFileDefinition())
.ThroughAction("write nav to output", i => Console.WriteLine(i.IsinCode));
config
.CrossApplyFolderFiles("get all position files", i => i.InputFilesRootFolderPath, "*PORTFVALEXTRACT*.csv", true)
.CrossApplyTextFile("parse position file", new RbcPositionFileDefinition(), i => i.Name)
.ThroughAction("write position to output", i => Console.WriteLine(i.FundName));
}
