protected void ProcessValueToOutput(SystemIO.Stream stream, FlatFileDefinition <TIn> mapping, TIn value) { _streamWriter.WriteLine(_serialize.Serialize(value)); }
public static ISortedStream <TIn, TKey> ThroughTextFile <TIn, TKey>(this ISortedStream <TIn, TKey> stream, string name, ISingleStream <SystemIO.Stream> resourceStream, FlatFileDefinition <TIn> mapping) { return(new ThroughFlatFileStreamNode <TIn, ISortedStream <TIn, TKey> >(name, new ThroughFlatFileArgs <TIn, ISortedStream <TIn, TKey> > { MainStream = stream, Mapping = mapping, TargetStream = resourceStream }).Output); }
public static IStream <TOut> CrossApplyTextFile <TParsed, TOut>(this IStream <Stream> stream, string name, FlatFileDefinition <TParsed> args, Func <TParsed, TOut> resultSelector, bool noParallelisation = false) where TParsed : new() { return(stream.CrossApply(name, new FlatFileValuesProvider <Stream, TParsed, TOut>(new FlatFileValuesProviderArgs <Stream, TParsed, TOut>() { DataStreamSelector = i => i, Mapping = args, NoParallelisation = noParallelisation, ResultSelector = (s, o) => resultSelector(o) }), i => i, (i, _) => i)); }
private void PreProcess(SystemIO.Stream stream, FlatFileDefinition <TIn> mapping) { _streamWriter = new StreamWriter(stream, Encoding.Default, 1024, true); this.ExecutionContext.AddDisposable(_streamWriter); _streamWriter.WriteLine(mapping.GenerateDefaultHeaderLine()); }
public static IStream <TOut> CrossApplyTextFile <TIn, TParsed, TOut>(this IStream <TIn> stream, string name, FlatFileDefinition <TParsed> args, Func <TIn, string> filePathSelector, Func <TIn, TParsed, TOut> resultSelector, bool noParallelisation = false) where TParsed : new() { return(stream.CrossApply(name, new FlatFileValuesProvider <TIn, TParsed, TOut>(new FlatFileValuesProviderArgs <TIn, TParsed, TOut>() { DataStreamSelector = i => File.OpenRead(filePathSelector(i)), Mapping = args, NoParallelisation = noParallelisation, ResultSelector = resultSelector }), i => i, (i, _) => i)); }
public static IKeyedStream <TIn, TKey> ToTextFile <TIn, TKey>(this IKeyedStream <TIn, TKey> stream, string name, IStream <SystemIO.StreamWriter> resourceStream, FlatFileDefinition <TIn> mapping) where TIn : new() { return(new ToFlatFileStreamNode <TIn, IKeyedStream <TIn, TKey> >(name, new ToFlatFileArgs <TIn, IKeyedStream <TIn, TKey> > { MainStream = stream, Mapping = mapping, TargetStream = resourceStream }).Output); }
public static IStream <TOut> CrossApplyTextFile <TOut>(this IStream <string> stream, string name, FlatFileDefinition <TOut> args, bool noParallelisation = false) where TOut : new() { return(stream.CrossApply(name, new FlatFileValuesProvider <string, TOut, TOut>(new FlatFileValuesProviderArgs <string, TOut, TOut>() { DataStreamSelector = i => File.OpenRead(i), Mapping = args, NoParallelisation = noParallelisation, ResultSelector = (i, o) => o }), i => i, (i, _) => i)); }
public static void Import(ISingleStream <string[]> contextStream) { var portfolioFileStream = contextStream .FromConnector("Get portfolio files", "PTF") .CrossApplyTextFile("Parse portfolio file", FlatFileDefinition.Create(i => new { SicavCode = i.ToColumn("SicavCode"), SicavName = i.ToColumn("SicavName"), SicavType = i.ToColumn("SicavType"), PortfolioCode = i.ToColumn("PortfolioCode"), PortfolioName = i.ToColumn("PortfolioName") }).IsColumnSeparated(',')) .SetForCorrelation("Correlate portfolio row"); var positionFileStream = contextStream .FromConnector("Get position files", "POS") .CrossApplyTextFile("Parse position file", FlatFileDefinition.Create(i => new { PortfolioCode = i.ToColumn("PortfolioCode"), SecurityCode = i.ToColumn("SecurityCode"), Isin = i.ToColumn("Isin"), SecurityName = i.ToColumn("SecurityName"), SecurityClass = i.ToColumn("SecurityClass"), Issuer = i.ToColumn("Issuer"), Date = i.ToDateColumn("Date", "yyyyMMdd"), Value = i.ToNumberColumn <decimal>("Value", "."), }).IsColumnSeparated(',')) .SetForCorrelation("Correlate position row"); var sicavStream = portfolioFileStream .Distinct("Distinct sicav", i => i.SicavCode, true) .Select("Create sicav", i => new DataAccess.Sicav { InternalCode = i.SicavCode, Name = i.SicavName, Type = i.SicavType == "UCITS" ? DataAccess.SicavType.UCITS : DataAccess.SicavType.AIFM }) .EfCoreSave("Save sicav", o => o .SeekOn(i => i.InternalCode) .DoNotUpdateIfExists()); var portfolioStream = portfolioFileStream .Distinct("Distinct portfolio", i => i.PortfolioCode, true) .CorrelateToSingle("Get related sicav and create portfolio", sicavStream, (row, sicav) => new DataAccess.Portfolio { InternalCode = row.PortfolioCode, Name = row.PortfolioName, SicavId = sicav.Id }) .EfCoreSave("Save portfolio", o => o .SeekOn(i => i.InternalCode) .DoNotUpdateIfExists()); var compositionStream = positionFileStream .Distinct("Distinct compositions", i => new { i.PortfolioCode, i.Date }, true) .Lookup("Get related portfolio", portfolioStream, i => i.PortfolioCode, i => i.InternalCode, (row, portfolio) => new { Portfolio = portfolio, Composition = new DataAccess.Composition { Date = row.Date, PortfolioId = portfolio.Id } }) .EfCoreSave("Save composition", o => o .Entity(i => i.Composition) .SeekOn(i => new { i.Date, i.PortfolioId }) .DoNotUpdateIfExists() .Output((i, e) => new { i.Portfolio, Composition = e })); var securityStream = positionFileStream .Distinct("Distinct securities", i => i.SecurityCode) .Select("Create security", i => { if (string.IsNullOrWhiteSpace(i.SecurityClass)) { return(new DataAccess.Equity { InternalCode = i.SecurityCode, Name = i.SecurityName, Isin = i.Isin, Issuer = i.Issuer } as DataAccess.Security); } return(new DataAccess.ShareClass { InternalCode = i.SecurityCode, Name = i.SecurityName, Isin = i.Isin, Class = i.SecurityClass } as DataAccess.Security); }) .EfCoreSave("Save security", o => o .SeekOn(i => i.Isin) .AlternativelySeekOn(i => i.InternalCode) .DoNotUpdateIfExists()); positionFileStream .CorrelateToSingle("Get related security", securityStream, (row, security) => new { Row = row, SecurityId = security.Id }) .CorrelateToSingle("Get related composition and create position", compositionStream, (row, composition) => new DataAccess.Position { Value = row.Row.Value, SecurityId = row.SecurityId, CompositionId = composition.Composition.Id }) .Distinct("Distinct positions", i => new { i.CompositionId, i.SecurityId }, o => o.ForProperty(i => i.Value, DistinctAggregator.Sum)) .EfCoreSave("Save position") .CorrelateToSingle("Get position portfolio", compositionStream, (p, c) => new { c.Portfolio.InternalCode, c.Composition.Date, p.Value }) .Distinct("Aggregate position per portfolio", i => new { i.InternalCode, i.Date }, o => o.ForProperty(i => i.Value, DistinctAggregator.Sum)) .ToTextFileValue("Export portfolios weight", "PortfoliosWeight.csv", FlatFileDefinition.Create(i => new { InternalCode = i.ToColumn("Code"), Date = i.ToDateColumn("Date", "ddd dd MMM, yyyy"), Value = i.ToNumberColumn <decimal>("Weight", "."), }).IsColumnSeparated(',')) .ToConnector("Save portfolios position file", "OUT"); }
private void PreProcess(SystemIO.StreamWriter streamWriter, FlatFileDefinition <TIn> mapping) { streamWriter.WriteLine(mapping.GenerateDefaultHeaderLine()); }