protected void ProcessValueToOutput(SystemIO.Stream stream, FlatFileDefinition <TIn> mapping, TIn value)
{
_streamWriter.WriteLine(_serialize.Serialize(value));
}
public static ISortedStream <TIn, TKey> ThroughTextFile <TIn, TKey>(this ISortedStream <TIn, TKey> stream, string name, ISingleStream <SystemIO.Stream> resourceStream, FlatFileDefinition <TIn> mapping)
{
return(new ThroughFlatFileStreamNode <TIn, ISortedStream <TIn, TKey> >(name, new ThroughFlatFileArgs <TIn, ISortedStream <TIn, TKey> >
{
MainStream = stream,
Mapping = mapping,
TargetStream = resourceStream
}).Output);
}
public static IStream <TOut> CrossApplyTextFile <TParsed, TOut>(this IStream <Stream> stream, string name, FlatFileDefinition <TParsed> args, Func <TParsed, TOut> resultSelector, bool noParallelisation = false) where TParsed : new()
{
return(stream.CrossApply(name, new FlatFileValuesProvider <Stream, TParsed, TOut>(new FlatFileValuesProviderArgs <Stream, TParsed, TOut>()
{
DataStreamSelector = i => i,
Mapping = args,
NoParallelisation = noParallelisation,
ResultSelector = (s, o) => resultSelector(o)
}), i => i, (i, _) => i));
}
private void PreProcess(SystemIO.Stream stream, FlatFileDefinition <TIn> mapping)
{
_streamWriter = new StreamWriter(stream, Encoding.Default, 1024, true);
this.ExecutionContext.AddDisposable(_streamWriter);
_streamWriter.WriteLine(mapping.GenerateDefaultHeaderLine());
}
public static IStream <TOut> CrossApplyTextFile <TIn, TParsed, TOut>(this IStream <TIn> stream, string name, FlatFileDefinition <TParsed> args, Func <TIn, string> filePathSelector, Func <TIn, TParsed, TOut> resultSelector, bool noParallelisation = false) where TParsed : new()
{
return(stream.CrossApply(name, new FlatFileValuesProvider <TIn, TParsed, TOut>(new FlatFileValuesProviderArgs <TIn, TParsed, TOut>()
{
DataStreamSelector = i => File.OpenRead(filePathSelector(i)),
Mapping = args,
NoParallelisation = noParallelisation,
ResultSelector = resultSelector
}), i => i, (i, _) => i));
}
public static IKeyedStream <TIn, TKey> ToTextFile <TIn, TKey>(this IKeyedStream <TIn, TKey> stream, string name, IStream <SystemIO.StreamWriter> resourceStream, FlatFileDefinition <TIn> mapping) where TIn : new()
{
return(new ToFlatFileStreamNode <TIn, IKeyedStream <TIn, TKey> >(name, new ToFlatFileArgs <TIn, IKeyedStream <TIn, TKey> >
{
MainStream = stream,
Mapping = mapping,
TargetStream = resourceStream
}).Output);
}
public static IStream <TOut> CrossApplyTextFile <TOut>(this IStream <string> stream, string name, FlatFileDefinition <TOut> args, bool noParallelisation = false)
where TOut : new()
{
return(stream.CrossApply(name, new FlatFileValuesProvider <string, TOut, TOut>(new FlatFileValuesProviderArgs <string, TOut, TOut>()
{
DataStreamSelector = i => File.OpenRead(i),
Mapping = args,
NoParallelisation = noParallelisation,
ResultSelector = (i, o) => o
}), i => i, (i, _) => i));
}
public static void Import(ISingleStream <string[]> contextStream)
{
var portfolioFileStream = contextStream
.FromConnector("Get portfolio files", "PTF")
.CrossApplyTextFile("Parse portfolio file", FlatFileDefinition.Create(i => new
{
SicavCode = i.ToColumn("SicavCode"),
SicavName = i.ToColumn("SicavName"),
SicavType = i.ToColumn("SicavType"),
PortfolioCode = i.ToColumn("PortfolioCode"),
PortfolioName = i.ToColumn("PortfolioName")
}).IsColumnSeparated(','))
.SetForCorrelation("Correlate portfolio row");
var positionFileStream = contextStream
.FromConnector("Get position files", "POS")
.CrossApplyTextFile("Parse position file", FlatFileDefinition.Create(i => new
{
PortfolioCode = i.ToColumn("PortfolioCode"),
SecurityCode = i.ToColumn("SecurityCode"),
Isin = i.ToColumn("Isin"),
SecurityName = i.ToColumn("SecurityName"),
SecurityClass = i.ToColumn("SecurityClass"),
Issuer = i.ToColumn("Issuer"),
Date = i.ToDateColumn("Date", "yyyyMMdd"),
Value = i.ToNumberColumn <decimal>("Value", "."),
}).IsColumnSeparated(','))
.SetForCorrelation("Correlate position row");
var sicavStream = portfolioFileStream
.Distinct("Distinct sicav", i => i.SicavCode, true)
.Select("Create sicav", i => new DataAccess.Sicav
{
InternalCode = i.SicavCode,
Name = i.SicavName,
Type = i.SicavType == "UCITS" ? DataAccess.SicavType.UCITS : DataAccess.SicavType.AIFM
})
.EfCoreSave("Save sicav", o => o
.SeekOn(i => i.InternalCode)
.DoNotUpdateIfExists());
var portfolioStream = portfolioFileStream
.Distinct("Distinct portfolio", i => i.PortfolioCode, true)
.CorrelateToSingle("Get related sicav and create portfolio", sicavStream, (row, sicav) => new DataAccess.Portfolio
{
InternalCode = row.PortfolioCode,
Name = row.PortfolioName,
SicavId = sicav.Id
})
.EfCoreSave("Save portfolio", o => o
.SeekOn(i => i.InternalCode)
.DoNotUpdateIfExists());
var compositionStream = positionFileStream
.Distinct("Distinct compositions", i => new { i.PortfolioCode, i.Date }, true)
.Lookup("Get related portfolio",
portfolioStream,
i => i.PortfolioCode,
i => i.InternalCode,
(row, portfolio) => new
{
Portfolio = portfolio,
Composition = new DataAccess.Composition
{
Date = row.Date,
PortfolioId = portfolio.Id
}
})
.EfCoreSave("Save composition", o => o
.Entity(i => i.Composition)
.SeekOn(i => new { i.Date, i.PortfolioId })
.DoNotUpdateIfExists()
.Output((i, e) => new
{
i.Portfolio,
Composition = e
}));
var securityStream = positionFileStream
.Distinct("Distinct securities", i => i.SecurityCode)
.Select("Create security", i =>
{
if (string.IsNullOrWhiteSpace(i.SecurityClass))
{
return(new DataAccess.Equity
{
InternalCode = i.SecurityCode,
Name = i.SecurityName,
Isin = i.Isin,
Issuer = i.Issuer
} as DataAccess.Security);
}
return(new DataAccess.ShareClass
{
InternalCode = i.SecurityCode,
Name = i.SecurityName,
Isin = i.Isin,
Class = i.SecurityClass
} as DataAccess.Security);
})
.EfCoreSave("Save security", o => o
.SeekOn(i => i.Isin)
.AlternativelySeekOn(i => i.InternalCode)
.DoNotUpdateIfExists());
positionFileStream
.CorrelateToSingle("Get related security", securityStream, (row, security) => new { Row = row, SecurityId = security.Id })
.CorrelateToSingle("Get related composition and create position", compositionStream, (row, composition) => new DataAccess.Position
{
Value = row.Row.Value,
SecurityId = row.SecurityId,
CompositionId = composition.Composition.Id
})
.Distinct("Distinct positions", i => new { i.CompositionId, i.SecurityId }, o => o.ForProperty(i => i.Value, DistinctAggregator.Sum))
.EfCoreSave("Save position")
.CorrelateToSingle("Get position portfolio", compositionStream, (p, c) => new { c.Portfolio.InternalCode, c.Composition.Date, p.Value })
.Distinct("Aggregate position per portfolio", i => new { i.InternalCode, i.Date }, o => o.ForProperty(i => i.Value, DistinctAggregator.Sum))
.ToTextFileValue("Export portfolios weight", "PortfoliosWeight.csv", FlatFileDefinition.Create(i => new
{
InternalCode = i.ToColumn("Code"),
Date = i.ToDateColumn("Date", "ddd dd MMM, yyyy"),
Value = i.ToNumberColumn <decimal>("Weight", "."),
}).IsColumnSeparated(','))
.ToConnector("Save portfolios position file", "OUT");
}
private void PreProcess(SystemIO.StreamWriter streamWriter, FlatFileDefinition <TIn> mapping)
{
streamWriter.WriteLine(mapping.GenerateDefaultHeaderLine());
}
