private static void DefineProcess(ISingleStream <string> contextStream)
{
contextStream
.CrossApplyFolderFiles("list all required files", "*.zip", true)
.CrossApplyZipFiles("extract files from zip", "*.csv")
.CrossApplyTextFile("parse file", FlatFileDefinition.Create(i => new Person
{
Email = i.ToColumn("email"),
FirstName = i.ToColumn("first name"),
LastName = i.ToColumn("last name"),
DateOfBirth = i.ToDateColumn("date of birth", "yyyy-MM-dd"),
Reputation = i.ToNumberColumn <int?>("reputation", ".")
}).IsColumnSeparated(','))
.Distinct("exclude duplicates based on the Email", i => i.Email)
.SqlServerSave("upsert using Email as key and ignore the Id", o => o
.ToTable("dbo.Person")
.SeekOn(p => p.Email)
.DoNotSave(p => p.Id))
.Select("define row to report", i => new { i.Email, i.Id })
.ToTextFileValue("write summary to file", "report.csv", FlatFileDefinition.Create(i => new
{
Email = i.ToColumn("Email"),
Id = i.ToNumberColumn <int>("new or existing Id", ".")
}).IsColumnSeparated(','))
.WriteToFile("save log file", i => i.Name);
}
static void Main(string[] args)
{
StreamProcessRunner.CreateAndExecuteAsync(
new
{
InputFilePath = Path.Combine(args[0], "simpleinputfile.csv"),
OutputFilePath = Path.Combine(args[0], "simpleoutputfile.csv")
},
rootStream =>
{
var outputFileS = rootStream.Select("open output file", i => (Stream)File.OpenWrite(i.OutputFilePath));
rootStream
.CrossApplyTextFile("read input file",
FlatFileDefinition.Create(i => new
{
Id = i.ToColumn <int>("#"),
Name = i.ToColumn <string>("Label"),
ValueType = i.ToColumn <string>("Value"),
CategoryCode = i.ToColumn <string>("Category")
}).IsColumnSeparated('\t'),
i => i.InputFilePath)
.ThroughAction("Write input file to console",
i => Console.WriteLine($"{i.Id}->{i.Name}->{i.CategoryCode}->{i.ValueType}"))
.Pivot("group and count",
i => i.CategoryCode,
i => new
{
Count = AggregationOperators.Count(),
CountA = AggregationOperators.Count().For(i.ValueType == "a"),
CountB = AggregationOperators.Count().For(i.ValueType == "b"),
})
.Select("create output row",
i => new
{
CategoryCode = i.Key,
i.Aggregation.Count,
i.Aggregation.CountA,
i.Aggregation.CountB
})
.Sort("sort output values",
i => new { i.CategoryCode })
.ThroughTextFile("write to text file",
outputFileS,
FlatFileDefinition.Create(i => new
{
CategoryCode = i.ToColumn <string>("MyCategoryCode"),
Count = i.ToColumn <int>("Count"),
CountA = i.ToColumn <int>("CountA"),
CountB = i.ToColumn <int>("CountB")
}));
}).Wait();
Console.WriteLine("Press a key...");
Console.ReadKey();
}
private void Tmp(ISingleStream <int> processContextStream)
{
var efaNavFileDefinition = FlatFileDefinition.Create(i => new
{
ShareCode = i.ToColumn <string>("Share_code"),
SubFundCode = i.ToColumn <string>("Sub-fund_code"),
IsinCode = i.ToColumn <string>("Isin_code"),
ShareCurrency = i.ToColumn <string>("CCY_NAV_share"),
NavDate = i.ToDateColumn("Valuation_date", "dd/MM/yyyy"),
NavPerShare = i.ToNumberColumn <double>("NAV_share", "."),
NumberOfSharesOutstanding = i.ToNumberColumn <double>("Numbre_of_Shares_outstanding", "."),
TotalNetAsset = i.ToNumberColumn <double>("Total_Net_Assets", "."),
NetAssetShareType = i.ToNumberColumn <double>("Net_assets_share_type", "."),
}).IsColumnSeparated(',');
var fileStream = processContextStream
.CrossApplyFolderFiles($"get local files", i => "/home/stephane/Desktop/zret", "*.csv", false);
var navFileStream = fileStream
.CrossApplyTextFile($"parse nav file", efaNavFileDefinition);
var managedSubFundStream = processContextStream.EfCoreSelect($"get subfunds from db", (c, i) => c.Set <SubFund>());
}
public static void Import(ISingleStream <string[]> contextStream)
{
var portfolioFileStream = contextStream
.FromConnector("Get portfolio files", "PTF")
.CrossApplyTextFile("Parse portfolio file", FlatFileDefinition.Create(i => new
{
SicavCode = i.ToColumn("SicavCode"),
SicavName = i.ToColumn("SicavName"),
SicavType = i.ToColumn("SicavType"),
PortfolioCode = i.ToColumn("PortfolioCode"),
PortfolioName = i.ToColumn("PortfolioName")
}).IsColumnSeparated(','))
.SetForCorrelation("Correlate portfolio row");
var positionFileStream = contextStream
.FromConnector("Get position files", "POS")
.CrossApplyTextFile("Parse position file", FlatFileDefinition.Create(i => new
{
PortfolioCode = i.ToColumn("PortfolioCode"),
SecurityCode = i.ToColumn("SecurityCode"),
Isin = i.ToColumn("Isin"),
SecurityName = i.ToColumn("SecurityName"),
SecurityClass = i.ToColumn("SecurityClass"),
Issuer = i.ToColumn("Issuer"),
Date = i.ToDateColumn("Date", "yyyyMMdd"),
Value = i.ToNumberColumn <decimal>("Value", "."),
}).IsColumnSeparated(','))
.SetForCorrelation("Correlate position row");
var sicavStream = portfolioFileStream
.Distinct("Distinct sicav", i => i.SicavCode, true)
.Select("Create sicav", i => new DataAccess.Sicav
{
InternalCode = i.SicavCode,
Name = i.SicavName,
Type = i.SicavType == "UCITS" ? DataAccess.SicavType.UCITS : DataAccess.SicavType.AIFM
})
.EfCoreSave("Save sicav", o => o
.SeekOn(i => i.InternalCode)
.DoNotUpdateIfExists());
var portfolioStream = portfolioFileStream
.Distinct("Distinct portfolio", i => i.PortfolioCode, true)
.CorrelateToSingle("Get related sicav and create portfolio", sicavStream, (row, sicav) => new DataAccess.Portfolio
{
InternalCode = row.PortfolioCode,
Name = row.PortfolioName,
SicavId = sicav.Id
})
.EfCoreSave("Save portfolio", o => o
.SeekOn(i => i.InternalCode)
.DoNotUpdateIfExists());
var compositionStream = positionFileStream
.Distinct("Distinct compositions", i => new { i.PortfolioCode, i.Date }, true)
.Lookup("Get related portfolio and create composition",
portfolioStream,
i => i.PortfolioCode,
i => i.InternalCode,
(row, portfolio) => new DataAccess.Composition
{
Date = row.Date,
PortfolioId = portfolio.Id
})
.EfCoreSave("Save composition", o => o
.SeekOn(i => new { i.Date, i.PortfolioId })
.DoNotUpdateIfExists());
var securityStream = positionFileStream
.Distinct("Distinct securities", i => i.SecurityCode)
.Select("Create security", i =>
{
if (string.IsNullOrWhiteSpace(i.SecurityClass))
{
return(new DataAccess.Equity
{
InternalCode = i.SecurityCode,
Name = i.SecurityName,
Isin = i.Isin,
Issuer = i.Issuer
} as DataAccess.Security);
}
return(new DataAccess.ShareClass
{
InternalCode = i.SecurityCode,
Name = i.SecurityName,
Isin = i.Isin,
Class = i.SecurityClass
} as DataAccess.Security);
})
.EfCoreSave("Save security", o => o
.SeekOn(i => i.Isin)
.AlternativelySeekOn(i => i.InternalCode)
.DoNotUpdateIfExists());
positionFileStream
.CorrelateToSingle("Get related security", securityStream, (row, security) => new { Row = row, SecurityId = security.Id })
.CorrelateToSingle("Get related composition and create position", compositionStream, (row, composition) => new DataAccess.Position
{
Value = row.Row.Value,
SecurityId = row.SecurityId,
CompositionId = composition.Id
})
.Distinct("Distinct positions", i => new { i.CompositionId, i.SecurityId }, o => o.ForProperty(i => i.Value, DistinctAggregator.Sum))
.EfCoreSave("Save position");
}
private static void DefineProcess5(ISingleStream <string> stream)
{
stream
.CrossApply("produce a list of dummy values for ", _ => Enumerable.Range(0, 100).Select(idx => new { Index = idx, Name = $"Index {idx}", CategoryId = idx % 3 }))
.GroupBy("process per group", i => i.CategoryId, (subStream, firstRow) => subStream
.Select("create row to save", i => new { i.Index, i.Name })
.ToTextFileValue("save into csv file", $"fileExport{firstRow?.CategoryId}.csv", FlatFileDefinition.Create(i => new
{
Index = i.ToNumberColumn <int>("Idx", "."),
Name = i.ToColumn("Title")
})))
.WriteToFile("write to folder", i => i.Name);
}
private static void DefineProcess6(ISingleStream <string> stream)
{
var streamOfRows = stream
.CrossApply("produce a list of dummy values for ", _ => Enumerable.Range(0, 100).Select(idx => new { Index = idx, Name = $"Index {idx}", CategoryId = idx % 3 }));
var fileStream = streamOfRows
.Select("create row to save", i => new { i.Index, i.Name })
.ToTextFileValue("save into csv file", $"fileExport.csv", FlatFileDefinition.Create(i => new
{
Index = i.ToNumberColumn <int>("Idx", "."),
Name = i.ToColumn("Title")
}).IsColumnSeparated(','));
fileStream.WriteToFile("write to folder", i => i.Name);
}
