public void LoadSecurities(string fileName, string source)
{
var fileLocation = string.Concat(_filePath, fileName);
_validFileInfo = new ValidFileInfo(fileLocation, source, FileTypes.Csv);
var valueDate = DateTime.Today.Date;
Func <DataTable, string, DateTime, Portfolios> consummer = PortfolioLoadingDelegate.Process;
Task <Portfolios> result = _dataLoader.LoadData(_validFileInfo, consummer, valueDate);
result.Wait();
var list = result.Result.ToList();
Assert.Greater(list.Count, 0);
list.ForEach(p =>
{
Assert.IsNotEmpty(p.Tenor);
Assert.Greater(p.PortfolioId, 0);
});
}
public static ValidFileInfo GetValidationInfo(LoaderConfig loaderConfig, IProgress <ProgressManager> progress)
{
var validFileInfo = new ValidFileInfo(loaderConfig.FileLocation, loaderConfig.Source, FileTypes.Txt);
if (progress != null)
{
validFileInfo.ProgressLogger = progress;
}
return(validFileInfo);
}
public static async Task <TimeSeries> GetTimeSeries(ValidFileInfo validFileInfo)
{
if (!Loaders.ContainsKey(validFileInfo.FileType))
{
throw new ArgumentException("Json file type has not be setup in the loaders configuration layer.");
}
var loader = Loaders[validFileInfo.FileType];
return(await loader.LoadData(validFileInfo));
}
public async Task <T> LoadData <T>(ValidFileInfo fileInfo, Func <DataTable, string, DateTime, T> dataConsummer, DateTime valueDate)
{
//return await Task.Run(() =>
//{
try
{
var oleDbConnectionString = new OleDbConnectionString(fileInfo);
var progressItem = new ProgressManager
{
TimeStamp = DateTime.UtcNow,
Progress = 1,
Action = "OleDbLoader Load data: ",
Message = "OleDbLoader Loader Load data started:" + fileInfo.Source
};
oleDbConnectionString.ValidFileInfo.ProgressLogger?.Report(progressItem);
var oledbConn = new System.Data.OleDb.OleDbConnection(oleDbConnectionString.Value);
var command = new System.Data.OleDb.OleDbDataAdapter(oleDbConnectionString.Sql, oledbConn);
command.TableMappings.Add("Table", "LoadTable");
var dtSet = new DataSet();
command.Fill(dtSet);
var table = dtSet.Tables[0];
var portfolios = dataConsummer.Invoke(table, fileInfo.Source, valueDate);
oledbConn.Close();
progressItem = new ProgressManager
{
TimeStamp = DateTime.UtcNow,
Progress = 1,
Action = "OleDbLoader Load data",
Message = "OleDbLoader Loader Load data stopped" + fileInfo.Source
};
oleDbConnectionString.ValidFileInfo.ProgressLogger?.Report(progressItem);
return(portfolios);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
return(default(T));
//});
}
public void LoadSecurities(string fileName, string source)
{
var fileLocation = string.Concat(_filePath, fileName);
_validFileInfo = new ValidFileInfo(fileLocation, source, FileTypes.Xml);
Task <TimeSeries> result = _dataLoader.LoadData(_validFileInfo);
result.Wait();
var list = result.Result.ToList();
Assert.Greater(list.Count, 0);
}
public OleDbConnectionString(ValidFileInfo fileInfo)
{
if (!fileInfo.FileType.Equals(FileTypes.Excel) && !fileInfo.FileType.Equals(FileTypes.Csv) && !fileInfo.FileType.Equals(FileTypes.Txt))
{
throw new ArgumentException("File format not setup for oledb loading");
}
ValidFileInfo = fileInfo;
TableName = $"[{fileInfo.Value.Name}$]";
Sql = $"select * from {TableName}";
if (fileInfo.FileType == FileTypes.Txt)
{
TableName = $"[{fileInfo.Value.Name}$]";
Sql = $"select * from [{fileInfo.Value.Name}]";
var provider = $"provider = Microsoft.Jet.OLEDB.4.0; Data Source = '{fileInfo.Value.Directory}\\';";
//var provider = $"provider = Microsoft.ACE.OLEDB.12.0; Data Source = '{fileInfo.Value.Directory}\\';";
Value = string.Concat(provider, "Extended Properties = 'text;HDR=Yes;FMT=Delimited(,)'; ");
return;
}
if (fileInfo.FileType == FileTypes.Excel)
{
//Value =
// $"provider = Microsoft.Jet.OLEDB.4.0; Data Source = '{fileInfo.Value.FullName}'; Extended Properties = Excel 8.0;";
Value =
$"provider = Microsoft.ACE.OLEDB.12.0; Data Source = '{fileInfo.Value.FullName}'; Extended Properties = Excel 12.0;";
return;
}
if (fileInfo.FileType == FileTypes.Csv)
{
//Value =
// $"Provider=Microsoft.Jet.OLEDB.4.0;Data Source='{fileInfo.Value.FullName}';Extended Properties=Text;HDR=Yes;FMT=Delimited;";
TableName = $"[{fileInfo.Value.Name}$]";
Sql = $"select * from [{fileInfo.Value.Name}]";
//var provider = $"provider = Microsoft.Jet.OLEDB.4.0; Data Source = '{fileInfo.Value.Directory}\\';";
var provider = $"provider = Microsoft.ACE.OLEDB.12.0; Data Source = '{fileInfo.Value.Directory}\\';";
Value = string.Concat(provider, "Extended Properties = 'text;HDR=Yes;FMT=Delimited(,)'; ");
}
}
public async Task <IEnumerable <T> > LoadData(ValidFileInfo fileInfo)
{
var stopWatch = new Stopwatch();
stopWatch.Start();
var progressItem = new ProgressManager
{
TimeStamp = DateTime.UtcNow,
Progress = 1,
Action = "JsonLoader Load data",
Message = "Json Loader Load data started"
};
fileInfo.ProgressLogger?.Report(progressItem);
using (StreamReader r = new StreamReader(fileInfo.Value.FullName))
{
var json = await r.ReadToEndAsync();
var items = JsonConvert.DeserializeObject <IEnumerable <T> >(json);
var results = items.OfType <IAudit>().Select(x =>
{
x.Source = fileInfo.Source;
x.LastUpdated = ConfigDataHandler.SystemName;
x.UpdateDate = DateTime.UtcNow;
return(x);
});
progressItem = new ProgressManager
{
TimeStamp = DateTime.UtcNow,
Progress = 1,
Action = "JsonLoader Load data",
Message = "Json Loader Load data started"
};
fileInfo.ProgressLogger?.Report(progressItem);
return(results.OfType <T>());
}
}
public void ValidateFileInfo()
{
var valid = new ValidFileInfo(path, "Source", FileTypes.Json);
Assert.IsNotNull(valid.Value);
}
public async Task <TimeSeries> LoadData(ValidFileInfo fileInfo)
{
return(await Task.Run(() =>
{
try
{
var oleDbConnectionString = new OleDbConnectionString(fileInfo);
var progressItem = new ProgressManager
{
TimeStamp = DateTime.UtcNow,
Progress = 1,
Action = "OleDbLoader Load data: ",
Message = "OleDbLoader Loader Load data started:" + fileInfo.Source
};
oleDbConnectionString.ValidFileInfo.ProgressLogger?.Report(progressItem);
var stopWatch = new Stopwatch();
stopWatch.Start();
var timeSeries = new TimeSeries {
Source = oleDbConnectionString.ValidFileInfo.Source
};
//;HDR=yes
//;Format=xlsx;IMEX=1
var oledbConn = new System.Data.OleDb.OleDbConnection(oleDbConnectionString.Value);
var command = new System.Data.OleDb.OleDbDataAdapter(oleDbConnectionString.Sql, oledbConn);
command.TableMappings.Add("Table", "LoadTable");
var dtSet = new DataSet();
command.Fill(dtSet);
var table = dtSet.Tables[0];
var dateCol = table.Columns["date"];
var closeCol = table.Columns["close"];
var nameCol = table.Columns["Name"];
foreach (DataRow tableRow in table.Rows)
{
var date = Utils.SafeCastToDate(tableRow[dateCol].ToString());
var close = Utils.SafeCastToDoubleParse(tableRow[closeCol].ToString());
var tsdp = new TimeSeriesDataPoint
{
Date = Utils.SafeCastToDate(date.ToString()),
Close = Utils.SafeCastToDoubleParse(close.ToString()),
Symbol = tableRow[nameCol].ToString(),
Source = oleDbConnectionString.ValidFileInfo.Source,
LastUpdated = ConfigDataHandler.SystemName,
UpdateDate = DateTime.UtcNow
};
timeSeries.Add(tsdp);
}
oledbConn.Close();
stopWatch.Stop();
timeSeries.LoadTime = stopWatch.Elapsed;
progressItem = new ProgressManager
{
TimeStamp = DateTime.UtcNow,
Progress = 1,
Action = "OleDbLoader Load data",
Message = "OleDbLoader Loader Load data stopped" + fileInfo.Source
};
oleDbConnectionString.ValidFileInfo.ProgressLogger?.Report(progressItem);
return timeSeries;
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
return new TimeSeries();
}));
}
public static async Task <Portfolios> GetData(ValidFileInfo validFileInfo, DateTime valueDate)
{
Func <DataTable, string, DateTime, Portfolios> consummer = PortfolioLoadingDelegate.Process;
return(await Loader.LoadData(validFileInfo, consummer, valueDate));
}
public async Task <TimeSeries> LoadData(ValidFileInfo fileInfo)
{
return(await Task.Run(() =>
{
var timeSeries = new TimeSeries {
Source = fileInfo.Source
};
var stopWatch = new Stopwatch();
stopWatch.Start();
var progressItem = new ProgressManager
{
TimeStamp = DateTime.UtcNow,
Progress = 1,
Action = "XmlLoader Load data",
Message = $"Xml Loader Load data started {fileInfo.Source}"
};
fileInfo.ProgressLogger?.Report(progressItem);
using (Stream s = File.OpenRead(fileInfo.Value.FullName))
{
var xdoc = XElement.Load(s, LoadOptions.None);
foreach (var element in xdoc.Elements())
{
//.AsParallel() hold of parallising for the moment.
//this can be improved to hold a column mappingname names.
var tsdp = new TimeSeriesDataPoint
{
Date = Utils.ConvertValueToDate(element, "date"),
Symbol = Utils.ConvertValueToString(element, "Name"),
Close = Utils.ConvertValueToDouble(element, "close"),
Source = fileInfo.Source,
LastUpdated = "AutoLoad",
UpdateDate = DateTime.UtcNow
};
timeSeries.Add(tsdp);
}
stopWatch.Stop();
timeSeries.LoadTime = stopWatch.Elapsed;
progressItem = new ProgressManager
{
TimeStamp = DateTime.UtcNow,
Progress = 1,
Action = "XmlLoader Load data",
Message = $"Xml Loader Load data stopped {fileInfo.Source}"
};
fileInfo.ProgressLogger?.Report(progressItem);
return timeSeries;
}
}));
//return await Task.Run(() =>
//{
// var deserializer = new XmlSerializer(typeof(TimeSeries));
// Stream s = File.OpenRead(fileInfo.Value.FullName);
// var xdoc = XElement.Load(s, LoadOptions.None);
// //s.Position = 0;
// //var x = new StreamReader(s);
// //var xy = deserializer.Deserialize(x);
// using (StreamReader reader = new StreamReader(fileInfo.Value.FullName))
// {
// var obj = deserializer.Deserialize(reader);
// var xmlData = (List<TimeSeriesDataPoint>)obj;
// return new TimeSeries();
// }
//});
}
public static async Task <IEnumerable <Ticker> > GetTickers(ValidFileInfo validFileInfo)
{
return(await Loader.LoadData(validFileInfo));
}