示例#1
0
        public void LoadSecurities(string fileName, string source)
        {
            var fileLocation = string.Concat(_filePath, fileName);

            _validFileInfo = new ValidFileInfo(fileLocation, source, FileTypes.Csv);


            var valueDate = DateTime.Today.Date;

            Func <DataTable, string, DateTime, Portfolios> consummer = PortfolioLoadingDelegate.Process;

            Task <Portfolios> result = _dataLoader.LoadData(_validFileInfo, consummer, valueDate);

            result.Wait();

            var list = result.Result.ToList();

            Assert.Greater(list.Count, 0);

            list.ForEach(p =>
            {
                Assert.IsNotEmpty(p.Tenor);
                Assert.Greater(p.PortfolioId, 0);
            });
        }
        public static ValidFileInfo GetValidationInfo(LoaderConfig loaderConfig, IProgress <ProgressManager> progress)
        {
            var validFileInfo = new ValidFileInfo(loaderConfig.FileLocation, loaderConfig.Source, FileTypes.Txt);

            if (progress != null)
            {
                validFileInfo.ProgressLogger = progress;
            }


            return(validFileInfo);
        }
        public static async Task <TimeSeries> GetTimeSeries(ValidFileInfo validFileInfo)
        {
            if (!Loaders.ContainsKey(validFileInfo.FileType))
            {
                throw new ArgumentException("Json file type has not be setup in the loaders configuration layer.");
            }


            var loader = Loaders[validFileInfo.FileType];

            return(await loader.LoadData(validFileInfo));
        }
        public async Task <T> LoadData <T>(ValidFileInfo fileInfo, Func <DataTable, string, DateTime, T> dataConsummer, DateTime valueDate)
        {
            //return await Task.Run(() =>
            //{
            try
            {
                var oleDbConnectionString = new OleDbConnectionString(fileInfo);

                var progressItem = new ProgressManager
                {
                    TimeStamp = DateTime.UtcNow,
                    Progress  = 1,
                    Action    = "OleDbLoader Load data: ",
                    Message   = "OleDbLoader Loader Load data started:" + fileInfo.Source
                };

                oleDbConnectionString.ValidFileInfo.ProgressLogger?.Report(progressItem);

                var oledbConn = new System.Data.OleDb.OleDbConnection(oleDbConnectionString.Value);
                var command   = new System.Data.OleDb.OleDbDataAdapter(oleDbConnectionString.Sql, oledbConn);
                command.TableMappings.Add("Table", "LoadTable");
                var dtSet = new DataSet();
                command.Fill(dtSet);
                var table = dtSet.Tables[0];

                var portfolios = dataConsummer.Invoke(table, fileInfo.Source, valueDate);

                oledbConn.Close();

                progressItem = new ProgressManager
                {
                    TimeStamp = DateTime.UtcNow,
                    Progress  = 1,
                    Action    = "OleDbLoader Load data",
                    Message   = "OleDbLoader Loader Load data stopped" + fileInfo.Source
                };

                oleDbConnectionString.ValidFileInfo.ProgressLogger?.Report(progressItem);

                return(portfolios);
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
            }



            return(default(T));

            //});
        }
        public void LoadSecurities(string fileName, string source)
        {
            var fileLocation = string.Concat(_filePath, fileName);

            _validFileInfo = new ValidFileInfo(fileLocation, source, FileTypes.Xml);
            Task <TimeSeries> result = _dataLoader.LoadData(_validFileInfo);

            result.Wait();

            var list = result.Result.ToList();

            Assert.Greater(list.Count, 0);
        }
        public OleDbConnectionString(ValidFileInfo fileInfo)
        {
            if (!fileInfo.FileType.Equals(FileTypes.Excel) && !fileInfo.FileType.Equals(FileTypes.Csv) && !fileInfo.FileType.Equals(FileTypes.Txt))
            {
                throw new ArgumentException("File format not setup for oledb loading");
            }


            ValidFileInfo = fileInfo;
            TableName     = $"[{fileInfo.Value.Name}$]";
            Sql           = $"select * from {TableName}";

            if (fileInfo.FileType == FileTypes.Txt)
            {
                TableName = $"[{fileInfo.Value.Name}$]";
                Sql       = $"select * from [{fileInfo.Value.Name}]";

                var provider = $"provider = Microsoft.Jet.OLEDB.4.0; Data Source = '{fileInfo.Value.Directory}\\';";
                //var provider = $"provider = Microsoft.ACE.OLEDB.12.0; Data Source = '{fileInfo.Value.Directory}\\';";
                Value = string.Concat(provider, "Extended Properties = 'text;HDR=Yes;FMT=Delimited(,)'; ");
                return;
            }


            if (fileInfo.FileType == FileTypes.Excel)
            {
                //Value =
                //	$"provider = Microsoft.Jet.OLEDB.4.0; Data Source = '{fileInfo.Value.FullName}'; Extended Properties = Excel 8.0;";

                Value =
                    $"provider = Microsoft.ACE.OLEDB.12.0; Data Source = '{fileInfo.Value.FullName}'; Extended Properties = Excel 12.0;";


                return;
            }


            if (fileInfo.FileType == FileTypes.Csv)
            {
                //Value =
                //	$"Provider=Microsoft.Jet.OLEDB.4.0;Data Source='{fileInfo.Value.FullName}';Extended Properties=Text;HDR=Yes;FMT=Delimited;";
                TableName = $"[{fileInfo.Value.Name}$]";
                Sql       = $"select * from [{fileInfo.Value.Name}]";

                //var provider = $"provider = Microsoft.Jet.OLEDB.4.0; Data Source = '{fileInfo.Value.Directory}\\';";
                var provider = $"provider = Microsoft.ACE.OLEDB.12.0; Data Source = '{fileInfo.Value.Directory}\\';";
                Value = string.Concat(provider, "Extended Properties = 'text;HDR=Yes;FMT=Delimited(,)'; ");
            }
        }
        public async Task <IEnumerable <T> > LoadData(ValidFileInfo fileInfo)
        {
            var stopWatch = new Stopwatch();

            stopWatch.Start();


            var progressItem = new ProgressManager
            {
                TimeStamp = DateTime.UtcNow,
                Progress  = 1,
                Action    = "JsonLoader Load data",
                Message   = "Json Loader Load data started"
            };

            fileInfo.ProgressLogger?.Report(progressItem);

            using (StreamReader r = new StreamReader(fileInfo.Value.FullName))
            {
                var json = await r.ReadToEndAsync();

                var items = JsonConvert.DeserializeObject <IEnumerable <T> >(json);

                var results = items.OfType <IAudit>().Select(x =>
                {
                    x.Source      = fileInfo.Source;
                    x.LastUpdated = ConfigDataHandler.SystemName;
                    x.UpdateDate  = DateTime.UtcNow;

                    return(x);
                });


                progressItem = new ProgressManager
                {
                    TimeStamp = DateTime.UtcNow,
                    Progress  = 1,
                    Action    = "JsonLoader Load data",
                    Message   = "Json Loader Load data started"
                };

                fileInfo.ProgressLogger?.Report(progressItem);

                return(results.OfType <T>());
            }
        }
        public void ValidateFileInfo()
        {
            var valid = new ValidFileInfo(path, "Source", FileTypes.Json);

            Assert.IsNotNull(valid.Value);
        }
        public async Task <TimeSeries> LoadData(ValidFileInfo fileInfo)
        {
            return(await Task.Run(() =>
            {
                try
                {
                    var oleDbConnectionString = new OleDbConnectionString(fileInfo);

                    var progressItem = new ProgressManager
                    {
                        TimeStamp = DateTime.UtcNow,
                        Progress = 1,
                        Action = "OleDbLoader Load data: ",
                        Message = "OleDbLoader Loader Load data started:" + fileInfo.Source
                    };

                    oleDbConnectionString.ValidFileInfo.ProgressLogger?.Report(progressItem);

                    var stopWatch = new Stopwatch();
                    stopWatch.Start();

                    var timeSeries = new TimeSeries {
                        Source = oleDbConnectionString.ValidFileInfo.Source
                    };



                    //;HDR=yes
                    //;Format=xlsx;IMEX=1

                    var oledbConn = new System.Data.OleDb.OleDbConnection(oleDbConnectionString.Value);

                    var command = new System.Data.OleDb.OleDbDataAdapter(oleDbConnectionString.Sql, oledbConn);

                    command.TableMappings.Add("Table", "LoadTable");
                    var dtSet = new DataSet();
                    command.Fill(dtSet);
                    var table = dtSet.Tables[0];



                    var dateCol = table.Columns["date"];
                    var closeCol = table.Columns["close"];
                    var nameCol = table.Columns["Name"];

                    foreach (DataRow tableRow in table.Rows)
                    {
                        var date = Utils.SafeCastToDate(tableRow[dateCol].ToString());
                        var close = Utils.SafeCastToDoubleParse(tableRow[closeCol].ToString());

                        var tsdp = new TimeSeriesDataPoint
                        {
                            Date = Utils.SafeCastToDate(date.ToString()),
                            Close = Utils.SafeCastToDoubleParse(close.ToString()),
                            Symbol = tableRow[nameCol].ToString(),
                            Source = oleDbConnectionString.ValidFileInfo.Source,
                            LastUpdated = ConfigDataHandler.SystemName,
                            UpdateDate = DateTime.UtcNow
                        };
                        timeSeries.Add(tsdp);
                    }

                    oledbConn.Close();


                    stopWatch.Stop();
                    timeSeries.LoadTime = stopWatch.Elapsed;

                    progressItem = new ProgressManager
                    {
                        TimeStamp = DateTime.UtcNow,
                        Progress = 1,
                        Action = "OleDbLoader Load data",
                        Message = "OleDbLoader Loader Load data stopped" + fileInfo.Source
                    };

                    oleDbConnectionString.ValidFileInfo.ProgressLogger?.Report(progressItem);

                    return timeSeries;
                }
                catch (Exception ex)
                {
                    Console.WriteLine(ex.Message);
                }



                return new TimeSeries();
            }));
        }
        public static async Task <Portfolios> GetData(ValidFileInfo validFileInfo, DateTime valueDate)
        {
            Func <DataTable, string, DateTime, Portfolios> consummer = PortfolioLoadingDelegate.Process;

            return(await Loader.LoadData(validFileInfo, consummer, valueDate));
        }
示例#11
0
        public async Task <TimeSeries> LoadData(ValidFileInfo fileInfo)
        {
            return(await Task.Run(() =>
            {
                var timeSeries = new TimeSeries {
                    Source = fileInfo.Source
                };
                var stopWatch = new Stopwatch();
                stopWatch.Start();

                var progressItem = new ProgressManager
                {
                    TimeStamp = DateTime.UtcNow,
                    Progress = 1,
                    Action = "XmlLoader Load data",
                    Message = $"Xml Loader Load data started {fileInfo.Source}"
                };

                fileInfo.ProgressLogger?.Report(progressItem);


                using (Stream s = File.OpenRead(fileInfo.Value.FullName))
                {
                    var xdoc = XElement.Load(s, LoadOptions.None);
                    foreach (var element in xdoc.Elements())
                    {
                        //.AsParallel() hold of parallising for the moment.
                        //this can be improved to hold a column mappingname names.
                        var tsdp = new TimeSeriesDataPoint
                        {
                            Date = Utils.ConvertValueToDate(element, "date"),
                            Symbol = Utils.ConvertValueToString(element, "Name"),
                            Close = Utils.ConvertValueToDouble(element, "close"),
                            Source = fileInfo.Source,
                            LastUpdated = "AutoLoad",
                            UpdateDate = DateTime.UtcNow
                        };
                        timeSeries.Add(tsdp);
                    }

                    stopWatch.Stop();
                    timeSeries.LoadTime = stopWatch.Elapsed;

                    progressItem = new ProgressManager
                    {
                        TimeStamp = DateTime.UtcNow,
                        Progress = 1,
                        Action = "XmlLoader Load data",
                        Message = $"Xml Loader Load data stopped {fileInfo.Source}"
                    };

                    fileInfo.ProgressLogger?.Report(progressItem);

                    return timeSeries;
                }
            }));


            //return await Task.Run(() =>
            //{


            //	var deserializer = new XmlSerializer(typeof(TimeSeries));

            //	Stream s = File.OpenRead(fileInfo.Value.FullName);
            //	var xdoc = XElement.Load(s, LoadOptions.None);

            //	//s.Position = 0;
            //	//var x = new StreamReader(s);
            //	//var xy = deserializer.Deserialize(x);

            //	using (StreamReader reader = new StreamReader(fileInfo.Value.FullName))
            //	{
            //		var obj = deserializer.Deserialize(reader);
            //		var xmlData = (List<TimeSeriesDataPoint>)obj;
            //		return new TimeSeries();
            //	}



            //});
        }
 public static async Task <IEnumerable <Ticker> > GetTickers(ValidFileInfo validFileInfo)
 {
     return(await Loader.LoadData(validFileInfo));
 }