private static void Main(string[] args)
        {
            var validators = new List <IValidator>()
            {
                new HotelNameValidator(), new RatingValidator(), new UrlValidator()
            };
            var dataFormatters = new List <IDataFormatter>()
            {
                new JsonFormatter(), new XmlFormatter()
            };

            string currentPath = Directory.GetCurrentDirectory();

            List <Hotel> _hotelModel = new List <Hotel>();

            using (IDataSource dataSource = new CsvDataSource())
            {
                foreach (var record in dataSource.GetHotelList())
                {
                    foreach (var validator in validators)
                    {
                        validator.Validate(record);
                    }

                    _hotelModel.Add(record);
                }
            }

            foreach (var formatter in dataFormatters)
            {
                formatter.PersistResult(_hotelModel);
            }

            Console.Write("Data Prscd");
        }
Beispiel #2
0
        public void TestRealVolumeDiff()
        {
            CsvDataSource csvDataSource = new CsvDataSource();
            CsvOptions    csvOptions    = (CsvOptions)csvDataSource.GetDefaultOptions();

            csvOptions.FileName  = Path.Combine(TestDir, "depthvalues.csv");
            csvOptions.GridStep  = "200 m";
            csvOptions.Separator = " ";

            DefaultTriangulation triangulation = new DefaultTriangulation();

            SourceSurface surfaceTop             = csvDataSource.GetSurface(csvOptions);
            var           triangulatedSurfaceTop = triangulation.GetTriangulatedSurface(surfaceTop, triangulation.GetDefaultOptions());

            SourceSurfaceMover mover = new SourceSurfaceMover();
            var moverOptions         = (SourceSurfaceMoverOptions)mover.GetDefaultOptions();

            moverOptions.Z = "100 m";
            SourceSurface surfaceBottom             = mover.GetModifiedSurface(surfaceTop, moverOptions);
            var           triangulatedSurfaceBottom = triangulation.GetTriangulatedSurface(surfaceBottom, triangulation.GetDefaultOptions());

            IVolumeService service = new VolumeService();

            var topVolume    = service.GetVolumeUnderSurface(triangulatedSurfaceTop, 10000);
            var bottomVolume = service.GetVolumeUnderSurface(triangulatedSurfaceBottom, 10000);

            var diff = (int)(topVolume - bottomVolume);

            Assert.AreEqual(200 * 200 * 100 * 15 * 25, diff);
        }
Beispiel #3
0
        public static void ZeroFilesShouldReturnNullReader()
        {
            var fileSystemService = new FakeFileSystemService(
                Enumerable.Empty <IFileInfo>(), Enumerable.Empty <string>());

            var target = new CsvDataSource("test source", FakeDataConnection, "*.csv", FolderMode.MultipleFile, new Csv.Configuration(), fileSystemService);

            Assert.Null(target.GetData(Enumerable.Empty <string>(), null));
        }
Beispiel #4
0
        public void DataSourceLoadedFromStringHasValidFrames()
        {
            var dataSource = CsvDataSource.CreateFormStrings(Mother.UnsortedRowsCsvData);

            Assert.IsNotNull(dataSource.Bars);
            Assert.AreEqual(2, dataSource.Bars.Keys.Count);
            Assert.Contains(Symbols.Sber, dataSource.Bars.Keys.ToList());
            Assert.Contains(Symbols.Vtbr, dataSource.Bars.Keys.ToList());
        }
Beispiel #5
0
        public void GetReviews()
        {
            var source  = new CsvDataSource(new NullLogger <CsvDataSource>(), Path.Combine(TestContext.CurrentContext.TestDirectory, @"Processing\Persistency\data.csv"));
            var reviews = source.GetReviews().ToArray();

            Assert.AreEqual(4, reviews.Length);
            Assert.AreEqual("561697961110167552", reviews[0].Data.Result.Id);
            Assert.AreEqual(1, reviews[0].Data.Result.Stars);
            Assert.AreEqual("putting apple's record-breaking quarter into context: $aapl URL_URL via @forbestech", reviews[0].Data.Result.Text);
        }
Beispiel #6
0
 public static IList<AddressPartReduction> LoadFromCSV(string path)
 {
     var list = new List<AddressPartReduction>();
     var dataSource = new CsvDataSource(AddressLoader.Log);
     using (var sr = new StreamReader(path))
     {
         var reader = dataSource.GetReader(sr);
         IEnumerable<AddressPartReduction> records = reader.GetRecords<AddressPartReduction>();
         list.AddRange(records);
     }
     return list;
 }
Beispiel #7
0
        public void DataSourceLoadedFromDailyScvIsValid()
        {
            var dataSource = CsvDataSource.CreateFormStrings(Mother.DailyCsvData);

            Assert.IsNotNull(dataSource.Bars);
            Assert.AreEqual(1, dataSource.Bars.Keys.Count);
            Assert.Contains(Symbols.Sber, dataSource.Bars.Keys.ToList());

            var sberBar = dataSource.Bars[Symbols.Sber];

            Assert.IsTrue(sberBar.Keys.Any());
        }
Beispiel #8
0
        /// <summary>
        /// Initializes a new instance of the <see cref="CsvDataSourceFacts"/> class.
        /// </summary>
        public CsvDataSourceFacts()
        {
            var fakeFiles    = simpleCsv.Select(f => new FakeFileInfoWrapper(f.Item1, f.Item3));
            var fakeContents = simpleCsv.Select(f => f.Item2);

            this.fileSystemService = new FakeFileSystemService(fakeFiles, fakeContents);

            this.target = new CsvDataSource("test source", FakeDataConnection, "*.csv", FolderMode.MultipleFile, new Csv.Configuration(), this.fileSystemService)
            {
                Parent = Substitute.For <IDataOperationInfo>()
            };
        }
Beispiel #9
0
        private static void DoMainBackTest()
        {
            var eventBus         = new QueuedEventBus();
            var dataSource       = CsvDataSource.CreateFromFiles("Data\\Min1", new[] { Symbols.Sber, Symbols.Vtbr });
            var marketData       = new ComposedMarketData(dataSource.Bars);
            var bars             = new HistoricDataHandler(eventBus, marketData);
            var strategy         = new BuyAndHoldStrategy(eventBus, bars);
            var executionHandler = new SimulatedExecutionHandler(eventBus, bars);
            var portfolio        = new NaivePortfolio(eventBus, bars, 10000m);
            var backTest         = new BackTest(eventBus, bars, strategy, portfolio, executionHandler);

            backTest.SimulateTrading();
        }
Beispiel #10
0
        public void TestCsvImportSeparator()
        {
            CsvDataSource csvDataSource = new CsvDataSource();
            CsvOptions    options       = (CsvOptions)csvDataSource.GetDefaultOptions();

            options.FileName         = Path.Combine(TestDir, "simpleSurface_separator.csv");
            options.GridStep         = "100 m";
            options.Separator        = ";";
            options.HeightMultiplier = "1 m";

            SourceSurface result = csvDataSource.GetSurface(options);

            AssertSimpleSurface(result);
        }
Beispiel #11
0
        public void TestRealTriangulation()
        {
            CsvDataSource csvDataSource = new CsvDataSource();
            CsvOptions    options       = (CsvOptions)csvDataSource.GetDefaultOptions();

            options.FileName  = Path.Combine(TestDir, "depthvalues.csv");
            options.GridStep  = "200 ft";
            options.Separator = " ";

            SourceSurface surface = csvDataSource.GetSurface(options);

            DefaultTriangulation triangulation = new DefaultTriangulation();

            var result = triangulation.GetTriangulatedSurface(surface, triangulation.GetDefaultOptions());

            Assert.AreEqual(15 * 25 * 2, result.Triangles.Length); // 16 x 26 points transformed to 15 x 25 rectangles, each is splitted into 2 triangles.
        }
Beispiel #12
0
        public void CsvDataString()
        {
            Document doc = new Document(MyDir + "Reporting engine template - CSV data destination.docx");

            CsvDataLoadOptions loadOptions = new CsvDataLoadOptions(true);

            loadOptions.Delimiter   = ';';
            loadOptions.CommentChar = '$';

            CsvDataSource dataSource = new CsvDataSource(MyDir + "List of people.csv", loadOptions);

            BuildReport(doc, dataSource, "persons");

            doc.Save(ArtifactsDir + "ReportingEngine.CsvDataString.docx");

            Assert.IsTrue(DocumentHelper.CompareDocs(ArtifactsDir + "ReportingEngine.CsvDataString.docx",
                                                     GoldsDir + "ReportingEngine.CsvData Gold.docx"));
        }
        public void ComposedMarketDataLoadedFromStringsIsValid()
        {
            var dataSource = CsvDataSource.CreateFormStrings(Mother.UnsortedRowsCsvData);
            var mdata      = new ComposedMarketData(dataSource.Bars);

            Assert.IsNotNull(mdata.Bars);
            Assert.IsTrue(mdata.RowKeys.Count > 0);
            Assert.IsTrue(mdata.Symbols.Count == 2);
            Assert.Contains(Symbols.Sber, mdata.Symbols.ToList());
            Assert.Contains(Symbols.Vtbr, mdata.Symbols.ToList());

            var sberBars = mdata.Bars[Symbols.Sber];

            foreach (var bar in sberBars.Values)
            {
                Console.WriteLine(bar);
            }
        }
        public void CsvDataStream()
        {
            Document doc = new Document(MyDir + "ReportingEngine.CsvData.docx");

            CsvDataLoadOptions loadOptions = new CsvDataLoadOptions(true);

            loadOptions.Delimiter   = ';';
            loadOptions.CommentChar = '$';

            using (FileStream stream = File.OpenRead(MyDir + "CsvData.csv"))
            {
                CsvDataSource dataSource = new CsvDataSource(stream, loadOptions);
                BuildReport(doc, dataSource, "persons");
            }

            doc.Save(ArtifactsDir + "ReportingEngine.CsvDataStream.docx");

            Assert.IsTrue(DocumentHelper.CompareDocs(ArtifactsDir + "ReportingEngine.CsvDataStream.docx",
                                                     GoldsDir + "ReportingEngine.CsvData Gold.docx"));
        }
        public void CanGetLastBarFromCsvMarketData()
        {
            var dataSource = CsvDataSource.CreateFormStrings(Mother.GenericCsvData);
            var marketData = new ComposedMarketData(dataSource.Bars);
            var eventBus   = new QueuedEventBus();

            var handler = new HistoricDataHandler(eventBus, marketData);

            Console.WriteLine("Total rows = {0}", marketData.RowKeys.Count);

            var i = 0;

            while (handler.ContinueBacktest)
            {
                handler.Update();
                i++;

                var sber = handler.GetLast(Symbols.Sber);
                var vtbr = handler.GetLast(Symbols.Vtbr);

                Console.WriteLine($"Iteration {i}");

                if (sber == null)
                {
                    Console.WriteLine($"{handler.CurrentTime} => SBER is NULL");
                }
                else
                {
                    sber.Print();
                }
                if (vtbr == null)
                {
                    Console.WriteLine($"{handler.CurrentTime} => VTBR is NULL");
                }
                else
                {
                    vtbr.Print();
                }
                Console.WriteLine("------------------");
            }
        }
        public void ComposedMarketDataLoadedFromStringsHasValidRowKeys()
        {
            var dataSource = CsvDataSource.CreateFormStrings(Mother.UnsortedRowsCsvData);
            var mdata      = new ComposedMarketData(dataSource.Bars);
            var rowKeys    = mdata.RowKeys;

            foreach (var dateTime in rowKeys)
            {
                Console.WriteLine("key={0}", dateTime);
            }

            DateTime?prev = null;

            foreach (var dateTime in rowKeys)
            {
                if (prev == null)
                {
                    prev = dateTime;
                    continue;
                }

                Assert.IsTrue(prev <= dateTime);
            }
        }
Beispiel #17
0
        public int OnExecute(CommandLineApplication app)
        {
            if (!string.IsNullOrEmpty(this.Connect))
            {
                var uri = new Uri(this.Connect);
                switch (uri.Scheme)
                {
                case "postgresql":
                    string userName = "";
                    string password = "";
                    if (!string.IsNullOrEmpty(uri.UserInfo))
                    {
                        var ss = uri.UserInfo.Split(":");
                        userName = ss[0];
                        password = ss.Length > 1 ? ss[1] : "";
                    }

                    if (!string.IsNullOrEmpty(this.UserName))
                    {
                        userName = this.UserName;
                    }
                    if (!string.IsNullOrEmpty(this.Password))
                    {
                        password = this.Password;
                    }

                    var port          = uri.Port > 0 ? uri.Port : 5432;
                    var database      = uri.AbsolutePath.Trim('/');
                    var connectString = $"Host={uri.Host};Port={port};Username={userName};Password={password};Database={database}";


                    var sql   = "";
                    int index = 0;
                    if (this.TableName.StartsWith("select", StringComparison.InvariantCultureIgnoreCase))
                    {
                        sql = this.TableName;
                    }
                    else
                    {
                        if (this.Index == null || int.TryParse(this.Index, out index))
                        {
                            sql = $"select * from {this.TableName}";
                        }
                        else
                        {
                            sql = $"select {this.Index} from {this.TableName}";
                        }
                    }


                    var dataSource = new DbDataSource(Npgsql.NpgsqlFactory.Instance, connectString);
                    var data       = dataSource.GetData(sql, index);

                    var stats = new  RunningStatistics(data);
                    Console.WriteLine($"总数:{stats.Count}");
                    Console.WriteLine($"最大值:{stats.Maximum}");
                    Console.WriteLine($"最小值:{stats.Minimum}");
                    Console.WriteLine($"平均值Mean::{stats.Mean}");
                    Console.WriteLine($"均方差StandardDeviation: {stats.StandardDeviation}");
                    Console.WriteLine($"{stats.ToString()}");
                    Console.WriteLine($"中位数: {data.Median()}");
                    Console.WriteLine(@"{0} - 有偏方差", data.PopulationVariance().ToString(" #0.00000;-#0.00000"));
                    Console.WriteLine(@"{0} - 无偏方差", data.Variance().ToString(" #0.00000;-#0.00000"));
                    Console.WriteLine(@"{0} - 标准偏差", data.StandardDeviation().ToString(" #0.00000;-#0.00000"));
                    Console.WriteLine(@"{0} - 标准有偏偏差", data.PopulationStandardDeviation().ToString(" #0.00000;-#0.00000"));
                    Console.WriteLine($"25% 中位数:{data.LowerQuartile()}");
                    Console.WriteLine($"75% 中位数:{data.UpperQuartile()}");


                    return(1);

                case "file":
                    this.FilePath = uri.LocalPath;
                    if (Directory.Exists(this.FilePath) && !string.IsNullOrEmpty(this.TableName))
                    {
                        this.FilePath = Path.Combine(this.FilePath, this.TableName);
                    }

                    break;

                default:
                    return(0);
                }
            }

            if (!string.IsNullOrEmpty(this.FilePath))
            {
                IEnumerable <double> data;
                var ext   = Path.GetExtension(this.FilePath);
                var index = 0;


                if (ext.Equals(".shp", StringComparison.InvariantCultureIgnoreCase))
                {
                    if (!int.TryParse(this.Index, out index))
                    {
                        index = ShapeFileHelper.FindIndex(this.FilePath, this.Index);
                    }
                    var shpDataSource = new ShapeDataSource(this.FilePath, index);
                    data = shpDataSource.GetData();
                }
                else
                {
                    if (!int.TryParse(this.Index, out index))
                    {
                        Console.WriteLine("索引参数不对!");
                        return(1);
                    }
                    var dataSource = new CsvDataSource(this.FilePath, index);
                    data = dataSource.GetData();
                }
                var stats = new  RunningStatistics(data);
                Console.WriteLine($"总数:{stats.Count}");
                Console.WriteLine($"最大值:{stats.Maximum}");
                Console.WriteLine($"最小值:{stats.Minimum}");
                Console.WriteLine($"平均值Mean::{stats.Mean}");
                Console.WriteLine($"均方差StandardDeviation: {stats.StandardDeviation}");
                Console.WriteLine($"{stats.ToString()}");
                Console.WriteLine($"中位数: {data.Median()}");
                Console.WriteLine(@"{0} - 有偏方差", data.PopulationVariance().ToString(" #0.00000;-#0.00000"));
                Console.WriteLine(@"{0} - 无偏方差", data.Variance().ToString(" #0.00000;-#0.00000"));
                Console.WriteLine(@"{0} - 标准偏差", data.StandardDeviation().ToString(" #0.00000;-#0.00000"));
                Console.WriteLine(@"{0} - 标准有偏偏差", data.PopulationStandardDeviation().ToString(" #0.00000;-#0.00000"));
                Console.WriteLine($"25% 中位数:{data.LowerQuartile()}");
                Console.WriteLine($"75% 中位数:{data.UpperQuartile()}");
                //new DescriptiveStatistics()

                var histogram = new Histogram(data, 100, stats.Minimum, stats.Maximum);
                //Console.WriteLine($"{histogram.ToString()}");
                for (var i = 0; i < 100; i++)
                {
                    var bucket = histogram[i];
                    Console.WriteLine($"({bucket.LowerBound}, {bucket.UpperBound}] {bucket.Count}");
                }

                return(1);
            }
            app.ShowHelp();
            return(1);
        }
Beispiel #18
0
        public static void SaveToCSV(IEnumerable<AddressPartReduction> list, string path)
        {
            // Копируем сокращения как есть из DBF в CSV
            if (File.Exists(path)) File.Delete(path);

            var dataSource = new CsvDataSource(AddressLoader.Log);
            using (var sw = new StreamWriter(path))
            {
                var writer = dataSource.GetWriter(sw);
                writer.WriteHeader<AddressPartReduction>();
                writer.WriteRecords(list);
            }
        }
 public void LoadCsv(CsvDataSource data)
 {
     _data = data;
     _rows = _data.Rows;
     dataGridView1.DataSource = _rows;
 }
Beispiel #20
0
        //Note that if file starts with 'classpath:' the resource is looked
        // up on the classpath instead.

        public static Configuration Load(string file)
        {
            var cfg        = new Configuration();
            var properties = new List <Property>();

            // Get the appropriate nodes using Linq to XML
            XElement xml = XElement.Load(file);

            // Get the threshold
            double threshold =
                xml.Elements("schema").Descendants("threshold").Select(x => double.Parse(x.Value)).FirstOrDefault();

            cfg.Threshold = threshold;

            // Get all of the properties
            IEnumerable <XElement> xmlProperties = from s in xml.Elements("schema")
                                                   from p in s.Descendants("property")
                                                   select p;

            foreach (XElement xElement in xmlProperties)
            {
                string propName = xElement.Descendants("name").First().Value;
                var    property = new Property(propName);

                // Check to see if this is an id property
                XAttribute xAttribute = xElement.Attribute("type");
                if (xAttribute != null)
                {
                    string id = xAttribute.Value;
                    if (id != null && id == "id")
                    {
                        property.IsIdProperty = true;
                    }
                }
                else
                {
                    string comparatorName = xElement.Descendants("comparator").FirstOrDefault().Value;
                    property.Comparator     = GetComparatorFromString(comparatorName);
                    property.LowProbability =
                        xElement.Descendants("low").Select(x => double.Parse(x.Value)).FirstOrDefault();
                    property.HighProbability =
                        xElement.Descendants("high").Select(x => double.Parse(x.Value)).FirstOrDefault();
                    properties.Add(property);
                }
            }

            cfg.SetProperties(properties);

            //// Get the datasources
            //XPathNodeIterator dsi = xpn.Select("/duke/*[not(self::schema)]");

            //while (dsi.MoveNext())
            //{
            //    if (dsi.Current != null && xpi.Current.Name == "csv")
            //    {
            //        var datasource = GetCsvDataSourceFromXml(dsi, xpn);
            //    }
            //}
            IEnumerable <XElement> dataSources = from d in xml.Elements()
                                                 where d.Name != "schema"
                                                 select d;

            foreach (XElement dataSource in dataSources)
            {
                if (dataSource.Name == "csv")
                {
                    var       csvDs     = new CsvDataSource();
                    Hashtable csvParams = GetParametersTable(dataSource);
                    csvDs.File = csvParams["input-file"].ToString();
                    if (csvParams.Contains("header-line"))
                    {
                        csvDs.HasHeader = (csvParams["header-line"].ToString().ToLower() == "true");
                    }

                    if (csvParams.Contains("skip-lines"))
                    {
                        int skipLines = 0;
                        csvDs.SkipLines = Int32.TryParse(csvParams["skip-lines"].ToString(), out skipLines)
                                              ? skipLines
                                              : 0;
                    }

                    csvDs.FileEncoding = csvParams.Contains("encoding")
                                             ? GetTextEncodingFromString(csvParams["encoding"].ToString())
                                             : Encoding.Default;

                    List <Column> cols = GetDataSourceColumns(dataSource);
                    foreach (Column column in cols)
                    {
                        csvDs.AddColumn(column);
                    }

                    cfg.AddDataSource(0, csvDs);
                }
            }

            return(cfg);
        }
Beispiel #21
0
        private static Dictionary<int, Dictionary<string, AddressPartReduction>> LoadReductionInternal()
        {
            var assem = typeof(AddressParserTests).Assembly;
            var resource = assem.GetManifestResourceStream("RUAddress.Tests.Resources.SOCRBASE.csv");

            var dataSource = new CsvDataSource(Log);
            using (var sr = new StreamReader(resource))
            {
                var reader = dataSource.GetReader(sr);
                IEnumerable<AddressPartReduction> records = reader.GetRecords<AddressPartReduction>();

                var reductions = new Dictionary<int, Dictionary<string, AddressPartReduction>>();
                foreach (var d in records)
                {
                    if (!reductions.ContainsKey(d.Level))
                    {
                        reductions.Add(d.Level, new Dictionary<string, AddressPartReduction>());
                    }
                    var dict = reductions[d.Level];
                    var reduction = d.Short.ToLower();
                    dict.Add(reduction, d);
                }
                return reductions;
            }
        }