private static void Main(string[] args) { var validators = new List <IValidator>() { new HotelNameValidator(), new RatingValidator(), new UrlValidator() }; var dataFormatters = new List <IDataFormatter>() { new JsonFormatter(), new XmlFormatter() }; string currentPath = Directory.GetCurrentDirectory(); List <Hotel> _hotelModel = new List <Hotel>(); using (IDataSource dataSource = new CsvDataSource()) { foreach (var record in dataSource.GetHotelList()) { foreach (var validator in validators) { validator.Validate(record); } _hotelModel.Add(record); } } foreach (var formatter in dataFormatters) { formatter.PersistResult(_hotelModel); } Console.Write("Data Prscd"); }
public void TestRealVolumeDiff() { CsvDataSource csvDataSource = new CsvDataSource(); CsvOptions csvOptions = (CsvOptions)csvDataSource.GetDefaultOptions(); csvOptions.FileName = Path.Combine(TestDir, "depthvalues.csv"); csvOptions.GridStep = "200 m"; csvOptions.Separator = " "; DefaultTriangulation triangulation = new DefaultTriangulation(); SourceSurface surfaceTop = csvDataSource.GetSurface(csvOptions); var triangulatedSurfaceTop = triangulation.GetTriangulatedSurface(surfaceTop, triangulation.GetDefaultOptions()); SourceSurfaceMover mover = new SourceSurfaceMover(); var moverOptions = (SourceSurfaceMoverOptions)mover.GetDefaultOptions(); moverOptions.Z = "100 m"; SourceSurface surfaceBottom = mover.GetModifiedSurface(surfaceTop, moverOptions); var triangulatedSurfaceBottom = triangulation.GetTriangulatedSurface(surfaceBottom, triangulation.GetDefaultOptions()); IVolumeService service = new VolumeService(); var topVolume = service.GetVolumeUnderSurface(triangulatedSurfaceTop, 10000); var bottomVolume = service.GetVolumeUnderSurface(triangulatedSurfaceBottom, 10000); var diff = (int)(topVolume - bottomVolume); Assert.AreEqual(200 * 200 * 100 * 15 * 25, diff); }
public static void ZeroFilesShouldReturnNullReader() { var fileSystemService = new FakeFileSystemService( Enumerable.Empty <IFileInfo>(), Enumerable.Empty <string>()); var target = new CsvDataSource("test source", FakeDataConnection, "*.csv", FolderMode.MultipleFile, new Csv.Configuration(), fileSystemService); Assert.Null(target.GetData(Enumerable.Empty <string>(), null)); }
public void DataSourceLoadedFromStringHasValidFrames() { var dataSource = CsvDataSource.CreateFormStrings(Mother.UnsortedRowsCsvData); Assert.IsNotNull(dataSource.Bars); Assert.AreEqual(2, dataSource.Bars.Keys.Count); Assert.Contains(Symbols.Sber, dataSource.Bars.Keys.ToList()); Assert.Contains(Symbols.Vtbr, dataSource.Bars.Keys.ToList()); }
public void GetReviews() { var source = new CsvDataSource(new NullLogger <CsvDataSource>(), Path.Combine(TestContext.CurrentContext.TestDirectory, @"Processing\Persistency\data.csv")); var reviews = source.GetReviews().ToArray(); Assert.AreEqual(4, reviews.Length); Assert.AreEqual("561697961110167552", reviews[0].Data.Result.Id); Assert.AreEqual(1, reviews[0].Data.Result.Stars); Assert.AreEqual("putting apple's record-breaking quarter into context: $aapl URL_URL via @forbestech", reviews[0].Data.Result.Text); }
public static IList<AddressPartReduction> LoadFromCSV(string path) { var list = new List<AddressPartReduction>(); var dataSource = new CsvDataSource(AddressLoader.Log); using (var sr = new StreamReader(path)) { var reader = dataSource.GetReader(sr); IEnumerable<AddressPartReduction> records = reader.GetRecords<AddressPartReduction>(); list.AddRange(records); } return list; }
public void DataSourceLoadedFromDailyScvIsValid() { var dataSource = CsvDataSource.CreateFormStrings(Mother.DailyCsvData); Assert.IsNotNull(dataSource.Bars); Assert.AreEqual(1, dataSource.Bars.Keys.Count); Assert.Contains(Symbols.Sber, dataSource.Bars.Keys.ToList()); var sberBar = dataSource.Bars[Symbols.Sber]; Assert.IsTrue(sberBar.Keys.Any()); }
/// <summary> /// Initializes a new instance of the <see cref="CsvDataSourceFacts"/> class. /// </summary> public CsvDataSourceFacts() { var fakeFiles = simpleCsv.Select(f => new FakeFileInfoWrapper(f.Item1, f.Item3)); var fakeContents = simpleCsv.Select(f => f.Item2); this.fileSystemService = new FakeFileSystemService(fakeFiles, fakeContents); this.target = new CsvDataSource("test source", FakeDataConnection, "*.csv", FolderMode.MultipleFile, new Csv.Configuration(), this.fileSystemService) { Parent = Substitute.For <IDataOperationInfo>() }; }
private static void DoMainBackTest() { var eventBus = new QueuedEventBus(); var dataSource = CsvDataSource.CreateFromFiles("Data\\Min1", new[] { Symbols.Sber, Symbols.Vtbr }); var marketData = new ComposedMarketData(dataSource.Bars); var bars = new HistoricDataHandler(eventBus, marketData); var strategy = new BuyAndHoldStrategy(eventBus, bars); var executionHandler = new SimulatedExecutionHandler(eventBus, bars); var portfolio = new NaivePortfolio(eventBus, bars, 10000m); var backTest = new BackTest(eventBus, bars, strategy, portfolio, executionHandler); backTest.SimulateTrading(); }
public void TestCsvImportSeparator() { CsvDataSource csvDataSource = new CsvDataSource(); CsvOptions options = (CsvOptions)csvDataSource.GetDefaultOptions(); options.FileName = Path.Combine(TestDir, "simpleSurface_separator.csv"); options.GridStep = "100 m"; options.Separator = ";"; options.HeightMultiplier = "1 m"; SourceSurface result = csvDataSource.GetSurface(options); AssertSimpleSurface(result); }
public void TestRealTriangulation() { CsvDataSource csvDataSource = new CsvDataSource(); CsvOptions options = (CsvOptions)csvDataSource.GetDefaultOptions(); options.FileName = Path.Combine(TestDir, "depthvalues.csv"); options.GridStep = "200 ft"; options.Separator = " "; SourceSurface surface = csvDataSource.GetSurface(options); DefaultTriangulation triangulation = new DefaultTriangulation(); var result = triangulation.GetTriangulatedSurface(surface, triangulation.GetDefaultOptions()); Assert.AreEqual(15 * 25 * 2, result.Triangles.Length); // 16 x 26 points transformed to 15 x 25 rectangles, each is splitted into 2 triangles. }
public void CsvDataString() { Document doc = new Document(MyDir + "Reporting engine template - CSV data destination.docx"); CsvDataLoadOptions loadOptions = new CsvDataLoadOptions(true); loadOptions.Delimiter = ';'; loadOptions.CommentChar = '$'; CsvDataSource dataSource = new CsvDataSource(MyDir + "List of people.csv", loadOptions); BuildReport(doc, dataSource, "persons"); doc.Save(ArtifactsDir + "ReportingEngine.CsvDataString.docx"); Assert.IsTrue(DocumentHelper.CompareDocs(ArtifactsDir + "ReportingEngine.CsvDataString.docx", GoldsDir + "ReportingEngine.CsvData Gold.docx")); }
public void ComposedMarketDataLoadedFromStringsIsValid() { var dataSource = CsvDataSource.CreateFormStrings(Mother.UnsortedRowsCsvData); var mdata = new ComposedMarketData(dataSource.Bars); Assert.IsNotNull(mdata.Bars); Assert.IsTrue(mdata.RowKeys.Count > 0); Assert.IsTrue(mdata.Symbols.Count == 2); Assert.Contains(Symbols.Sber, mdata.Symbols.ToList()); Assert.Contains(Symbols.Vtbr, mdata.Symbols.ToList()); var sberBars = mdata.Bars[Symbols.Sber]; foreach (var bar in sberBars.Values) { Console.WriteLine(bar); } }
public void CsvDataStream() { Document doc = new Document(MyDir + "ReportingEngine.CsvData.docx"); CsvDataLoadOptions loadOptions = new CsvDataLoadOptions(true); loadOptions.Delimiter = ';'; loadOptions.CommentChar = '$'; using (FileStream stream = File.OpenRead(MyDir + "CsvData.csv")) { CsvDataSource dataSource = new CsvDataSource(stream, loadOptions); BuildReport(doc, dataSource, "persons"); } doc.Save(ArtifactsDir + "ReportingEngine.CsvDataStream.docx"); Assert.IsTrue(DocumentHelper.CompareDocs(ArtifactsDir + "ReportingEngine.CsvDataStream.docx", GoldsDir + "ReportingEngine.CsvData Gold.docx")); }
public void CanGetLastBarFromCsvMarketData() { var dataSource = CsvDataSource.CreateFormStrings(Mother.GenericCsvData); var marketData = new ComposedMarketData(dataSource.Bars); var eventBus = new QueuedEventBus(); var handler = new HistoricDataHandler(eventBus, marketData); Console.WriteLine("Total rows = {0}", marketData.RowKeys.Count); var i = 0; while (handler.ContinueBacktest) { handler.Update(); i++; var sber = handler.GetLast(Symbols.Sber); var vtbr = handler.GetLast(Symbols.Vtbr); Console.WriteLine($"Iteration {i}"); if (sber == null) { Console.WriteLine($"{handler.CurrentTime} => SBER is NULL"); } else { sber.Print(); } if (vtbr == null) { Console.WriteLine($"{handler.CurrentTime} => VTBR is NULL"); } else { vtbr.Print(); } Console.WriteLine("------------------"); } }
public void ComposedMarketDataLoadedFromStringsHasValidRowKeys() { var dataSource = CsvDataSource.CreateFormStrings(Mother.UnsortedRowsCsvData); var mdata = new ComposedMarketData(dataSource.Bars); var rowKeys = mdata.RowKeys; foreach (var dateTime in rowKeys) { Console.WriteLine("key={0}", dateTime); } DateTime?prev = null; foreach (var dateTime in rowKeys) { if (prev == null) { prev = dateTime; continue; } Assert.IsTrue(prev <= dateTime); } }
public int OnExecute(CommandLineApplication app) { if (!string.IsNullOrEmpty(this.Connect)) { var uri = new Uri(this.Connect); switch (uri.Scheme) { case "postgresql": string userName = ""; string password = ""; if (!string.IsNullOrEmpty(uri.UserInfo)) { var ss = uri.UserInfo.Split(":"); userName = ss[0]; password = ss.Length > 1 ? ss[1] : ""; } if (!string.IsNullOrEmpty(this.UserName)) { userName = this.UserName; } if (!string.IsNullOrEmpty(this.Password)) { password = this.Password; } var port = uri.Port > 0 ? uri.Port : 5432; var database = uri.AbsolutePath.Trim('/'); var connectString = $"Host={uri.Host};Port={port};Username={userName};Password={password};Database={database}"; var sql = ""; int index = 0; if (this.TableName.StartsWith("select", StringComparison.InvariantCultureIgnoreCase)) { sql = this.TableName; } else { if (this.Index == null || int.TryParse(this.Index, out index)) { sql = $"select * from {this.TableName}"; } else { sql = $"select {this.Index} from {this.TableName}"; } } var dataSource = new DbDataSource(Npgsql.NpgsqlFactory.Instance, connectString); var data = dataSource.GetData(sql, index); var stats = new RunningStatistics(data); Console.WriteLine($"总数:{stats.Count}"); Console.WriteLine($"最大值:{stats.Maximum}"); Console.WriteLine($"最小值:{stats.Minimum}"); Console.WriteLine($"平均值Mean::{stats.Mean}"); Console.WriteLine($"均方差StandardDeviation: {stats.StandardDeviation}"); Console.WriteLine($"{stats.ToString()}"); Console.WriteLine($"中位数: {data.Median()}"); Console.WriteLine(@"{0} - 有偏方差", data.PopulationVariance().ToString(" #0.00000;-#0.00000")); Console.WriteLine(@"{0} - 无偏方差", data.Variance().ToString(" #0.00000;-#0.00000")); Console.WriteLine(@"{0} - 标准偏差", data.StandardDeviation().ToString(" #0.00000;-#0.00000")); Console.WriteLine(@"{0} - 标准有偏偏差", data.PopulationStandardDeviation().ToString(" #0.00000;-#0.00000")); Console.WriteLine($"25% 中位数:{data.LowerQuartile()}"); Console.WriteLine($"75% 中位数:{data.UpperQuartile()}"); return(1); case "file": this.FilePath = uri.LocalPath; if (Directory.Exists(this.FilePath) && !string.IsNullOrEmpty(this.TableName)) { this.FilePath = Path.Combine(this.FilePath, this.TableName); } break; default: return(0); } } if (!string.IsNullOrEmpty(this.FilePath)) { IEnumerable <double> data; var ext = Path.GetExtension(this.FilePath); var index = 0; if (ext.Equals(".shp", StringComparison.InvariantCultureIgnoreCase)) { if (!int.TryParse(this.Index, out index)) { index = ShapeFileHelper.FindIndex(this.FilePath, this.Index); } var shpDataSource = new ShapeDataSource(this.FilePath, index); data = shpDataSource.GetData(); } else { if (!int.TryParse(this.Index, out index)) { Console.WriteLine("索引参数不对!"); return(1); } var dataSource = new CsvDataSource(this.FilePath, index); data = dataSource.GetData(); } var stats = new RunningStatistics(data); Console.WriteLine($"总数:{stats.Count}"); Console.WriteLine($"最大值:{stats.Maximum}"); Console.WriteLine($"最小值:{stats.Minimum}"); Console.WriteLine($"平均值Mean::{stats.Mean}"); Console.WriteLine($"均方差StandardDeviation: {stats.StandardDeviation}"); Console.WriteLine($"{stats.ToString()}"); Console.WriteLine($"中位数: {data.Median()}"); Console.WriteLine(@"{0} - 有偏方差", data.PopulationVariance().ToString(" #0.00000;-#0.00000")); Console.WriteLine(@"{0} - 无偏方差", data.Variance().ToString(" #0.00000;-#0.00000")); Console.WriteLine(@"{0} - 标准偏差", data.StandardDeviation().ToString(" #0.00000;-#0.00000")); Console.WriteLine(@"{0} - 标准有偏偏差", data.PopulationStandardDeviation().ToString(" #0.00000;-#0.00000")); Console.WriteLine($"25% 中位数:{data.LowerQuartile()}"); Console.WriteLine($"75% 中位数:{data.UpperQuartile()}"); //new DescriptiveStatistics() var histogram = new Histogram(data, 100, stats.Minimum, stats.Maximum); //Console.WriteLine($"{histogram.ToString()}"); for (var i = 0; i < 100; i++) { var bucket = histogram[i]; Console.WriteLine($"({bucket.LowerBound}, {bucket.UpperBound}] {bucket.Count}"); } return(1); } app.ShowHelp(); return(1); }
public static void SaveToCSV(IEnumerable<AddressPartReduction> list, string path) { // Копируем сокращения как есть из DBF в CSV if (File.Exists(path)) File.Delete(path); var dataSource = new CsvDataSource(AddressLoader.Log); using (var sw = new StreamWriter(path)) { var writer = dataSource.GetWriter(sw); writer.WriteHeader<AddressPartReduction>(); writer.WriteRecords(list); } }
public void LoadCsv(CsvDataSource data) { _data = data; _rows = _data.Rows; dataGridView1.DataSource = _rows; }
//Note that if file starts with 'classpath:' the resource is looked // up on the classpath instead. public static Configuration Load(string file) { var cfg = new Configuration(); var properties = new List <Property>(); // Get the appropriate nodes using Linq to XML XElement xml = XElement.Load(file); // Get the threshold double threshold = xml.Elements("schema").Descendants("threshold").Select(x => double.Parse(x.Value)).FirstOrDefault(); cfg.Threshold = threshold; // Get all of the properties IEnumerable <XElement> xmlProperties = from s in xml.Elements("schema") from p in s.Descendants("property") select p; foreach (XElement xElement in xmlProperties) { string propName = xElement.Descendants("name").First().Value; var property = new Property(propName); // Check to see if this is an id property XAttribute xAttribute = xElement.Attribute("type"); if (xAttribute != null) { string id = xAttribute.Value; if (id != null && id == "id") { property.IsIdProperty = true; } } else { string comparatorName = xElement.Descendants("comparator").FirstOrDefault().Value; property.Comparator = GetComparatorFromString(comparatorName); property.LowProbability = xElement.Descendants("low").Select(x => double.Parse(x.Value)).FirstOrDefault(); property.HighProbability = xElement.Descendants("high").Select(x => double.Parse(x.Value)).FirstOrDefault(); properties.Add(property); } } cfg.SetProperties(properties); //// Get the datasources //XPathNodeIterator dsi = xpn.Select("/duke/*[not(self::schema)]"); //while (dsi.MoveNext()) //{ // if (dsi.Current != null && xpi.Current.Name == "csv") // { // var datasource = GetCsvDataSourceFromXml(dsi, xpn); // } //} IEnumerable <XElement> dataSources = from d in xml.Elements() where d.Name != "schema" select d; foreach (XElement dataSource in dataSources) { if (dataSource.Name == "csv") { var csvDs = new CsvDataSource(); Hashtable csvParams = GetParametersTable(dataSource); csvDs.File = csvParams["input-file"].ToString(); if (csvParams.Contains("header-line")) { csvDs.HasHeader = (csvParams["header-line"].ToString().ToLower() == "true"); } if (csvParams.Contains("skip-lines")) { int skipLines = 0; csvDs.SkipLines = Int32.TryParse(csvParams["skip-lines"].ToString(), out skipLines) ? skipLines : 0; } csvDs.FileEncoding = csvParams.Contains("encoding") ? GetTextEncodingFromString(csvParams["encoding"].ToString()) : Encoding.Default; List <Column> cols = GetDataSourceColumns(dataSource); foreach (Column column in cols) { csvDs.AddColumn(column); } cfg.AddDataSource(0, csvDs); } } return(cfg); }
private static Dictionary<int, Dictionary<string, AddressPartReduction>> LoadReductionInternal() { var assem = typeof(AddressParserTests).Assembly; var resource = assem.GetManifestResourceStream("RUAddress.Tests.Resources.SOCRBASE.csv"); var dataSource = new CsvDataSource(Log); using (var sr = new StreamReader(resource)) { var reader = dataSource.GetReader(sr); IEnumerable<AddressPartReduction> records = reader.GetRecords<AddressPartReduction>(); var reductions = new Dictionary<int, Dictionary<string, AddressPartReduction>>(); foreach (var d in records) { if (!reductions.ContainsKey(d.Level)) { reductions.Add(d.Level, new Dictionary<string, AddressPartReduction>()); } var dict = reductions[d.Level]; var reduction = d.Short.ToLower(); dict.Add(reduction, d); } return reductions; } }