private static void Main(string[] args)
{
var validators = new List <IValidator>()
{
new HotelNameValidator(), new RatingValidator(), new UrlValidator()
};
var dataFormatters = new List <IDataFormatter>()
{
new JsonFormatter(), new XmlFormatter()
};
string currentPath = Directory.GetCurrentDirectory();
List <Hotel> _hotelModel = new List <Hotel>();
using (IDataSource dataSource = new CsvDataSource())
{
foreach (var record in dataSource.GetHotelList())
{
foreach (var validator in validators)
{
validator.Validate(record);
}
_hotelModel.Add(record);
}
}
foreach (var formatter in dataFormatters)
{
formatter.PersistResult(_hotelModel);
}
Console.Write("Data Prscd");
}
public void TestRealVolumeDiff()
{
CsvDataSource csvDataSource = new CsvDataSource();
CsvOptions csvOptions = (CsvOptions)csvDataSource.GetDefaultOptions();
csvOptions.FileName = Path.Combine(TestDir, "depthvalues.csv");
csvOptions.GridStep = "200 m";
csvOptions.Separator = " ";
DefaultTriangulation triangulation = new DefaultTriangulation();
SourceSurface surfaceTop = csvDataSource.GetSurface(csvOptions);
var triangulatedSurfaceTop = triangulation.GetTriangulatedSurface(surfaceTop, triangulation.GetDefaultOptions());
SourceSurfaceMover mover = new SourceSurfaceMover();
var moverOptions = (SourceSurfaceMoverOptions)mover.GetDefaultOptions();
moverOptions.Z = "100 m";
SourceSurface surfaceBottom = mover.GetModifiedSurface(surfaceTop, moverOptions);
var triangulatedSurfaceBottom = triangulation.GetTriangulatedSurface(surfaceBottom, triangulation.GetDefaultOptions());
IVolumeService service = new VolumeService();
var topVolume = service.GetVolumeUnderSurface(triangulatedSurfaceTop, 10000);
var bottomVolume = service.GetVolumeUnderSurface(triangulatedSurfaceBottom, 10000);
var diff = (int)(topVolume - bottomVolume);
Assert.AreEqual(200 * 200 * 100 * 15 * 25, diff);
}
public static void ZeroFilesShouldReturnNullReader()
{
var fileSystemService = new FakeFileSystemService(
Enumerable.Empty <IFileInfo>(), Enumerable.Empty <string>());
var target = new CsvDataSource("test source", FakeDataConnection, "*.csv", FolderMode.MultipleFile, new Csv.Configuration(), fileSystemService);
Assert.Null(target.GetData(Enumerable.Empty <string>(), null));
}
public void DataSourceLoadedFromStringHasValidFrames()
{
var dataSource = CsvDataSource.CreateFormStrings(Mother.UnsortedRowsCsvData);
Assert.IsNotNull(dataSource.Bars);
Assert.AreEqual(2, dataSource.Bars.Keys.Count);
Assert.Contains(Symbols.Sber, dataSource.Bars.Keys.ToList());
Assert.Contains(Symbols.Vtbr, dataSource.Bars.Keys.ToList());
}
public void GetReviews()
{
var source = new CsvDataSource(new NullLogger <CsvDataSource>(), Path.Combine(TestContext.CurrentContext.TestDirectory, @"Processing\Persistency\data.csv"));
var reviews = source.GetReviews().ToArray();
Assert.AreEqual(4, reviews.Length);
Assert.AreEqual("561697961110167552", reviews[0].Data.Result.Id);
Assert.AreEqual(1, reviews[0].Data.Result.Stars);
Assert.AreEqual("putting apple's record-breaking quarter into context: $aapl URL_URL via @forbestech", reviews[0].Data.Result.Text);
}
public static IList<AddressPartReduction> LoadFromCSV(string path)
{
var list = new List<AddressPartReduction>();
var dataSource = new CsvDataSource(AddressLoader.Log);
using (var sr = new StreamReader(path))
{
var reader = dataSource.GetReader(sr);
IEnumerable<AddressPartReduction> records = reader.GetRecords<AddressPartReduction>();
list.AddRange(records);
}
return list;
}
public void DataSourceLoadedFromDailyScvIsValid()
{
var dataSource = CsvDataSource.CreateFormStrings(Mother.DailyCsvData);
Assert.IsNotNull(dataSource.Bars);
Assert.AreEqual(1, dataSource.Bars.Keys.Count);
Assert.Contains(Symbols.Sber, dataSource.Bars.Keys.ToList());
var sberBar = dataSource.Bars[Symbols.Sber];
Assert.IsTrue(sberBar.Keys.Any());
}
/// <summary>
/// Initializes a new instance of the <see cref="CsvDataSourceFacts"/> class.
/// </summary>
public CsvDataSourceFacts()
{
var fakeFiles = simpleCsv.Select(f => new FakeFileInfoWrapper(f.Item1, f.Item3));
var fakeContents = simpleCsv.Select(f => f.Item2);
this.fileSystemService = new FakeFileSystemService(fakeFiles, fakeContents);
this.target = new CsvDataSource("test source", FakeDataConnection, "*.csv", FolderMode.MultipleFile, new Csv.Configuration(), this.fileSystemService)
{
Parent = Substitute.For <IDataOperationInfo>()
};
}
private static void DoMainBackTest()
{
var eventBus = new QueuedEventBus();
var dataSource = CsvDataSource.CreateFromFiles("Data\\Min1", new[] { Symbols.Sber, Symbols.Vtbr });
var marketData = new ComposedMarketData(dataSource.Bars);
var bars = new HistoricDataHandler(eventBus, marketData);
var strategy = new BuyAndHoldStrategy(eventBus, bars);
var executionHandler = new SimulatedExecutionHandler(eventBus, bars);
var portfolio = new NaivePortfolio(eventBus, bars, 10000m);
var backTest = new BackTest(eventBus, bars, strategy, portfolio, executionHandler);
backTest.SimulateTrading();
}
public void TestCsvImportSeparator()
{
CsvDataSource csvDataSource = new CsvDataSource();
CsvOptions options = (CsvOptions)csvDataSource.GetDefaultOptions();
options.FileName = Path.Combine(TestDir, "simpleSurface_separator.csv");
options.GridStep = "100 m";
options.Separator = ";";
options.HeightMultiplier = "1 m";
SourceSurface result = csvDataSource.GetSurface(options);
AssertSimpleSurface(result);
}
public void TestRealTriangulation()
{
CsvDataSource csvDataSource = new CsvDataSource();
CsvOptions options = (CsvOptions)csvDataSource.GetDefaultOptions();
options.FileName = Path.Combine(TestDir, "depthvalues.csv");
options.GridStep = "200 ft";
options.Separator = " ";
SourceSurface surface = csvDataSource.GetSurface(options);
DefaultTriangulation triangulation = new DefaultTriangulation();
var result = triangulation.GetTriangulatedSurface(surface, triangulation.GetDefaultOptions());
Assert.AreEqual(15 * 25 * 2, result.Triangles.Length); // 16 x 26 points transformed to 15 x 25 rectangles, each is splitted into 2 triangles.
}
public void CsvDataString()
{
Document doc = new Document(MyDir + "Reporting engine template - CSV data destination.docx");
CsvDataLoadOptions loadOptions = new CsvDataLoadOptions(true);
loadOptions.Delimiter = ';';
loadOptions.CommentChar = '$';
CsvDataSource dataSource = new CsvDataSource(MyDir + "List of people.csv", loadOptions);
BuildReport(doc, dataSource, "persons");
doc.Save(ArtifactsDir + "ReportingEngine.CsvDataString.docx");
Assert.IsTrue(DocumentHelper.CompareDocs(ArtifactsDir + "ReportingEngine.CsvDataString.docx",
GoldsDir + "ReportingEngine.CsvData Gold.docx"));
}
public void ComposedMarketDataLoadedFromStringsIsValid()
{
var dataSource = CsvDataSource.CreateFormStrings(Mother.UnsortedRowsCsvData);
var mdata = new ComposedMarketData(dataSource.Bars);
Assert.IsNotNull(mdata.Bars);
Assert.IsTrue(mdata.RowKeys.Count > 0);
Assert.IsTrue(mdata.Symbols.Count == 2);
Assert.Contains(Symbols.Sber, mdata.Symbols.ToList());
Assert.Contains(Symbols.Vtbr, mdata.Symbols.ToList());
var sberBars = mdata.Bars[Symbols.Sber];
foreach (var bar in sberBars.Values)
{
Console.WriteLine(bar);
}
}
public void CsvDataStream()
{
Document doc = new Document(MyDir + "ReportingEngine.CsvData.docx");
CsvDataLoadOptions loadOptions = new CsvDataLoadOptions(true);
loadOptions.Delimiter = ';';
loadOptions.CommentChar = '$';
using (FileStream stream = File.OpenRead(MyDir + "CsvData.csv"))
{
CsvDataSource dataSource = new CsvDataSource(stream, loadOptions);
BuildReport(doc, dataSource, "persons");
}
doc.Save(ArtifactsDir + "ReportingEngine.CsvDataStream.docx");
Assert.IsTrue(DocumentHelper.CompareDocs(ArtifactsDir + "ReportingEngine.CsvDataStream.docx",
GoldsDir + "ReportingEngine.CsvData Gold.docx"));
}
public void CanGetLastBarFromCsvMarketData()
{
var dataSource = CsvDataSource.CreateFormStrings(Mother.GenericCsvData);
var marketData = new ComposedMarketData(dataSource.Bars);
var eventBus = new QueuedEventBus();
var handler = new HistoricDataHandler(eventBus, marketData);
Console.WriteLine("Total rows = {0}", marketData.RowKeys.Count);
var i = 0;
while (handler.ContinueBacktest)
{
handler.Update();
i++;
var sber = handler.GetLast(Symbols.Sber);
var vtbr = handler.GetLast(Symbols.Vtbr);
Console.WriteLine($"Iteration {i}");
if (sber == null)
{
Console.WriteLine($"{handler.CurrentTime} => SBER is NULL");
}
else
{
sber.Print();
}
if (vtbr == null)
{
Console.WriteLine($"{handler.CurrentTime} => VTBR is NULL");
}
else
{
vtbr.Print();
}
Console.WriteLine("------------------");
}
}
public void ComposedMarketDataLoadedFromStringsHasValidRowKeys()
{
var dataSource = CsvDataSource.CreateFormStrings(Mother.UnsortedRowsCsvData);
var mdata = new ComposedMarketData(dataSource.Bars);
var rowKeys = mdata.RowKeys;
foreach (var dateTime in rowKeys)
{
Console.WriteLine("key={0}", dateTime);
}
DateTime?prev = null;
foreach (var dateTime in rowKeys)
{
if (prev == null)
{
prev = dateTime;
continue;
}
Assert.IsTrue(prev <= dateTime);
}
}
public int OnExecute(CommandLineApplication app)
{
if (!string.IsNullOrEmpty(this.Connect))
{
var uri = new Uri(this.Connect);
switch (uri.Scheme)
{
case "postgresql":
string userName = "";
string password = "";
if (!string.IsNullOrEmpty(uri.UserInfo))
{
var ss = uri.UserInfo.Split(":");
userName = ss[0];
password = ss.Length > 1 ? ss[1] : "";
}
if (!string.IsNullOrEmpty(this.UserName))
{
userName = this.UserName;
}
if (!string.IsNullOrEmpty(this.Password))
{
password = this.Password;
}
var port = uri.Port > 0 ? uri.Port : 5432;
var database = uri.AbsolutePath.Trim('/');
var connectString = $"Host={uri.Host};Port={port};Username={userName};Password={password};Database={database}";
var sql = "";
int index = 0;
if (this.TableName.StartsWith("select", StringComparison.InvariantCultureIgnoreCase))
{
sql = this.TableName;
}
else
{
if (this.Index == null || int.TryParse(this.Index, out index))
{
sql = $"select * from {this.TableName}";
}
else
{
sql = $"select {this.Index} from {this.TableName}";
}
}
var dataSource = new DbDataSource(Npgsql.NpgsqlFactory.Instance, connectString);
var data = dataSource.GetData(sql, index);
var stats = new RunningStatistics(data);
Console.WriteLine($"总数：{stats.Count}");
Console.WriteLine($"最大值：{stats.Maximum}");
Console.WriteLine($"最小值：{stats.Minimum}");
Console.WriteLine($"平均值Mean：：{stats.Mean}");
Console.WriteLine($"均方差StandardDeviation： {stats.StandardDeviation}");
Console.WriteLine($"{stats.ToString()}");
Console.WriteLine($"中位数： {data.Median()}");
Console.WriteLine(@"{0} - 有偏方差", data.PopulationVariance().ToString(" #0.00000;-#0.00000"));
Console.WriteLine(@"{0} - 无偏方差", data.Variance().ToString(" #0.00000;-#0.00000"));
Console.WriteLine(@"{0} - 标准偏差", data.StandardDeviation().ToString(" #0.00000;-#0.00000"));
Console.WriteLine(@"{0} - 标准有偏偏差", data.PopulationStandardDeviation().ToString(" #0.00000;-#0.00000"));
Console.WriteLine($"25% 中位数：{data.LowerQuartile()}");
Console.WriteLine($"75% 中位数：{data.UpperQuartile()}");
return(1);
case "file":
this.FilePath = uri.LocalPath;
if (Directory.Exists(this.FilePath) && !string.IsNullOrEmpty(this.TableName))
{
this.FilePath = Path.Combine(this.FilePath, this.TableName);
}
break;
default:
return(0);
}
}
if (!string.IsNullOrEmpty(this.FilePath))
{
IEnumerable <double> data;
var ext = Path.GetExtension(this.FilePath);
var index = 0;
if (ext.Equals(".shp", StringComparison.InvariantCultureIgnoreCase))
{
if (!int.TryParse(this.Index, out index))
{
index = ShapeFileHelper.FindIndex(this.FilePath, this.Index);
}
var shpDataSource = new ShapeDataSource(this.FilePath, index);
data = shpDataSource.GetData();
}
else
{
if (!int.TryParse(this.Index, out index))
{
Console.WriteLine("索引参数不对！");
return(1);
}
var dataSource = new CsvDataSource(this.FilePath, index);
data = dataSource.GetData();
}
var stats = new RunningStatistics(data);
Console.WriteLine($"总数：{stats.Count}");
Console.WriteLine($"最大值：{stats.Maximum}");
Console.WriteLine($"最小值：{stats.Minimum}");
Console.WriteLine($"平均值Mean：：{stats.Mean}");
Console.WriteLine($"均方差StandardDeviation： {stats.StandardDeviation}");
Console.WriteLine($"{stats.ToString()}");
Console.WriteLine($"中位数： {data.Median()}");
Console.WriteLine(@"{0} - 有偏方差", data.PopulationVariance().ToString(" #0.00000;-#0.00000"));
Console.WriteLine(@"{0} - 无偏方差", data.Variance().ToString(" #0.00000;-#0.00000"));
Console.WriteLine(@"{0} - 标准偏差", data.StandardDeviation().ToString(" #0.00000;-#0.00000"));
Console.WriteLine(@"{0} - 标准有偏偏差", data.PopulationStandardDeviation().ToString(" #0.00000;-#0.00000"));
Console.WriteLine($"25% 中位数：{data.LowerQuartile()}");
Console.WriteLine($"75% 中位数：{data.UpperQuartile()}");
//new DescriptiveStatistics()
var histogram = new Histogram(data, 100, stats.Minimum, stats.Maximum);
//Console.WriteLine($"{histogram.ToString()}");
for (var i = 0; i < 100; i++)
{
var bucket = histogram[i];
Console.WriteLine($"({bucket.LowerBound}, {bucket.UpperBound}] {bucket.Count}");
}
return(1);
}
app.ShowHelp();
return(1);
}
public static void SaveToCSV(IEnumerable<AddressPartReduction> list, string path)
{
// Копируем сокращения как есть из DBF в CSV
if (File.Exists(path)) File.Delete(path);
var dataSource = new CsvDataSource(AddressLoader.Log);
using (var sw = new StreamWriter(path))
{
var writer = dataSource.GetWriter(sw);
writer.WriteHeader<AddressPartReduction>();
writer.WriteRecords(list);
}
}
public void LoadCsv(CsvDataSource data)
{
_data = data;
_rows = _data.Rows;
dataGridView1.DataSource = _rows;
}
//Note that if file starts with 'classpath:' the resource is looked
// up on the classpath instead.
public static Configuration Load(string file)
{
var cfg = new Configuration();
var properties = new List <Property>();
// Get the appropriate nodes using Linq to XML
XElement xml = XElement.Load(file);
// Get the threshold
double threshold =
xml.Elements("schema").Descendants("threshold").Select(x => double.Parse(x.Value)).FirstOrDefault();
cfg.Threshold = threshold;
// Get all of the properties
IEnumerable <XElement> xmlProperties = from s in xml.Elements("schema")
from p in s.Descendants("property")
select p;
foreach (XElement xElement in xmlProperties)
{
string propName = xElement.Descendants("name").First().Value;
var property = new Property(propName);
// Check to see if this is an id property
XAttribute xAttribute = xElement.Attribute("type");
if (xAttribute != null)
{
string id = xAttribute.Value;
if (id != null && id == "id")
{
property.IsIdProperty = true;
}
}
else
{
string comparatorName = xElement.Descendants("comparator").FirstOrDefault().Value;
property.Comparator = GetComparatorFromString(comparatorName);
property.LowProbability =
xElement.Descendants("low").Select(x => double.Parse(x.Value)).FirstOrDefault();
property.HighProbability =
xElement.Descendants("high").Select(x => double.Parse(x.Value)).FirstOrDefault();
properties.Add(property);
}
}
cfg.SetProperties(properties);
//// Get the datasources
//XPathNodeIterator dsi = xpn.Select("/duke/*[not(self::schema)]");
//while (dsi.MoveNext())
//{
// if (dsi.Current != null && xpi.Current.Name == "csv")
// {
// var datasource = GetCsvDataSourceFromXml(dsi, xpn);
// }
//}
IEnumerable <XElement> dataSources = from d in xml.Elements()
where d.Name != "schema"
select d;
foreach (XElement dataSource in dataSources)
{
if (dataSource.Name == "csv")
{
var csvDs = new CsvDataSource();
Hashtable csvParams = GetParametersTable(dataSource);
csvDs.File = csvParams["input-file"].ToString();
if (csvParams.Contains("header-line"))
{
csvDs.HasHeader = (csvParams["header-line"].ToString().ToLower() == "true");
}
if (csvParams.Contains("skip-lines"))
{
int skipLines = 0;
csvDs.SkipLines = Int32.TryParse(csvParams["skip-lines"].ToString(), out skipLines)
? skipLines
: 0;
}
csvDs.FileEncoding = csvParams.Contains("encoding")
? GetTextEncodingFromString(csvParams["encoding"].ToString())
: Encoding.Default;
List <Column> cols = GetDataSourceColumns(dataSource);
foreach (Column column in cols)
{
csvDs.AddColumn(column);
}
cfg.AddDataSource(0, csvDs);
}
}
return(cfg);
}
private static Dictionary<int, Dictionary<string, AddressPartReduction>> LoadReductionInternal()
{
var assem = typeof(AddressParserTests).Assembly;
var resource = assem.GetManifestResourceStream("RUAddress.Tests.Resources.SOCRBASE.csv");
var dataSource = new CsvDataSource(Log);
using (var sr = new StreamReader(resource))
{
var reader = dataSource.GetReader(sr);
IEnumerable<AddressPartReduction> records = reader.GetRecords<AddressPartReduction>();
var reductions = new Dictionary<int, Dictionary<string, AddressPartReduction>>();
foreach (var d in records)
{
if (!reductions.ContainsKey(d.Level))
{
reductions.Add(d.Level, new Dictionary<string, AddressPartReduction>());
}
var dict = reductions[d.Level];
var reduction = d.Short.ToLower();
dict.Add(reduction, d);
}
return reductions;
}
}
