public void RemoveSpecificStock(Company company)
{
string connectionStr = $"server=(localdb)\\MSSQLLocalDB;Initial Catalog={nameof(RemoveSpecificStock)};Integrated Security=True;";
using (var unitOfWork = new StockEfUnitOfWork(new StockEfTestContext(connectionStr)))
{
Assert.Equal(0, unitOfWork.Stocks.Repository.Count());
new CompanyBulkInserter(connectionStr).BulkInsert(company);
Assert.Equal(1, unitOfWork.Stocks.Repository.Count());
unitOfWork.Stocks.Repository.Remove(company);
unitOfWork.Complete();
Assert.Equal(0, unitOfWork.Stocks.Repository.Count());
}
using (var connection = new SqlConnection(connectionStr))
using (var command = new SqlCommand("Select count(*) from [Companies]", connection))
{
connection.Open();
var result = (int)command.ExecuteScalar();
Assert.Equal(0, result);
}
}
static TrainTestData <BasicMLDataSet> GetBasicMlDataSet(string connectionStr, bool recreateDb, ILogger logger, DirectoryInfo inputDirectory, int ommitStocksSmallerThan, int ommitDeadStocksDate, Random rnProvider, decimal ratioTrainingSet)
{
var context = new StockEfContext(connectionStr);
var unitOfWork = new StockEfUnitOfWork(context);
var stocksDeserialized = default(List <Company>);
var watch = Stopwatch.StartNew();
if (context.DbExists() && !recreateDb)
{
stocksDeserialized = unitOfWork.Stocks.GetAll().ToList();
logger.LogInfo(
$@"Found {stocksDeserialized.Count} companies in Db in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
}
else
{
if (context.DbExists())
{
context.DropDbIfExists();
logger.LogInfo($@"Dropped Db in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
}
context.CreateDbIfNotExists();
logger.LogInfo($@"Created Db in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
var directoryService = new IoService();
var stocksRaw = directoryService.ReadDirectory(inputDirectory);
logger.LogInfo($@"Read {stocksRaw.Count} in {watch.ElapsedMilliseconds.AsTime()} from {inputDirectory.Name}");
watch.Restart();
stocksDeserialized = new StocksBulkDeserializer(new StocksDeserializer(new StockQuoteCsvClassMap())).Deserialize(stocksRaw);
logger.LogInfo($@"Deserialized {stocksDeserialized.Count} in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
var bulkInserter = new CompanyBulkInserter(connectionStr);
bulkInserter.BulkInsert(stocksDeserialized);
logger.LogInfo($@"Saved {stocksDeserialized.Count} to {connectionStr} in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
}
var normalizer = new StockQuotesToNormalizedMatrix();
var allStocksNormalized = new List <BasicMLDataSet>();
var matrixConverter = new MatrixToMlData();
var ommitedDueToLength = 0;
var ommitedDueToInvalidity = 0;
foreach (var stock in stocksDeserialized)
{
if (stock.Quotes.Count < ommitStocksSmallerThan)
{
++ommitedDueToLength;
}
else if (stock.Quotes.Max(s => s.Date) < ommitDeadStocksDate)
{
++ommitedDueToInvalidity;
}
else
{
allStocksNormalized.Add(matrixConverter.ConvertToHighPred(normalizer.Convert(stock.Quotes.ToList())));
}
}
logger.LogInfo(
$@"Loaded, converted and normalized {allStocksNormalized.Count} ({allStocksNormalized.Sum(s => s.Count)} samples) in {watch.ElapsedMilliseconds.AsTime()}. Ommited {
stocksDeserialized.Count - allStocksNormalized.Count
}.{(ommitedDueToLength > 0 || ommitedDueToInvalidity > 0 ? " Reason:" : string.Empty)}{
(ommitedDueToLength > 0 ? $" {ommitedDueToLength} too small" : string.Empty)}{(ommitedDueToLength > 0 && ommitedDueToInvalidity > 0 ? "," : string.Empty)}{
(ommitedDueToInvalidity > 0 ? $" {ommitedDueToInvalidity} invalid" : string.Empty)}");
watch.Restart();
var trainDataSet = new BasicMLDataSet();
var testDataSet = new BasicMLDataSet();
var i = 0;
for (; i < allStocksNormalized.Count * ratioTrainingSet; ++i)
{
foreach (var mlDataPair in allStocksNormalized[i].Data)
{
trainDataSet.Add(mlDataPair);
}
}
for (; i < allStocksNormalized.Count; ++i)
{
foreach (var mlDataPair in allStocksNormalized[i].Data)
{
testDataSet.Add(mlDataPair);
}
}
logger.LogInfo($@"Constructed training and test datasets with {trainDataSet.Count} and {testDataSet.Count} samples in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
trainDataSet.Data.Shuffle(rnProvider);
logger.LogInfo($@"Finished shuffling trainDataSet ({trainDataSet.Count} samples) in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
testDataSet.Data.Shuffle(rnProvider);
logger.LogInfo($@"Finished shuffling testDataSet ({testDataSet.Count} samples) in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
return(new TrainTestData <BasicMLDataSet> {
TrainingSet = trainDataSet, TestSet = testDataSet
});
}
