public void DeserializeGivenValidParametersDeserializes()
{
// Arrange
var input = @"06MAGNA,20210219,2.4400,2.5800,2.3200,2.5200,81891
11BIT,20210219,565.0000,568.0000,550.0000,560.0000,5269
3RGAMES,20210219,1.0000,1.0200,1.0000,1.0200,11933";
var output = new List <StockQuote>
{
new StockQuote {
Ticker = "test1"
},
new StockQuote {
Ticker = "test2"
},
new StockQuote {
Ticker = "test3"
},
};
var deserializerMock = Substitute.For <IStocksDeserializer>();
deserializerMock.DeserializeQuotes(Arg.Any <string>()).Returns(output);
var tested = new StocksBulkDeserializer(deserializerMock);
// Act
var actual = tested.Deserialize(input);
// Assert
Assert.True(actual.TrueForAll(c => c.Ticker.StartsWith("test")));
Assert.Equal(3, actual.Count);
}
public void BulkDeserializer_WhenCreated_HasCorrectFields()
{
var deserializerMock = Substitute.For <IStocksDeserializer>();
var tested = new StocksBulkDeserializer(deserializerMock);
// Assert
Assert.Equal(deserializerMock, tested.Deserializer);
}
public async Task DeserializeAsync_GivenValidParameters_Deserializes()
{
const string ticer = "test123";
var stubCompany = new Company {
Ticker = ticer
};
var stubDict = new Dictionary <string, string>
{
{ "test1", "test" },
{ "test2", "test" },
{ "test3", "test" },
{ "test4", "test" }
};
var deserializerMock = Substitute.For <IStocksDeserializer>();
deserializerMock.DeserializeAsync(Arg.Any <string>()).Returns(stubCompany);
var tested = new StocksBulkDeserializer(deserializerMock);
// Act
var actual = await tested.DeserializeAsync(stubDict);
// Assert
Assert.True(actual.TrueForAll(c => c.Ticker == ticer));
Assert.Equal(stubDict.Count, actual.Count);
}
static TrainTestData <BasicMLDataSet> GetBasicMlDataSet(string connectionStr, bool recreateDb, ILogger logger, DirectoryInfo inputDirectory, int ommitStocksSmallerThan, int ommitDeadStocksDate, Random rnProvider, decimal ratioTrainingSet)
{
var context = new StockEfContext(connectionStr);
var unitOfWork = new StockEfUnitOfWork(context);
var stocksDeserialized = default(List <Company>);
var watch = Stopwatch.StartNew();
if (context.DbExists() && !recreateDb)
{
stocksDeserialized = unitOfWork.Stocks.GetAll().ToList();
logger.LogInfo(
$@"Found {stocksDeserialized.Count} companies in Db in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
}
else
{
if (context.DbExists())
{
context.DropDbIfExists();
logger.LogInfo($@"Dropped Db in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
}
context.CreateDbIfNotExists();
logger.LogInfo($@"Created Db in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
var directoryService = new IoService();
var stocksRaw = directoryService.ReadDirectory(inputDirectory);
logger.LogInfo($@"Read {stocksRaw.Count} in {watch.ElapsedMilliseconds.AsTime()} from {inputDirectory.Name}");
watch.Restart();
stocksDeserialized = new StocksBulkDeserializer(new StocksDeserializer(new StockQuoteCsvClassMap())).Deserialize(stocksRaw);
logger.LogInfo($@"Deserialized {stocksDeserialized.Count} in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
var bulkInserter = new CompanyBulkInserter(connectionStr);
bulkInserter.BulkInsert(stocksDeserialized);
logger.LogInfo($@"Saved {stocksDeserialized.Count} to {connectionStr} in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
}
var normalizer = new StockQuotesToNormalizedMatrix();
var allStocksNormalized = new List <BasicMLDataSet>();
var matrixConverter = new MatrixToMlData();
var ommitedDueToLength = 0;
var ommitedDueToInvalidity = 0;
foreach (var stock in stocksDeserialized)
{
if (stock.Quotes.Count < ommitStocksSmallerThan)
{
++ommitedDueToLength;
}
else if (stock.Quotes.Max(s => s.Date) < ommitDeadStocksDate)
{
++ommitedDueToInvalidity;
}
else
{
allStocksNormalized.Add(matrixConverter.ConvertToHighPred(normalizer.Convert(stock.Quotes.ToList())));
}
}
logger.LogInfo(
$@"Loaded, converted and normalized {allStocksNormalized.Count} ({allStocksNormalized.Sum(s => s.Count)} samples) in {watch.ElapsedMilliseconds.AsTime()}. Ommited {
stocksDeserialized.Count - allStocksNormalized.Count
}.{(ommitedDueToLength > 0 || ommitedDueToInvalidity > 0 ? " Reason:" : string.Empty)}{
(ommitedDueToLength > 0 ? $" {ommitedDueToLength} too small" : string.Empty)}{(ommitedDueToLength > 0 && ommitedDueToInvalidity > 0 ? "," : string.Empty)}{
(ommitedDueToInvalidity > 0 ? $" {ommitedDueToInvalidity} invalid" : string.Empty)}");
watch.Restart();
var trainDataSet = new BasicMLDataSet();
var testDataSet = new BasicMLDataSet();
var i = 0;
for (; i < allStocksNormalized.Count * ratioTrainingSet; ++i)
{
foreach (var mlDataPair in allStocksNormalized[i].Data)
{
trainDataSet.Add(mlDataPair);
}
}
for (; i < allStocksNormalized.Count; ++i)
{
foreach (var mlDataPair in allStocksNormalized[i].Data)
{
testDataSet.Add(mlDataPair);
}
}
logger.LogInfo($@"Constructed training and test datasets with {trainDataSet.Count} and {testDataSet.Count} samples in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
trainDataSet.Data.Shuffle(rnProvider);
logger.LogInfo($@"Finished shuffling trainDataSet ({trainDataSet.Count} samples) in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
testDataSet.Data.Shuffle(rnProvider);
logger.LogInfo($@"Finished shuffling testDataSet ({testDataSet.Count} samples) in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
return(new TrainTestData <BasicMLDataSet> {
TrainingSet = trainDataSet, TestSet = testDataSet
});
}
private static async Task Main(string[] args)
{
var project = new ProjectSettings();
var logger = new AggregateLogger(
new ConsoleLogger {
InfoColor = ConsoleColor.Gray, Formatter = (level, message) => $"{message}{Environment.NewLine}"
},
new FileLoggerBase(Path.Combine(project.WorkingDirectory.FullName, project.LogFileName)));
var quotesDownloader = new StockQuotesDownloadService(new Downloader(), logger);
Dictionary <string, string> unzippedStocks = null;
logger.LogInfo($"Hello in {project.Name}. This test will read a directory and load it into Dictionary of deserialized stock quotes.");
logger.LogInfo($"Would you like to download latest stocks from {project.QuotesDownloadUrl} ? (y/n)");
var response = GetBinaryDecisionFromUser();
byte[] rawBytes = null;
if (response)
{
await quotesDownloader.Download(project);
}
else
{
logger.LogInfo("Would you like to read archive from disk? (y/n)");
var readArchive = GetBinaryDecisionFromUser();
FileInfo inputFile = null;
var useExistingFile = false;
if (readArchive)
{
if (File.Exists(Path.Combine(project.WorkingDirectory.FullName, project.ArchiveFileName)))
{
logger.LogInfo($"Would you like to use existing archive from {project.WorkingDirectory.FullName}? (y/n)");
useExistingFile = GetBinaryDecisionFromUser();
if (useExistingFile)
{
inputFile = new FileInfo(Path.Combine(project.WorkingDirectory.FullName, project.ArchiveFileName));
}
}
if (!useExistingFile)
{
logger.LogInfo("Point to the archive:");
var userInput = GetPathToExistingFileFromUser(".zip");
inputFile = new FileInfo(userInput);
}
var fileReader = new FileService();
var taskToReadFile = fileReader.ReadAllBytesAsync(inputFile.FullName);
var elapsed = ShowSpinnerUntilTaskIsRunning(taskToReadFile);
rawBytes = await taskToReadFile;
logger.LogInfo($"Read {rawBytes.Length} bytes in {elapsed.AsTime()}");
}
else
{
logger.LogInfo("Would you like to read unzipped files from disk? (y/n)");
var readFiles = GetBinaryDecisionFromUser();
if (readFiles)
{
var useFilesFromDefaultDir = false;
if (project.UnzippedFilesDirectory.Exists)
{
var filesFound = Directory.GetFiles(project.UnzippedFilesDirectory.FullName).Select(f => new FileInfo(f))
.Count(f => f.Extension.EndsWith(project.QuotesFileExtension));
if (filesFound > 0)
{
logger.LogInfo($"Would you like to use {filesFound} files from {project.WorkingDirectory.FullName}? (y/n)");
useFilesFromDefaultDir = GetBinaryDecisionFromUser();
//if (useFilesFromDefaultDir)
//{
// unzippedFilesDirectory = new DirectoryInfo(Path.Combine(workingDirectory.FullName, unzippedFilesDirectoryName));
//}
}
}
if (!useFilesFromDefaultDir)
{
logger.LogInfo("Not supported. Bye!");
}
var directoryStocksReader = new DirectoryService(new FileService());
var taskToRead = directoryStocksReader.ReadTopDirectoryAsync(project.UnzippedFilesDirectory.FullName, $"*.{project.QuotesFileExtension}");
var elapsed = ShowSpinnerUntilTaskIsRunning(taskToRead);
unzippedStocks = await taskToRead;
logger.LogInfo($"Read {unzippedStocks.Count} stocks in {elapsed.AsTime()}");
}
}
}
if (rawBytes?.Length > 0)
{
Console.WriteLine();
var unzipper = new Unzipper();
var taskToUnzip = unzipper.UnzipAsync(rawBytes);
var elapsed = ShowSpinnerUntilTaskIsRunning(taskToUnzip);
unzippedStocks = await taskToUnzip;
logger.LogInfo($"Unzipped {unzippedStocks.Count} stocks in {elapsed.AsTime()}");
}
if (unzippedStocks?.Count > 0)
{
logger.LogInfo($"Deserializing {unzippedStocks.Count} stocks:");
var bullkDeserializer = new StocksBulkDeserializer(new StocksDeserializer(new StockQuoteCsvClassMap()));
var taskToDeserialize = bullkDeserializer.DeserializeAsync(unzippedStocks);
var elapsed = ShowSpinnerUntilTaskIsRunning(taskToDeserialize);
var deserialized = await taskToDeserialize;
logger.LogInfo($"Deserialzed {unzippedStocks.Count} stocks in {elapsed.AsTime()}");
logger.LogInfo($"Would you like to print some stock quotes? (y/n)");
var decision = GetBinaryDecisionFromUser();
if (decision)
{
logger.LogInfo("Please enter the TICKER for the stock of your choice:");
var line = GetNonEmptyStringFromUser();
Company found = null;
while ((found = deserialized.FirstOrDefault(c => c.Ticker.Equals(line, StringComparison.InvariantCultureIgnoreCase))) == null)
{
logger.LogInfo($"{line} not found in the collection.");
line = GetNonEmptyStringFromUser();
}
found.Quotes.ForEach(q => Console.Out.WriteLine($"{q} Open: {q.Open} High: {q.High} Low: {q.Low} Close: {q.Close}"));
}
logger.LogInfo($"Would you like save files to {project.UnzippedFilesDirectory.FullName}? (y/n)");
var saveFiles = GetBinaryDecisionFromUser();
if (saveFiles)
{
if (!project.UnzippedFilesDirectory.Exists)
{
project.UnzippedFilesDirectory.Create();
}
var tasksToSave = new List <Task>();
foreach (var stock in unzippedStocks)
{
tasksToSave.Add(File.WriteAllTextAsync(Path.Combine(project.UnzippedFilesDirectory.FullName, stock.Key), stock.Value));
}
var taskForSave = Task.WhenAll(tasksToSave);
var elapsedWhenSaving = ShowSpinnerUntilTaskIsRunning(taskForSave);
logger.LogInfo($"Saved {unzippedStocks.Count} stocks to {project.UnzippedFilesDirectory.FullName} in {elapsed.AsTime()}");
}
}
Console.WriteLine("press any key to exit...");
Console.ReadKey();
}