public IEnumerable<double> ToRawValues(StockPriceType features)
{
var values = new List<double>();
if (features.HasFlag(StockPriceType.Open))
values.Add(Open);
if (features.HasFlag(StockPriceType.High))
values.Add(High);
if (features.HasFlag(StockPriceType.Low))
values.Add(Low);
if (features.HasFlag(StockPriceType.Close))
values.Add(Close);
return values;
}
public IStockAnalysisResults Analyze(IEnumerable<IStock> stocks,
StockPriceType features,
int clusters,
Dictionary<string, string> settings)
{
// Don't start analyzing if not stocks are enabled
if (!stocks.Any())
return new StockAnalysisResults
{
Duration = new TimeSpan(),
IsSuccess = false,
Error = "No stocks to analyze",
Results = null
};
// Indicate analyze process is in progress hence cannot be started
IsAnalyzing = true;
// start stopwatch timer
var stopwatch = Stopwatch.StartNew();
// begin alayzing process
bool isSuccess = true;
string error = string.Empty;
IEnumerable<IStockAnalysisResult> results = default(IEnumerable<IStockAnalysisResult>);
try
{
//Export the stocks to the input directory
Export("input", stocks, features);
// initate map reduce process
var resultsFile = LaunchMapReduce(settings, clusters);
// get clustering results
var clusteringResults = GetClusteringResults(resultsFile);
// set results
results = stocks.Where(stock => clusteringResults.ContainsKey(stock.Symbol))
.Select(stock => new StockAnalysisResult(stock)
{
Cluster = clusteringResults[stock.Symbol]
});
}
catch (Exception ex)
{
isSuccess = false;
error = ex?.InnerException?.Message ?? ex?.Message;
}
finally
{
// Indicate analyze process has finished hence can be started
IsAnalyzing = false;
// stop stopwatch
stopwatch.Stop();
}
return new StockAnalysisResults
{
Duration = stopwatch.Elapsed,
IsSuccess = isSuccess,
Error = error,
Results = results,
EmptyClusters = clusters - results?.Distinct(new ClusterComparer()).Count() ?? clusters
};
}
private void Export(string directory, IEnumerable<IStock> stocks, StockPriceType features)
{
_statusUpdater.UpdateStatus("Exporting stocks data to csv ...");
// create the directory if needed
if (!Directory.Exists(directory))
Directory.CreateDirectory(directory);
// Get the normalized data per feature, per day, per stock
var data = from stock in stocks
where stock.Tickers
.Any()
let rawFeatures = stock.Tickers
.SelectMany(t => t.ToRawValues(features))
select new
{
Symbol = stock.Symbol,
NormalizedFeatures = rawFeatures.Normalize()
};
// project an initial index
int index = 1;
// export each certain number of stocks to a different size
data.Chunk(STOCKS_PER_FILE)
.ToList()
.ForEach(sublist =>
{
var name = Path.Combine(directory, $"stocks_{index++}.csv");
using (var sw = new StreamWriter(name))
using (var writer = new CsvWriter(sw))
{
foreach (var stock in sublist)
{
writer.WriteField(stock.Symbol);
// write each value
foreach (var feature in stock.NormalizedFeatures)
{
writer.WriteField(feature);
}
//ensure you write end of record when you are using WriteField method
writer.NextRecord();
}
}
});
}