public void RunAllMetrics(CalculationRequest calculationRequest, MapReduceWrapper mapReduceWrapper)
{
foreach (var metric in _metrics)
{
var response = metric.Calculate(calculationRequest, mapReduceWrapper);
PrintResponse(response);
}
}
public CalculationResponse Calculate(CalculationRequest calculationRequest, MapReduceWrapper mapReduceWrapper)
{
var res = mapReduceWrapper.mapReduce.ToList().OrderByDescending(kvp => kvp.Value).First();
var response = new CalculationResponse
{
MetricName = this.GetType().Name,
ResultWord = res.Key,
ResultWordCount = res.Value
};
return(response);
}
public CalculationResponse Calculate(CalculationRequest calculationRequest, MapReduceWrapper mapReduceWrapper)
{
var resList = mapReduceWrapper.mapReduce.ToList().OrderByDescending(kvp => kvp.Value).Take(10);
var response = new CalculationResponse
{
MetricName = this.GetType().Name,
ResultWords = resList.Select(kvp => kvp.Key).ToList(),
ResultWordsCount = resList.Select(kvp => kvp.Value).ToList()
};
return(response);
}
public void Process()
{
Console.WriteLine($"Start processing file {CalculationRequest.SourceFileName}");
long count = 0;
var inputText = new List <string>();
var mapReduceDict = new ConcurrentDictionary <string, uint>(CalculationRequest.StringComparer);
var metricsCalculator = new MetricsCalculator(Metrics);
var mapReduceCreator = new MapReduceCreator(CalculationRequest);
using (FileStream fs = File.Open(CalculationRequest.SourceFileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
using (BufferedStream bs = new BufferedStream(fs, CalculationRequest.BufferSize))
{
using (StreamReader sr = new StreamReader(bs))
{
string line;
while ((line = sr.ReadLine()) != null)
{
inputText.Add(line);
count++;
if (count % 10000 == 0)
{
Console.WriteLine($"Read {count} lines");
}
if (count % 100000 == 0)
{
mapReduceDict = mapReduceCreator.CreateMapReduce(inputText, mapReduceDict);
inputText = new List <string>();
}
}
mapReduceDict = mapReduceCreator.CreateMapReduce(inputText, mapReduceDict);
var mapReduceWrapper = new MapReduceWrapper {
mapReduce = mapReduceDict
};
metricsCalculator.RunAllMetrics(CalculationRequest, mapReduceWrapper);
}
}
}
}
public CalculationResponse Calculate(CalculationRequest calculationRequest, MapReduceWrapper mapReduceWrapper)
{
var result = new Dictionary <char, long>();
for (char c = 'A'; c <= 'z'; c++)
{
var count = mapReduceWrapper.mapReduce.ToList().Sum(kvp => GetCharCount(kvp.Key, c) * kvp.Value);
result.Add(c, count);
}
var resKVP = result.OrderByDescending(x => x.Value).First();
var response = new CalculationResponse()
{
MetricName = this.GetType().Name,
ResultChar = resKVP.Key,
ResultCharCount = resKVP.Value
};
return(response);
}
