public void MergeFrom(TextStatistician another)
{
foreach (var pair in another.Statistic.KeyFrequency)
{
Statistic.Add(pair.Key, pair.Value);
}
}
static void AnalyzeFiles(TextStatistician statistician, IEnumerable <string> filePaths)
{
foreach (var filePath in filePaths)
{
DoAndPrintTime(() => statistician.AnalyzeFile(filePath),
$"Analyze: {filePath}");
}
}
public static void RemoveLowFrequency(string statPath)
{
DoAndPrintTime(() =>
{
var statistician = new TextStatistician();
statistician.Load(statPath);
statistician.RemoveLowFrequency((int)(statistician.Total * MinRate));
statistician.Save(statPath);
}, $"Remove low frequency ({MinRate}): {statPath}");
}
public static void MergeFiles(IEnumerable <string> filePaths, string outFilePath)
{
var stat = new TextStatistician();
foreach (var filePath in filePaths)
{
DoAndPrintTime(() => stat.MergeFrom(new TextStatistician(filePath)),
$"Merge: {filePath}");
}
DoAndPrintTime(() => stat.RemoveLowFrequency((int)(stat.Total * MinRate)), $"Remove low frequency ({MinRate})");
DoAndPrintTime(() => stat.Save(outFilePath), $"Save to {outFilePath}");
}
public static void AnalyzeFiles(IEnumerable <string> filePaths, string statPath, bool append = false)
{
var statistician = new TextStatistician();
if (append)
{
statistician.Load(statPath);
}
AnalyzeFiles(statistician, filePaths);
statistician.RemoveLowFrequency((int)(statistician.Total * MinRate));
DoAndPrintTime(() => statistician.Save(statPath), $"Writing to file: {statPath}");
}
public static void AnalyzeFilesSeparately(IEnumerable <string> filePaths, string outputDir = null)
{
foreach (var filePath in filePaths)
{
var fileInfo = new FileInfo(filePath);
var outFilePath = $"{outputDir ?? fileInfo.DirectoryName}/{fileInfo.Name}_stat.csv";
var statistician = new TextStatistician();
DoAndPrintTime(() =>
{
statistician.AnalyzeFile(filePath);
statistician.RemoveLowFrequency((int)(statistician.Total * MinRate));
statistician.Save(outFilePath);
}, $"Analyze: {filePath}");
}
}
public override void FromStatistician(TextStatistician stat)
{
base.FromStatistician(stat);
var statByPre = new Dictionary <string, Statistic <char> >();
foreach (var pair in stat.StringFrequency)
{
string str = pair.Key;
string pre = str.Substring(0, str.Length - 1);
char c = str[str.Length - 1];
int freq = pair.Value;
statByPre.GetOrAddDefault(pre).Add(c, freq);
}
dict = statByPre.ToDictionary(pair => pair.Key, pair => pair.Value.ToDistribute());
}
public override void FromStatistician(TextStatistician stat)
{
base.FromStatistician(stat);
var stat0 = new Statistic <char>();
foreach (var pair in stat.StringFrequency)
{
if (pair.Key.Length != 1)
{
continue;
}
char c = pair.Key[0];
int freq = pair.Value;
stat0.Add(c, freq);
}
dtb = stat0.ToDistribute();
}
public override void FromStatistician(TextStatistician stat)
{
base.FromStatistician(stat);
var statByChar = new Dictionary <string, Statistic <char> >();
foreach (var pair in stat.StringFrequency)
{
if (pair.Key.Length != N)
{
continue;
}
string pre = pair.Key.Substring(0, N - 1);
char c2 = pair.Key[N - 1];
int freq = pair.Value;
statByChar.GetOrAddDefault(pre).Add(c2, freq);
}
dict = statByChar.ToDictionary(pair => pair.Key, pair => pair.Value.ToDistribute());
}
public override void FromStatistician(TextStatistician stat)
{
base.FromStatistician(stat);
var statByChar = new Dictionary <char, Statistic <char> >();
foreach (var pair in stat.StringFrequency)
{
if (pair.Key.Length != 2)
{
continue;
}
char c0 = pair.Key[0];
char c1 = pair.Key[1];
int freq = pair.Value;
statByChar.GetOrAddDefault(c0).Add(c1, freq);
}
dict = statByChar.ToDictionary(pair => pair.Key, pair => pair.Value.ToDistribute());
}
public override void FromStatistician(TextStatistician stat)
{
throw new InvalidOperationException();
}
// Abstract Method
public virtual void FromStatistician(TextStatistician stat)
{
SourceName = stat.SourceName;
}
