public void Execute(IAlgorithmData data)
{
DataTEST dt = (DataTEST)data;
foreach (var item in dt.tokens)
{
dt.ngram.TryAdd(item, 0);
}
foreach (var item in dt.tokens)
{
dt.ngram[item] = dt.ngram[item] + 1;
}
dt.tokens = dt.tokens.Distinct().ToList <string>();
foreach (var item in dt.tokens)
{
dt.ngram[item] /= dt.ngram.Count();
}
Console.Write("Dictionary: ");
foreach (var item in dt.tokens)
{
if (dt.ngram[item].ToString().Length > 6)
{
Console.Write("|" + item + " " + dt.ngram[item].ToString().Substring(0, 6));
}
else
{
Console.Write("|" + item + " " + dt.ngram[item].ToString());
}
}
Console.WriteLine();
Console.Write("Dictionary size: " + dt.ngram.Count());
Console.WriteLine();
}
public void Execute(IAlgorithmData data)
{
DataTEST dt = (DataTEST)data;
Console.WriteLine("Original: " + dt.data);
dt.normalized = dt.data.ToLower();
dt.normalized = dt.normalized.Replace(" ", "");
dt.normalized = dt.normalized.Replace(".", "");
dt.normalized = dt.normalized.Replace(",", "");
Console.WriteLine("Normalized: " + dt.normalized);
}
public void Execute(IAlgorithmData data)
{
DataTEST dt = (DataTEST)data;
for (int i = 0; i < dt.normalized.Length - 1; i++)
{
dt.tokens.Add(dt.normalized[i].ToString() + dt.normalized[i + 1].ToString());
}
Console.Write("Tokens: ");
foreach (var item in dt.tokens)
{
Console.Write(item + " ");
}
Console.WriteLine();
}
public double Distance(IAlgorithmData data1, IAlgorithmData data2)
{
double temp = 0;
DataTEST dt1 = (DataTEST)data1;
DataTEST dt2 = (DataTEST)data2;
foreach (var item in dt1.ngram)
{
if (dt2.ngram.ContainsKey(item.Key))
{
temp += Math.Abs(item.Value - dt2.ngram[item.Key]);
}
else
{
temp += item.Value;
}
}
Console.WriteLine("Distance from " + dt1.name + " to " + dt2.name + " = " + temp);
return(temp);
}
static void Main(string[] args)
{
DataTEST data = new DataTEST();
data.data = "Шла Саша по шоссе и сосала сушку.";
data.name = "First";
NormalTEST nt = new NormalTEST();
TokenTEST tt = new TokenTEST();
AlgTEST at = new AlgTEST();
FullAlgorithm fa = new FullAlgorithm();
fa.EnqueueStep(nt);
fa.EnqueueStep(tt);
fa.EnqueueStep(at);
fa.Execute(data);
Console.WriteLine("*** *** ***");
DataTEST data2 = new DataTEST();
//data2.data = "Карл у Клары украл кораллы, а Клара у Карла украла кларнет.";
//data2.data = "Шла Саша по шоссе, а Карл у Клары украл Кларнет.";
data2.data = "Шла Саша по шоссе и сосала сушку. Она ещё не знала, что Карл у Клары украл кораллы, а Клара у Карла украла кларнет.";
data2.name = "Second";
NormalTEST nt2 = new NormalTEST();
TokenTEST tt2 = new TokenTEST();
AlgTEST at2 = new AlgTEST();
FullAlgorithm fa2 = new FullAlgorithm();
fa2.EnqueueStep(nt2);
fa2.EnqueueStep(tt2);
fa2.EnqueueStep(at2);
fa2.Execute(data2);
Console.WriteLine("*** *** ***");
DistTEST dt = new DistTEST();
dt.Distance(data, data2);
}