/// <summary>
/// Generate Language Profile from Text File
/// <pre>
/// usage: --genprofile-text -l [language code] [text file path]
/// </pre>
/// </summary>
private void generateProfileFromText()
{
if (arglist.Count != 1)
{
Console.Error.WriteLine("Need to specify text file path");
return;
}
string file = arglist[0];
if (!File.Exists(file))
{
Console.Error.WriteLine("Need to specify existing text file path");
return;
}
string lang = Get("lang");
if (lang == null)
{
Console.Error.WriteLine("Need to specify langage code(-l)");
return;
}
FileStream os = null;
try
{
LangProfile profile = GenProfile.LoadFromText(lang, file);
profile.OmitLessFreq();
string profile_path = lang;
File.WriteAllText(profile_path, JsonSerializer.Serialize(profile));
}
catch (NotSupportedException e)
{
Debug.WriteLine(e);
}
catch (IOException e)
{
Debug.WriteLine(e);
}
catch (LangDetectException e)
{
Debug.WriteLine(e);
}
finally
{
try
{
if (os != null)
{
os.Close();
}
}
catch (IOException e) { }
}
}
public void testAdd()
{
LangProfile profile = new LangProfile("en");
profile.Add("a");
Assert.AreEqual((int)profile.Freq["a"], 1);
profile.Add("a");
Assert.AreEqual((int)profile.Freq["a"], 2);
profile.OmitLessFreq();
}
/// <summary>
/// Generate Language Profile from Wikipedia Abstract Database File
/// <pre>
/// usage: --genprofile -d [abstracts directory] [language names]
/// </pre>
/// </summary>
public void GenerateProfile()
{
string directory = Get("directory");
foreach (string lang in arglist)
{
string file = SearchFile(directory, lang + "wiki-.*-abstract\\.xml.*");
if (file == null)
{
Console.Error.WriteLine("Not Found abstract xml : lang = " + lang);
continue;
}
FileStream os = null;
try
{
LangProfile profile = GenProfile.LoadFromWikipediaAbstract(lang, file);
profile.OmitLessFreq();
string profile_path = Get("directory") + "/profiles/" + lang;
File.WriteAllText(profile_path, JsonSerializer.Serialize(profile));
}
catch (NotSupportedException e)
{
Debug.WriteLine(e);
}
catch (IOException e)
{
Debug.WriteLine(e);
}
catch (LangDetectException e)
{
Debug.WriteLine(e);
}
finally
{
try
{
if (os != null)
{
os.Close();
}
}
catch (IOException e) { }
}
}
}
public void testOmitLessFreq()
{
LangProfile profile = new LangProfile("en");
string[] grams = "a b c \u3042 \u3044 \u3046 \u3048 \u304a \u304b \u304c \u304d \u304e \u304f".Split(" ");
for (int i = 0; i < 5; ++i)
{
foreach (string g in grams)
{
profile.Add(g);
}
}
profile.Add("\u3050");
Assert.AreEqual((int)profile.Freq["a"], 5);
Assert.AreEqual((int)profile.Freq["\u3042"], 5);
Assert.AreEqual((int)profile.Freq["\u3050"], 1);
profile.OmitLessFreq();
Assert.AreEqual(profile.Freq["a"], null); // omitted
Assert.AreEqual((int)profile.Freq["\u3042"], 5);
Assert.AreEqual(profile.Freq["\u3050"], null); // omitted
}
public void testOmitLessFreqIllegally()
{
LangProfile profile = new LangProfile();
profile.OmitLessFreq(); // ignore
}