protected override DictionaryEntry Parse(string line)
{
string[] fields = DictionaryEntryLineParser.ParseLine(line);
DictionaryEntry entry = new DictionaryEntry(fields);
return(entry);
}
public void TestQuoteEscape()
{
String input = "日本経済新聞,1292,1292,4980,名詞,固有名詞,組織,*,*,\"1,0\",日本経済新聞,ニホンケイザイシンブン,ニホンケイザイシンブン";
String expected = "\"日本経済新聞,1292,1292,4980,名詞,固有名詞,組織,*,*,\"\"1,0\"\",日本経済新聞,ニホンケイザイシンブン,ニホンケイザイシンブン\"";
Assert.AreEqual(expected, DictionaryEntryLineParser.Escape(input));
}
public void TestEscapeRoundTrip()
{
string original = "3,\"14";
Assert.AreEqual("\"3,\"\"14\"", DictionaryEntryLineParser.Escape(original));
Assert.AreEqual(original,
DictionaryEntryLineParser.Unescape(DictionaryEntryLineParser.Escape(original))
);
}
public void testUnmatchedQuote()
{
try
{
DictionaryEntryLineParser.ParseLine("this is an entry with \"\"\"unmatched quote");
Assert.Fail();
}
catch { }
}
public void TestTrivial()
{
Assert.IsTrue(NLPJDictTest.TestUtils.IsArrayEqual(
new string[] {
"日本経済新聞", "日本 経済 新聞", "ニホン ケイザイ シンブン", "カスタム名詞"
},
DictionaryEntryLineParser.ParseLine("日本経済新聞,日本 経済 新聞,ニホン ケイザイ シンブン,カスタム名詞")
));
}
public void TestUnescape()
{
Assert.AreEqual("A", DictionaryEntryLineParser.Unescape("\"A\""));
Assert.AreEqual("\"A\"", DictionaryEntryLineParser.Unescape("\"\"\"A\"\"\""));
Assert.AreEqual("\"", DictionaryEntryLineParser.Unescape("\"\"\"\""));
Assert.AreEqual("\"\"", DictionaryEntryLineParser.Unescape("\"\"\"\"\"\""));
Assert.AreEqual("\"\"\"", DictionaryEntryLineParser.Unescape("\"\"\"\"\"\"\"\""));
Assert.AreEqual("\"\"\"\"\"", DictionaryEntryLineParser.Unescape("\"\"\"\"\"\"\"\"\"\"\"\""));
}
public string GetAllFeatures(int wordId)
{
string[] features = GetAllFeaturesArray(wordId);
for (int i = 0; i < features.Length; i++)
{
String feature = features[i];
features[i] = DictionaryEntryLineParser.Escape(feature);
}
return(String.Join(FEATURE_SEPARATOR, features));
}
public void TestEmptyQuotedQuotes()
{
Assert.IsTrue(NLPJDictTest.TestUtils.IsArrayEqual(
new String[] {
"\"",
"\"",
"quote",
"punctuation"
},
DictionaryEntryLineParser.ParseLine(
"\"\"\"\",\"\"\"\",quote,punctuation"
)
));
}
public void TtestQuotedQuotes()
{
Assert.IsTrue(NLPJDictTest.TestUtils.IsArrayEqual(
new String[] {
"Java \"Platform\"",
"Java \"Platform\"",
"Java \"Platform\"",
"カスタム名詞"
},
DictionaryEntryLineParser.ParseLine(
"\"Java \"\"Platform\"\"\",\"Java \"\"Platform\"\"\",\"Java \"\"Platform\"\"\",カスタム名詞"
)
));
}
public void TestTab()
{
Assert.IsTrue(NLPJDictTest.TestUtils.IsArrayEqual(
new String[] {
"A\tB",
"A B",
"A B",
"tab"
},
DictionaryEntryLineParser.ParseLine(
"A\tB,A B,A B,tab"
)
));
}
public void TestFrancoisWhiteBuffaloBota()
{
Assert.IsTrue(NLPJDictTest.TestUtils.IsArrayEqual(
new String[] {
"フランソワ\"ザホワイトバッファロー\"ボタ",
"フランソワ\"ザホワイトバッファロー\"ボタ",
"フランソワ\"ザホワイトバッファロー\"ボタ",
"名詞"
},
DictionaryEntryLineParser.ParseLine(
"\"フランソワ\"\"ザホワイトバッファロー\"\"ボタ\",\"フランソワ\"\"ザホワイトバッファロー\"\"ボタ\",\"フランソワ\"\"ザホワイトバッファロー\"\"ボタ\",名詞"
)
));
}
public void TestCSharp()
{
Assert.IsTrue(NLPJDictTest.TestUtils.IsArrayEqual(
new String[] {
"C#",
"C #",
"シーシャープ",
"プログラミング言語"
},
DictionaryEntryLineParser.ParseLine(
"\"C#\",\"C #\",シーシャープ,プログラミング言語"
)
));
}
public void AddEntry(String entry)
{
string[] values = DictionaryEntryLineParser.ParseLine(entry);
if (values.Length == SIMPLE_USERDICT_FIELDS)
{
AddSimpleEntry(values);
}
else if (values.Length == totalFeatures + 4)
{ // 4 = surface, left id, right id, word cost
AddFullEntry(values);
}
else
{
throw new Exception("Illegal user dictionary entry " + entry);
}
}
private string extractMultipleFeatures(int wordId, int[] fields)
{
if (fields.Length == 0)
{
return(GetAllFeatures(wordId));
}
if (fields.Length == 1)
{
return(ExtractSingleFeature(wordId, fields[0]));
}
String[] allFeatures = GetAllFeaturesArray(wordId);
String[] features = new String[fields.Length];
for (int i = 0; i < fields.Length; i++)
{
int featureNumber = fields[i];
features[i] = DictionaryEntryLineParser.Escape(
allFeatures[featureNumber]
);
}
return(String.Join(FEATURE_SEPARATOR, features));
}
protected override DictionaryEntry Parse(string line)
{
var fields = DictionaryEntryLineParser.ParseLine(line);
return(new DictionaryEntry(fields));
}
protected override DictionaryEntry Parse(string line)
{
return(new DictionaryEntry(DictionaryEntryLineParser.ParseLine(line)));
}
private string Given(string input)
{
return(DictionaryEntryLineParser.ParseLine(input).Array2String());
}
