public static List <TokenObj> Process(List <TokenObj> inTokenList, int ndMaxSplitNo, Dictionary <string, string> infExpMap, bool debugFlag)
{
DebugPrint.PrintProcess("1. NonDictionary", debugFlag);
DebugPrint.PrintInText(TextObj.TokenListToText(inTokenList), debugFlag);
// process on each tokenObj
List <TokenObj> outTokenList = new List <TokenObj>(inTokenList
.Select(token => XmlHtmlHandler.Process(token, debugFlag))
.Select(token => EndingPuncSplitter.Process(token, ndMaxSplitNo, debugFlag))
.SelectMany(token => TextObj.FlatTokenToArrayList(token))
.Select(token => LeadingPuncSplitter.Process(token, ndMaxSplitNo, debugFlag))
.SelectMany(token => TextObj.FlatTokenToArrayList(token))
.Select(token => LeadingDigitSplitter.Process(token, debugFlag))
.SelectMany(token => TextObj.FlatTokenToArrayList(token))
.Select(token => EndingDigitSplitter.Process(token, debugFlag))
.SelectMany(token => TextObj.FlatTokenToArrayList(token))
.Select(token => InformalExpHandler.Process(token, infExpMap, debugFlag)).ToList());
/*
* List<TokenObj> outTokenList2 = inTokenList
* .Select(token => XmlHtmlHandler.Process(token, debugFlag))
* .Select(token => EndingPuncSplitter.Process(token, ndMaxSplitNo, debugFlag)).ToList();
* foreach(var item in outTokenList2) {
* Console.WriteLine(item.GetTokenStr());
* }
* List<TokenObj> outTokenList3 = inTokenList
* .Select(token => XmlHtmlHandler.Process(token, debugFlag))
* .Select(token => EndingPuncSplitter.Process(token, ndMaxSplitNo, debugFlag))
* .SelectMany(token => TextObj.FlatTokenToArrayList(token)).ToList();
* foreach(var item in outTokenList3) {
* Console.WriteLine(item.GetTokenStr());
* }
*
*
* List<TokenObj> outTokenList = inTokenList
* .Select(token => XmlHtmlHandler.Process(token, debugFlag))
* .Select(token => EndingPuncSplitter.Process(token, ndMaxSplitNo, debugFlag))
* .Select(token => LeadingPuncSplitter.Process(token, ndMaxSplitNo, debugFlag))
* .Select(token => LeadingDigitSplitter.Process(token, debugFlag))
* .Select(token => EndingDigitSplitter.Process(token, debugFlag))
* .Select(token => InformalExpHandler.Process(token, infExpMap, debugFlag)).ToList();
*/
return(outTokenList);
}
private static void TestProcess(Dictionary <string, string> informalExpMap)
{
// init
Console.WriteLine("----- Test Process Text: -----");
string inText = "u rolling &amp; pls(12years).";
// test process: must use ArrayList<TextObj>
List <TokenObj> inTokenList = TextObj.TextToTokenList(inText);
List <TokenObj> outTokenList = new List <TokenObj>(inTokenList.Select(token => XmlHtmlHandler.Process(token))
.Select(token => LeadingPuncSplitter.Process(token)).SelectMany(token => TextObj.FlatTokenToArrayList(token))
.Select(token => LeadingDigitSplitter.Process(token)).SelectMany(token => TextObj.FlatTokenToArrayList(token))
.Select(token => InformalExpHandler.Process(token, informalExpMap)).ToList());
// result
string outText = TextObj.TokenListToText(outTokenList);
// print out
Console.WriteLine("--------- LeadingDigitSplitter( ) Test ----------");
Console.WriteLine("In: [" + inText + "]");
Console.WriteLine("Out: [" + outText + "]");
Console.WriteLine("----- Details -----------");
int index = 0;
foreach (TokenObj tokenObj in outTokenList)
{
Console.WriteLine(index + "|" + tokenObj.ToHistString());
index++;
}
}
// the element is TokenObj
private static void TestProcess()
{
// init
Console.WriteLine("----- Test Process Text: -----");
string inText = "Xml: head rolling &amp; rock, (5'8").";
// test process: must use ArrayList<TextObj>
List <TokenObj> inTokenList = TextObj.TextToTokenList(inText);
List <TokenObj> outTokenList = new List <TokenObj>(inTokenList.Select(token => XmlHtmlHandler.Process(token)).ToList());
// result
string outText = TextObj.TokenListToText(outTokenList);
// print out
Console.WriteLine("--------- XmlHtmlHandler( ) Test -----------");
Console.WriteLine("In: [" + inText + "]");
Console.WriteLine("Out: [" + outText + "]");
Console.WriteLine("----- Details -----------");
int index = 0;
foreach (TokenObj tokenObj in outTokenList)
{
Console.WriteLine(index + "|" + tokenObj.ToHistString());
index++;
}
}