public static String DeTokenize(String[] tokens, DetokenizationDictionary.Operation operation)
{
DetokenizationDictionary.Operation[] operations = new DetokenizationDictionary.Operation[tokens.Length];
for (int i = 0; i < tokens.Length; i++)
{
operations[i] = operation;
}
DetokenizationDictionary dictionary = new DetokenizationDictionary(
tokens, operations);
DictionaryDetokenizer detokenizer = new DictionaryDetokenizer(
dictionary);
return(detokenizer.detokenize(tokens, " "));
}
public void TestDetokenizer()
{
var dict = new DetokenizationDictionary {
{ ".", DetokenizationDictionary.Operation.MoveLeft },
{ "!", DetokenizationDictionary.Operation.MoveLeft },
{ "(", DetokenizationDictionary.Operation.MoveRight },
{ ")", DetokenizationDictionary.Operation.MoveLeft },
{ "\"", DetokenizationDictionary.Operation.RightLeftMatching },
{ "-", DetokenizationDictionary.Operation.MoveBoth }
};
var detokenizer = new DictionaryDetokenizer(dict);
var detokenizeOperations = detokenizer.Detokenize(new[] { "Simple", "test", ".", "co", "-", "worker" });
Assert.AreEqual(DetokenizationOperation.NoOperation, detokenizeOperations[0]);
Assert.AreEqual(DetokenizationOperation.NoOperation, detokenizeOperations[1]);
Assert.AreEqual(DetokenizationOperation.MergeToLeft, detokenizeOperations[2]);
Assert.AreEqual(DetokenizationOperation.NoOperation, detokenizeOperations[3]);
Assert.AreEqual(DetokenizationOperation.MergeBoth, detokenizeOperations[4]);
Assert.AreEqual(DetokenizationOperation.NoOperation, detokenizeOperations[5]);
}
public override void run(string[] args)
{
if (args.Length != 1)
{
Console.WriteLine(Help);
}
else
{
Detokenizer detokenizer = new DictionaryDetokenizer((new DetokenizationDictionaryLoader()).load(new File(args[0])));
ObjectStream <string> tokenizedLineStream = new PlainTextByLineStream(new InputStreamReader(Console.OpenStandardInput));
PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "sent");
perfMon.start();
try
{
string tokenizedLine;
while ((tokenizedLine = tokenizedLineStream.read()) != null)
{
// white space tokenize line
string[] tokens = WhitespaceTokenizer.INSTANCE.tokenize(tokenizedLine);
Console.WriteLine(detokenizer.detokenize(tokens, null));
perfMon.incrementCounter();
}
}
catch (IOException e)
{
CmdLineUtil.handleStdinIoError(e);
}
perfMon.stopAndPrintFinalResult();
}
}