public static String DeTokenize(String[] tokens, DetokenizationDictionary.Operation operation) { DetokenizationDictionary.Operation[] operations = new DetokenizationDictionary.Operation[tokens.Length]; for (int i = 0; i < tokens.Length; i++) { operations[i] = operation; } DetokenizationDictionary dictionary = new DetokenizationDictionary( tokens, operations); DictionaryDetokenizer detokenizer = new DictionaryDetokenizer( dictionary); return(detokenizer.detokenize(tokens, " ")); }
public void TestDetokenizer() { var dict = new DetokenizationDictionary { { ".", DetokenizationDictionary.Operation.MoveLeft }, { "!", DetokenizationDictionary.Operation.MoveLeft }, { "(", DetokenizationDictionary.Operation.MoveRight }, { ")", DetokenizationDictionary.Operation.MoveLeft }, { "\"", DetokenizationDictionary.Operation.RightLeftMatching }, { "-", DetokenizationDictionary.Operation.MoveBoth } }; var detokenizer = new DictionaryDetokenizer(dict); var detokenizeOperations = detokenizer.Detokenize(new[] { "Simple", "test", ".", "co", "-", "worker" }); Assert.AreEqual(DetokenizationOperation.NoOperation, detokenizeOperations[0]); Assert.AreEqual(DetokenizationOperation.NoOperation, detokenizeOperations[1]); Assert.AreEqual(DetokenizationOperation.MergeToLeft, detokenizeOperations[2]); Assert.AreEqual(DetokenizationOperation.NoOperation, detokenizeOperations[3]); Assert.AreEqual(DetokenizationOperation.MergeBoth, detokenizeOperations[4]); Assert.AreEqual(DetokenizationOperation.NoOperation, detokenizeOperations[5]); }
public override void run(string[] args) { if (args.Length != 1) { Console.WriteLine(Help); } else { Detokenizer detokenizer = new DictionaryDetokenizer((new DetokenizationDictionaryLoader()).load(new File(args[0]))); ObjectStream <string> tokenizedLineStream = new PlainTextByLineStream(new InputStreamReader(Console.OpenStandardInput)); PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "sent"); perfMon.start(); try { string tokenizedLine; while ((tokenizedLine = tokenizedLineStream.read()) != null) { // white space tokenize line string[] tokens = WhitespaceTokenizer.INSTANCE.tokenize(tokenizedLine); Console.WriteLine(detokenizer.detokenize(tokens, null)); perfMon.incrementCounter(); } } catch (IOException e) { CmdLineUtil.handleStdinIoError(e); } perfMon.stopAndPrintFinalResult(); } }