List <Driver> getDrivers()
{
if (mDrivers == null)
{
mDrivers = new List <Driver>();
LineNumberReader r = new LineNumberReader(new FileReader("/proc/tty/drivers"));
string l;
while ((l = r.ReadLine()) != null)
{
// Issue 3:
// Since driver name may contain spaces, we do not extract driver name with split()
string drivername = l.Substring(0, 0x15).Trim();
string[] w = System.Text.RegularExpressions.Regex.Split(l, @"\s{1,}");
if ((w.Length >= 5) && (w[w.Length - 1].Equals("serial")))
{
Log.Debug(TAG, "Found new driver " + drivername + " on " + w[w.Length - 4]);
mDrivers.Add(new Driver {
mDriverName = drivername, mDeviceRoot = w[w.Length - 4]
});
}
}
r.Close();
}
return(mDrivers);
}
private void LoadUniversalMap(string path)
{
LineNumberReader reader = null;
try
{
reader = new LineNumberReader(new FileReader(path));
for (string line; (line = reader.ReadLine()) != null;)
{
if (line.Trim().Equals(string.Empty))
{
continue;
}
string[] toks = line.Trim().Split("\\s+");
if (toks.Length != 2)
{
throw new Exception("Invalid mapping line: " + line);
}
universalMap[toks[0]] = toks[1];
}
reader.Close();
}
catch (FileNotFoundException)
{
System.Console.Error.Printf("%s: File not found %s%n", this.GetType().FullName, path);
}
catch (IOException e)
{
int lineId = (reader == null) ? -1 : reader.GetLineNumber();
System.Console.Error.Printf("%s: Error at line %d%n", this.GetType().FullName, lineId);
Sharpen.Runtime.PrintStackTrace(e);
}
}
private ICollection <string> MakeSplitSet(string splitFileName)
{
splitFileName = DataFilePaths.Convert(splitFileName);
ICollection <string> splitSet = Generics.NewHashSet();
LineNumberReader reader = null;
try
{
reader = new LineNumberReader(new FileReader(splitFileName));
for (string line; (line = reader.ReadLine()) != null;)
{
splitSet.Add(line.Trim());
}
reader.Close();
}
catch (FileNotFoundException e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
catch (IOException e)
{
System.Console.Error.Printf("%s: Error reading %s (line %d)%n", this.GetType().FullName, splitFileName, reader.GetLineNumber());
Sharpen.Runtime.PrintStackTrace(e);
}
return(splitSet);
}
protected internal virtual ICollection <string> BuildSplitMap(string path)
{
path = DataFilePaths.Convert(path);
ICollection <string> fileSet = Generics.NewHashSet();
LineNumberReader reader = null;
try
{
reader = new LineNumberReader(new FileReader(path));
while (reader.Ready())
{
string line = reader.ReadLine();
fileSet.Add(line.Trim());
}
reader.Close();
}
catch (FileNotFoundException)
{
System.Console.Error.Printf("%s: Could not open split file %s\n", this.GetType().FullName, path);
}
catch (IOException)
{
System.Console.Error.Printf("%s: Error reading split file %s (line %d)\n", this.GetType().FullName, path, reader.GetLineNumber());
}
return(fileSet);
}
public virtual void TestJobMonitorAndPrint()
{
JobStatus jobStatus_1 = new JobStatus(new JobID("job_000", 1), 1f, 0.1f, 0.1f, 0f
, JobStatus.State.Running, JobPriority.High, "tmp-user", "tmp-jobname", "tmp-queue"
, "tmp-jobfile", "tmp-url", true);
JobStatus jobStatus_2 = new JobStatus(new JobID("job_000", 1), 1f, 1f, 1f, 1f, JobStatus.State
.Succeeded, JobPriority.High, "tmp-user", "tmp-jobname", "tmp-queue", "tmp-jobfile"
, "tmp-url", true);
Org.Mockito.Mockito.DoAnswer(new _Answer_85()).When(job).GetTaskCompletionEvents(
Matchers.AnyInt(), Matchers.AnyInt());
Org.Mockito.Mockito.DoReturn(new TaskReport[5]).When(job).GetTaskReports(Matchers.IsA
<TaskType>());
Org.Mockito.Mockito.When(clientProtocol.GetJobStatus(Matchers.Any <JobID>())).ThenReturn
(jobStatus_1, jobStatus_2);
// setup the logger to capture all logs
Layout layout = Logger.GetRootLogger().GetAppender("stdout").GetLayout();
ByteArrayOutputStream os = new ByteArrayOutputStream();
WriterAppender appender = new WriterAppender(layout, os);
appender.SetThreshold(Level.All);
Logger qlogger = Logger.GetLogger(typeof(Job));
qlogger.AddAppender(appender);
job.MonitorAndPrintJob();
qlogger.RemoveAppender(appender);
LineNumberReader r = new LineNumberReader(new StringReader(os.ToString()));
string line;
bool foundHundred = false;
bool foundComplete = false;
bool foundUber = false;
string uberModeMatch = "uber mode : true";
string progressMatch = "map 100% reduce 100%";
string completionMatch = "completed successfully";
while ((line = r.ReadLine()) != null)
{
if (line.Contains(uberModeMatch))
{
foundUber = true;
}
foundHundred = line.Contains(progressMatch);
if (foundHundred)
{
break;
}
}
line = r.ReadLine();
foundComplete = line.Contains(completionMatch);
NUnit.Framework.Assert.IsTrue(foundUber);
NUnit.Framework.Assert.IsTrue(foundHundred);
NUnit.Framework.Assert.IsTrue(foundComplete);
System.Console.Out.WriteLine("The output of job.toString() is : \n" + job.ToString
());
NUnit.Framework.Assert.IsTrue(job.ToString().Contains("Number of maps: 5\n"));
NUnit.Framework.Assert.IsTrue(job.ToString().Contains("Number of reduces: 5\n"));
}
public virtual void Build()
{
LineNumberReader infile = null;
PrintWriter outfile = null;
string currentInfile = string.Empty;
try
{
outfile = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outFileName), "UTF-8")));
foreach (File path in pathsToData)
{
infile = new LineNumberReader(new BufferedReader(new InputStreamReader(new FileInputStream(path), "UTF-8")));
currentInfile = path.GetPath();
while (infile.Ready())
{
List <Word> sent = SentenceUtils.ToUntaggedList(infile.ReadLine().Split("\\s+"));
foreach (Word token in sent)
{
Matcher hasArabic = utf8ArabicChart.Matcher(token.Word());
if (hasArabic.Find())
{
token.SetWord(escaper.Apply(token.Word()));
token.SetWord(lexMapper.Map(null, token.Word()));
}
}
outfile.Println(SentenceUtils.ListToString(sent));
}
toStringBuffer.Append(string.Format(" Read %d input lines from %s", infile.GetLineNumber(), path.GetPath()));
}
infile.Close();
}
catch (UnsupportedEncodingException e)
{
System.Console.Error.Printf("%s: Filesystem does not support UTF-8 output\n", this.GetType().FullName);
Sharpen.Runtime.PrintStackTrace(e);
}
catch (FileNotFoundException)
{
System.Console.Error.Printf("%s: Could not open %s for writing\n", this.GetType().FullName, outFileName);
}
catch (IOException)
{
System.Console.Error.Printf("%s: Error reading from %s (line %d)\n", this.GetType().FullName, currentInfile, infile.GetLineNumber());
}
catch (Exception e)
{
System.Console.Error.Printf("%s: Input sentence from %s contains token mapped to null (line %d)\n", this.GetType().FullName, currentInfile, infile.GetLineNumber());
Sharpen.Runtime.PrintStackTrace(e);
}
finally
{
if (outfile != null)
{
outfile.Close();
}
}
}
public virtual void Setup(File path, params string[] options)
{
if (path == null || !path.Exists())
{
return;
}
LineNumberReader reader = null;
try
{
reader = new LineNumberReader(new FileReader(path));
bool insideTagMap = false;
for (string line; (line = reader.ReadLine()) != null;)
{
line = line.Trim();
Matcher isStartSymbol = startOfTagMap.Matcher(line);
insideTagMap = (isStartSymbol.Matches() || insideTagMap);
if (insideTagMap)
{
//Comment line
if (line.StartsWith(";"))
{
continue;
}
Matcher mappingLine = mapping.Matcher(line);
if (mappingLine.Find())
{
if (mappingLine.GroupCount() == numExpectedTokens)
{
string finalShortTag = ProcessShortTag(mappingLine.Group(1), mappingLine.Group(2));
tagMap[mappingLine.Group(1)] = finalShortTag;
}
else
{
System.Console.Error.Printf("%s: Skipping bad mapping in %s (line %d)%n", this.GetType().FullName, path.GetPath(), reader.GetLineNumber());
}
}
Matcher isEndSymbol = endOfTagMap.Matcher(line);
if (isEndSymbol.Matches())
{
break;
}
}
}
reader.Close();
}
catch (FileNotFoundException)
{
System.Console.Error.Printf("%s: Could not open mapping file %s%n", this.GetType().FullName, path.GetPath());
}
catch (IOException)
{
int lineNum = (reader == null) ? -1 : reader.GetLineNumber();
System.Console.Error.Printf("%s: Error reading %s (line %d)%n", this.GetType().FullName, path.GetPath(), lineNum);
}
}
/// <exception cref="System.IO.IOException"/>
private IList <string> ReadMockParams()
{
List <string> ret = new List <string>();
LineNumberReader reader = new LineNumberReader(new FileReader(mockParamFile));
string line;
while ((line = reader.ReadLine()) != null)
{
ret.AddItem(line);
}
reader.Close();
return(ret);
}
/**
* /// Create a Lattice from a LAT file. LAT files are created by the method Lattice.dump()
*
* /// @param fileName
*/
public Lattice(String fileName)
{
try
{
Console.WriteLine(@"Loading from " + fileName);
// load the nodes
var numberReader = new LineNumberReader(new StreamReader(fileName));
string line;
while ((line = numberReader.ReadLine()) != null)
{
var tokens = new StringTokenizer(line);
if (tokens.hasMoreTokens())
{
var type = tokens.nextToken();
if (type.Equals("edge:"))
{
Edge.Load(this, tokens);
}
else if (type.Equals("node:"))
{
Node.Load(this, tokens);
}
else if (type.Equals("initialNode:"))
{
InitialNode = GetNode(tokens.nextToken());
}
else if (type.Equals("terminalNode:"))
{
TerminalNode = GetNode(tokens.nextToken());
}
else if (type.Equals("logBase:"))
{
LogBase = Double.Parse(tokens.nextToken(), CultureInfo.InvariantCulture.NumberFormat);
}
else
{
numberReader.Close();
throw new Exception("SYNTAX ERROR: " + fileName +
'[' + numberReader.LineNumber + "] " + line);
}
}
}
numberReader.Close();
}
catch (Exception e)
{
throw new Exception(e.ToString());
}
}
protected internal override GrammarNode createGrammar()
{
this.initialNode = null;
this.finalNode = this.createGrammarNode(true);
try
{
try
{
LineNumberReader lineNumberReader = new LineNumberReader(new FileReader(this.refFile));
for (;;)
{
string text = lineNumberReader.readLine();
if (text == null)
{
break;
}
if (String.instancehelper_isEmpty(text))
{
break;
}
int num = String.instancehelper_indexOf(text, 40) + 1;
int num2 = String.instancehelper_indexOf(text, 41);
if (num >= 0)
{
if (num <= num2)
{
string text2 = String.instancehelper_substring(text, num, num2);
string text3 = String.instancehelper_trim(String.instancehelper_substring(text, 0, num - 1));
if (!String.instancehelper_isEmpty(text3))
{
this.initialNode = this.createGrammarNode("<sil>");
this.createForcedAlignerGrammar(this.initialNode, this.finalNode, text3);
this.__grammars.put(text2, this.initialNode);
this.currentUttName = text2;
}
}
}
}
lineNumberReader.close();
}
catch (FileNotFoundException ex)
{
throw new Error(ex);
}
}
catch (IOException ex3)
{
throw new Error(ex3);
}
return(this.initialNode);
}
/**
* /// Create a Lattice from a LAT file. LAT files are created by the method Lattice.dump()
*
* /// @param fileName
*/
public Lattice(String fileName)
{
try {
Debug.Print("Loading from " + fileName);
// load the nodes
LineNumberReader _in = new LineNumberReader(new StreamReader(fileName));
String line = _in.readLine();
while (line != null)
{
StringTokenizer tokens = new StringTokenizer(line);
if (tokens.hasMoreTokens())
{
String type = tokens.nextToken();
if (type.Equals("edge:"))
{
Edge.load(this, tokens);
}
else if (type.Equals("node:"))
{
Node.load(this, tokens);
}
else if (type.Equals("initialNode:"))
{
setInitialNode(getNode(tokens.nextToken()));
}
else if (type.Equals("terminalNode:"))
{
setTerminalNode(getNode(tokens.nextToken()));
}
else if (type.Equals("logBase:"))
{
logBase = Double.Parse(tokens.nextToken());
}
else
{
_in.close();
throw new Exception("SYNTAX ERROR: " + fileName +
'[' + _in.getLineNumber() + "] " + line);
}
}
line = _in.readLine();
}
_in.close();
}
catch (Exception e) {
throw new Exception(e.ToString());
}
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: private static String readLine(java.io.LineNumberReader r) throws java.io.IOException
private static string readLine(LineNumberReader r)
{
string line = null;
while ((line = r.readLine()) != null)
{
line = line.Trim();
if (line.Length > 0 && line[0] != '#')
{
return(line);
}
}
return(line);
}
/// <summary>
/// Counts and returns number of lines in a file </summary>
/// <returns> number of lines in a file </returns>
/// <exception cref="FileNotFoundException"> </exception>
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: private long countFileLines() throws java.io.FileNotFoundException
private long countFileLines()
{
LineNumberReader lnr = new LineNumberReader(new FileReader(file));
try {
// lnr.skip(Long.MAX_VALUE);
while (lnr.skip(long.MaxValue) > 0)
{
}
;
} catch (IOException ex) {
}
return(lnr.getLineNumber() + 1);
}
public Lattice(string fileName) : this()
{
try
{
java.lang.System.err.println(new StringBuilder().append("Loading from ").append(fileName).toString());
LineNumberReader lineNumberReader = new LineNumberReader(new FileReader(fileName));
string text;
while ((text = lineNumberReader.readLine()) != null)
{
StringTokenizer stringTokenizer = new StringTokenizer(text);
if (stringTokenizer.hasMoreTokens())
{
string text2 = stringTokenizer.nextToken();
if (String.instancehelper_equals(text2, "edge:"))
{
Edge.load(this, stringTokenizer);
}
else if (String.instancehelper_equals(text2, "node:"))
{
Node.load(this, stringTokenizer);
}
else if (String.instancehelper_equals(text2, "initialNode:"))
{
this.setInitialNode(this.getNode(stringTokenizer.nextToken()));
}
else if (String.instancehelper_equals(text2, "terminalNode:"))
{
this.setTerminalNode(this.getNode(stringTokenizer.nextToken()));
}
else
{
if (!String.instancehelper_equals(text2, "logBase:"))
{
lineNumberReader.close();
string text3 = new StringBuilder().append("SYNTAX ERROR: ").append(fileName).append('[').append(lineNumberReader.getLineNumber()).append("] ").append(text).toString();
throw new Error(text3);
}
this.logBase = Double.parseDouble(stringTokenizer.nextToken());
}
}
}
lineNumberReader.close();
}
catch (System.Exception ex)
{
throw new Error(Throwable.instancehelper_toString(ex));
}
}
public override void Parse(Reader @in)
{
LineNumberReader br = new LineNumberReader(@in);
try
{
string line = null;
string lastSynSetID = "";
CharsRef[] synset = new CharsRef[8];
int synsetSize = 0;
while ((line = br.readLine()) != null)
{
string synSetID = line.Substring(2, 9);
if (!synSetID.Equals(lastSynSetID))
{
addInternal(synset, synsetSize);
synsetSize = 0;
}
if (synset.Length <= synsetSize + 1)
{
CharsRef[] larger = new CharsRef[synset.Length * 2];
Array.Copy(synset, 0, larger, 0, synsetSize);
synset = larger;
}
synset[synsetSize] = parseSynonym(line, synset[synsetSize]);
synsetSize++;
lastSynSetID = synSetID;
}
// final synset in the file
addInternal(synset, synsetSize);
}
catch (System.ArgumentException e)
{
ParseException ex = new ParseException("Invalid synonym rule at line " + br.LineNumber, 0);
ex.initCause(e);
throw ex;
}
finally
{
br.close();
}
}
public override void Parse(Reader @in)
{
LineNumberReader br = new LineNumberReader(@in);
try
{
string line = null;
string lastSynSetID = "";
CharsRef[] synset = new CharsRef[8];
int synsetSize = 0;
while ((line = br.readLine()) != null)
{
string synSetID = line.Substring(2, 9);
if (!synSetID.Equals(lastSynSetID))
{
addInternal(synset, synsetSize);
synsetSize = 0;
}
if (synset.Length <= synsetSize+1)
{
CharsRef[] larger = new CharsRef[synset.Length * 2];
Array.Copy(synset, 0, larger, 0, synsetSize);
synset = larger;
}
synset[synsetSize] = parseSynonym(line, synset[synsetSize]);
synsetSize++;
lastSynSetID = synSetID;
}
// final synset in the file
addInternal(synset, synsetSize);
}
catch (System.ArgumentException e)
{
ParseException ex = new ParseException("Invalid synonym rule at line " + br.LineNumber, 0);
ex.initCause(e);
throw ex;
}
finally
{
br.close();
}
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: private static Step parseStep(java.io.LineNumberReader r, String header) throws java.io.IOException
private static Step parseStep(LineNumberReader r, string header)
{
Matcher matcher = headerPattern.matcher(header);
if (!matcher.find())
{
throw new Exception("Illegal Step header specified at line " + r.LineNumber);
}
Debug.Assert(matcher.groupCount() == 4);
string name = matcher.group(1);
int min = int.Parse(matcher.group(2));
int type = int.Parse(matcher.group(3));
string[] suffixes = parseList(matcher.group(4));
Rule[] rules = parseRules(r, type);
return(new Step(name, rules, min, suffixes));
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: private static Rule[] parseRules(java.io.LineNumberReader r, int type) throws java.io.IOException
private static Rule[] parseRules(LineNumberReader r, int type)
{
IList <Rule> rules = new List <Rule>();
string line;
while ((line = readLine(r)) != null)
{
Matcher matcher = stripPattern.matcher(line);
if (matcher.matches())
{
rules.Add(new Rule(matcher.group(1), int.Parse(matcher.group(2)), ""));
}
else
{
matcher = repPattern.matcher(line);
if (matcher.matches())
{
rules.Add(new Rule(matcher.group(1), int.Parse(matcher.group(2)), matcher.group(3)));
}
else
{
matcher = excPattern.matcher(line);
if (matcher.matches())
{
if (type == 0)
{
rules.Add(new RuleWithSuffixExceptions(matcher.group(1), int.Parse(matcher.group(2)), matcher.group(3), parseList(matcher.group(4))));
}
else
{
rules.Add(new RuleWithSetExceptions(matcher.group(1), int.Parse(matcher.group(2)), matcher.group(3), parseList(matcher.group(4))));
}
}
else
{
throw new Exception("Illegal Step rule specified at line " + r.LineNumber);
}
}
}
if (line.EndsWith(";", StringComparison.Ordinal))
{
return(rules.ToArray());
}
}
return(null);
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public void parse(java.io.Reader in) throws java.io.IOException, java.text.ParseException
public override void parse(Reader @in)
{
LineNumberReader br = new LineNumberReader(@in);
try
{
addInternal(br);
}
catch (System.ArgumentException e)
{
ParseException ex = new ParseException("Invalid synonym rule at line " + br.LineNumber, 0);
ex.initCause(e);
throw ex;
}
finally
{
br.close();
}
}
public override void Parse(Reader @in)
{
LineNumberReader br = new LineNumberReader(@in);
try
{
addInternal(br);
}
catch (System.ArgumentException e)
{
ParseException ex = new ParseException("Invalid synonym rule at line " + br.LineNumber, 0);
ex.initCause(e);
throw ex;
}
finally
{
br.close();
}
}
/// <summary>
/// Parse a resource file into an RSLP stemmer description. </summary>
/// <returns> a Map containing the named Steps in this description. </returns>
protected internal static IDictionary <string, Step> parse(Type clazz, string resource)
{
// TODO: this parser is ugly, but works. use a jflex grammar instead.
try
{
InputStream @is = clazz.getResourceAsStream(resource);
LineNumberReader r = new LineNumberReader(new InputStreamReader(@is, StandardCharsets.UTF_8));
IDictionary <string, Step> steps = new Dictionary <string, Step>();
string step;
while ((step = readLine(r)) != null)
{
Step s = parseStep(r, step);
steps[s.name] = s;
}
r.close();
return(steps);
}
catch (IOException e)
{
throw new Exception(e);
}
}
public static Lattice ReadSlf(String fileName)
{
var lattice = new Lattice();
var @in = new LineNumberReader(new StreamReader(fileName));
String line;
var readingNodes = false;
var readingEdges = false;
var startIdx = 0;
var endIdx = 1;
var lmscale = 9.5;
while ((line = @in.ReadLine()) != null)
{
if (line.Contains("Node definitions"))
{
readingEdges = false;
readingNodes = true;
continue;
}
if (line.Contains("Link definitions"))
{
readingEdges = true;
readingNodes = false;
continue;
}
if (line.StartsWith("#"))
{
//skip commented line
continue;
}
if (readingNodes)
{
//reading node info, format:
//I=id t=start_time_sec W=word_transcription
var parts = line.Split("\\s+");
if (parts.Length != 3 || !parts[0].StartsWith("I=") || !parts[1].StartsWith("t=") || !parts[2].StartsWith("W="))
{
@in.Close();
throw new IOException("Unknown node definition: " + line);
}
var idx = Convert.ToInt32(parts[0].Substring(2), CultureInfo.InvariantCulture.NumberFormat);
//convert to milliseconds inplace
var beginTime = (long)(Convert.ToDouble(parts[1].Substring(2), CultureInfo.InvariantCulture.NumberFormat) * 1000);
var wordStr = parts[2].Substring(2);
var isFiller = false;
if (idx == startIdx || wordStr.Equals("!ENTER"))
{
wordStr = "<s>";
isFiller = true;
}
if (idx == endIdx || wordStr.Equals("!EXIT"))
{
wordStr = "</s>";
isFiller = true;
}
if (wordStr.Equals("!NULL"))
{
wordStr = "<sil>";
isFiller = true;
}
if (wordStr.StartsWith("["))
{
isFiller = true;
}
var word = new Word(wordStr, new Pronunciation[0], isFiller);
var node = lattice.AddNode(idx.ToString(CultureInfo.InvariantCulture), word, beginTime, -1);
if (wordStr.Equals("<s>"))
{
lattice.InitialNode = node;
}
if (wordStr.Equals("</s>"))
{
lattice.TerminalNode = node;
}
}
else if (readingEdges)
{
//reading edge info, format:
//J=id S=from_node E=to_node a=acoustic_score l=language_score
var parts = line.Split("\\s+");
if (parts.Length != 5 || !parts[1].StartsWith("S=") || !parts[2].StartsWith("E=") ||
!parts[3].StartsWith("a=") || !parts[4].StartsWith("l="))
{
@in.Close();
throw new IOException("Unknown edge definition: " + line);
}
var fromId = parts[1].Substring(2);
var toId = parts[2].Substring(2);
var ascore = Convert.ToDouble(parts[3].Substring(2), CultureInfo.InvariantCulture.NumberFormat);
var lscore = Convert.ToDouble(parts[4].Substring(2), CultureInfo.InvariantCulture.NumberFormat) * lmscale;
lattice.AddEdge(lattice.Nodes.Get(fromId), lattice.Nodes.Get(toId), ascore, lscore);
}
else
{
//reading header here if needed
if (line.StartsWith("start="))
{
startIdx = Convert.ToInt32(line.Replace("start=", ""), CultureInfo.InvariantCulture.NumberFormat);
}
if (line.StartsWith("end="))
{
endIdx = Convert.ToInt32(line.Replace("end=", ""), CultureInfo.InvariantCulture.NumberFormat);
}
if (line.StartsWith("lmscale="))
{
lmscale = Convert.ToDouble(line.Replace("lmscale=", ""), CultureInfo.InvariantCulture.NumberFormat);
}
}
}
foreach (var node in lattice.Nodes.Values)
{
//calculate end time of nodes depending successors begin time
foreach (var edge in node.LeavingEdges)
{
if (node.EndTime < 0 || node.EndTime > edge.ToNode.BeginTime)
{
node.EndTime = edge.ToNode.BeginTime;
}
}
}
@in.Close();
return(lattice);
}
/// <summary>
/// Parse a resource file into an RSLP stemmer description. </summary>
/// <returns> a Map containing the named Steps in this description. </returns>
protected internal static IDictionary<string, Step> parse(Type clazz, string resource)
{
// TODO: this parser is ugly, but works. use a jflex grammar instead.
try
{
InputStream @is = clazz.getResourceAsStream(resource);
LineNumberReader r = new LineNumberReader(new InputStreamReader(@is, StandardCharsets.UTF_8));
IDictionary<string, Step> steps = new Dictionary<string, Step>();
string step;
while ((step = readLine(r)) != null)
{
Step s = parseStep(r, step);
steps[s.name] = s;
}
r.close();
return steps;
}
catch (IOException e)
{
throw new Exception(e);
}
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: private static Rule[] parseRules(java.io.LineNumberReader r, int type) throws java.io.IOException
private static Rule[] parseRules(LineNumberReader r, int type)
{
IList<Rule> rules = new List<Rule>();
string line;
while ((line = readLine(r)) != null)
{
Matcher matcher = stripPattern.matcher(line);
if (matcher.matches())
{
rules.Add(new Rule(matcher.group(1), int.Parse(matcher.group(2)), ""));
}
else
{
matcher = repPattern.matcher(line);
if (matcher.matches())
{
rules.Add(new Rule(matcher.group(1), int.Parse(matcher.group(2)), matcher.group(3)));
}
else
{
matcher = excPattern.matcher(line);
if (matcher.matches())
{
if (type == 0)
{
rules.Add(new RuleWithSuffixExceptions(matcher.group(1), int.Parse(matcher.group(2)), matcher.group(3), parseList(matcher.group(4))));
}
else
{
rules.Add(new RuleWithSetExceptions(matcher.group(1), int.Parse(matcher.group(2)), matcher.group(3), parseList(matcher.group(4))));
}
}
else
{
throw new Exception("Illegal Step rule specified at line " + r.LineNumber);
}
}
}
if (line.EndsWith(";", StringComparison.Ordinal))
{
return rules.ToArray();
}
}
return null;
}
public virtual void Parse()
{
int lineNum = 0;
try
{
LineNumberReader reader = new LineNumberReader(new FileReader(configFile));
Properties paramsForDataset = null;
while (reader.Ready())
{
string line = reader.ReadLine();
lineNum = reader.GetLineNumber();
//For exception handling
Matcher m = skipLine.Matcher(line);
if (m.LookingAt())
{
continue;
}
m = setDelim.Matcher(line);
if (m.Matches() && paramsForDataset != null)
{
datasetList.Add(paramsForDataset);
paramsForDataset = null;
continue;
}
else
{
if (paramsForDataset == null)
{
paramsForDataset = new Properties();
}
}
bool matched = false;
foreach (string param in patternsMap.Keys)
{
Pair <Pattern, Pattern> paramTemplate = patternsMap[param];
Matcher paramToken = paramTemplate.first.Matcher(line);
if (paramToken.LookingAt())
{
matched = true;
string[] tokens = line.Split(Delim);
if (tokens.Length != 2)
{
System.Console.Error.Printf("%s: Skipping malformed parameter in %s (line %d)%n", this.GetType().FullName, configFile, reader.GetLineNumber());
break;
}
string actualParam = tokens[0].Trim();
string paramValue = tokens[1].Trim();
if (paramTemplate.second != null)
{
paramToken = paramTemplate.second.Matcher(paramValue);
if (paramToken.Matches())
{
paramsForDataset.SetProperty(actualParam, paramValue);
}
else
{
System.Console.Error.Printf("%s: Skipping illegal parameter value in %s (line %d)%n", this.GetType().FullName, configFile, reader.GetLineNumber());
break;
}
}
else
{
paramsForDataset.SetProperty(actualParam, paramValue);
}
}
}
if (!matched)
{
string error = this.GetType().FullName + ": Unknown token in " + configFile + " (line " + reader.GetLineNumber() + ")%n";
System.Console.Error.Printf(error);
throw new ArgumentException(error);
}
}
if (paramsForDataset != null)
{
datasetList.Add(paramsForDataset);
}
reader.Close();
}
catch (FileNotFoundException)
{
System.Console.Error.Printf("%s: Cannot open file %s%n", this.GetType().FullName, configFile);
}
catch (IOException)
{
System.Console.Error.Printf("%s: Error reading %s (line %d)%n", this.GetType().FullName, configFile, lineNum);
}
}
public virtual void TestContainerLaunch()
{
string appSubmitter = "nobody";
string appId = "APP_ID";
string containerId = "CONTAINER_ID";
string testImage = "\"sequenceiq/hadoop-docker:2.4.1\"";
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>(Org.Mockito.Mockito.ReturnsDeepStubs);
ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(Org.Mockito.Mockito.ReturnsDeepStubs
);
ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
();
Dictionary <string, string> env = new Dictionary <string, string>();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
Org.Mockito.Mockito.When(cId.GetApplicationAttemptId().GetApplicationId().ToString
()).ThenReturn(appId);
Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
env[YarnConfiguration.NmDockerContainerExecutorImageName] = testImage;
Path scriptPath = new Path("file:///bin/echo");
Path tokensPath = new Path("file:///dev/null");
Path pidFile = new Path(workDir, "pid");
dockerContainerExecutor.ActivateContainer(cId, pidFile);
int ret = dockerContainerExecutor.LaunchContainer(container, scriptPath, tokensPath
, appSubmitter, appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs
());
NUnit.Framework.Assert.AreEqual(0, ret);
//get the script
Path sessionScriptPath = new Path(workDir, Shell.AppendScriptExtension(DockerContainerExecutor
.DockerContainerExecutorSessionScript));
LineNumberReader lnr = new LineNumberReader(new FileReader(sessionScriptPath.ToString
()));
bool cmdFound = false;
IList <string> localDirs = DirsToMount(dirsHandler.GetLocalDirs());
IList <string> logDirs = DirsToMount(dirsHandler.GetLogDirs());
IList <string> workDirMount = DirsToMount(Sharpen.Collections.SingletonList(workDir
.ToUri().GetPath()));
IList <string> expectedCommands = new AList <string>(Arrays.AsList(DockerLaunchCommand
, "run", "--rm", "--net=host", "--name", containerId));
Sharpen.Collections.AddAll(expectedCommands, localDirs);
Sharpen.Collections.AddAll(expectedCommands, logDirs);
Sharpen.Collections.AddAll(expectedCommands, workDirMount);
string shellScript = workDir + "/launch_container.sh";
Sharpen.Collections.AddAll(expectedCommands, Arrays.AsList(testImage.ReplaceAll("['\"]"
, string.Empty), "bash", "\"" + shellScript + "\""));
string expectedPidString = "echo `/bin/true inspect --format {{.State.Pid}} " + containerId
+ "` > " + pidFile.ToString() + ".tmp";
bool pidSetterFound = false;
while (lnr.Ready())
{
string line = lnr.ReadLine();
Log.Debug("line: " + line);
if (line.StartsWith(DockerLaunchCommand))
{
IList <string> command = new AList <string>();
foreach (string s in line.Split("\\s+"))
{
command.AddItem(s.Trim());
}
NUnit.Framework.Assert.AreEqual(expectedCommands, command);
cmdFound = true;
}
else
{
if (line.StartsWith("echo"))
{
NUnit.Framework.Assert.AreEqual(expectedPidString, line);
pidSetterFound = true;
}
}
}
NUnit.Framework.Assert.IsTrue(cmdFound);
NUnit.Framework.Assert.IsTrue(pidSetterFound);
}
public static Lattice readSlf(InputStream stream)
{
Lattice lattice = new Lattice();
LineNumberReader lineNumberReader = new LineNumberReader(new InputStreamReader(stream));
int num = 0;
int num2 = 0;
int num3 = 0;
int num4 = 1;
double num5 = 9.5;
string text;
while ((text = lineNumberReader.readLine()) != null)
{
string text2 = text;
object obj = "Node definitions";
CharSequence charSequence = CharSequence.Cast(obj);
if (String.instancehelper_contains(text2, charSequence))
{
num2 = 0;
num = 1;
}
else
{
string text3 = text;
obj = "Link definitions";
charSequence = CharSequence.Cast(obj);
if (String.instancehelper_contains(text3, charSequence))
{
num2 = 1;
num = 0;
}
else if (!String.instancehelper_startsWith(text, "#"))
{
if (num != 0)
{
string[] array = String.instancehelper_split(text, "\\s+");
if (array.Length != 3 || !String.instancehelper_startsWith(array[0], "I=") || !String.instancehelper_startsWith(array[1], "t=") || !String.instancehelper_startsWith(array[2], "W="))
{
lineNumberReader.close();
string text4 = new StringBuilder().append("Unknown node definition: ").append(text).toString();
throw new IOException(text4);
}
int num6 = Integer.parseInt(String.instancehelper_substring(array[0], 2));
long beginTime = ByteCodeHelper.d2l(Double.parseDouble(String.instancehelper_substring(array[1], 2)) * 1000.0);
string text5 = String.instancehelper_substring(array[2], 2);
int isFiller = 0;
if (num6 == num3 || String.instancehelper_equals(text5, "!ENTER"))
{
text5 = "<s>";
isFiller = 1;
}
if (num6 == num4 || String.instancehelper_equals(text5, "!EXIT"))
{
text5 = "</s>";
isFiller = 1;
}
if (String.instancehelper_equals(text5, "!NULL"))
{
text5 = "<sil>";
isFiller = 1;
}
if (String.instancehelper_startsWith(text5, "["))
{
isFiller = 1;
}
Word word = new Word(text5, new Pronunciation[0], isFiller != 0);
Node node = lattice.addNode(Integer.toString(num6), word, beginTime, -1L);
if (String.instancehelper_equals(text5, "<s>"))
{
lattice.setInitialNode(node);
}
if (String.instancehelper_equals(text5, "</s>"))
{
lattice.setTerminalNode(node);
}
}
else if (num2 != 0)
{
string[] array = String.instancehelper_split(text, "\\s+");
if (array.Length != 5 || !String.instancehelper_startsWith(array[1], "S=") || !String.instancehelper_startsWith(array[2], "E=") || !String.instancehelper_startsWith(array[3], "a=") || !String.instancehelper_startsWith(array[4], "l="))
{
lineNumberReader.close();
string text6 = new StringBuilder().append("Unknown edge definition: ").append(text).toString();
throw new IOException(text6);
}
string text7 = String.instancehelper_substring(array[1], 2);
string text8 = String.instancehelper_substring(array[2], 2);
double acousticScore = Double.parseDouble(String.instancehelper_substring(array[3], 2));
double lmScore = Double.parseDouble(String.instancehelper_substring(array[4], 2)) * num5;
lattice.addEdge((Node)lattice.nodes.get(text7), (Node)lattice.nodes.get(text8), acousticScore, lmScore);
}
else
{
if (String.instancehelper_startsWith(text, "start="))
{
string text9 = text;
object obj2 = "start=";
obj = "";
object obj3 = obj2;
charSequence = CharSequence.Cast(obj3);
CharSequence charSequence2 = charSequence;
obj3 = obj;
charSequence = CharSequence.Cast(obj3);
num3 = Integer.parseInt(String.instancehelper_replace(text9, charSequence2, charSequence));
}
if (String.instancehelper_startsWith(text, "end="))
{
string text10 = text;
object obj4 = "end=";
object obj3 = "";
obj = obj4;
charSequence = CharSequence.Cast(obj);
CharSequence charSequence3 = charSequence;
obj = obj3;
charSequence = CharSequence.Cast(obj);
num4 = Integer.parseInt(String.instancehelper_replace(text10, charSequence3, charSequence));
}
if (String.instancehelper_startsWith(text, "lmscale="))
{
string text11 = text;
object obj5 = "lmscale=";
obj = "";
object obj3 = obj5;
charSequence = CharSequence.Cast(obj3);
CharSequence charSequence4 = charSequence;
obj3 = obj;
charSequence = CharSequence.Cast(obj3);
num5 = Double.parseDouble(String.instancehelper_replace(text11, charSequence4, charSequence));
}
}
}
}
}
Iterator iterator = lattice.nodes.values().iterator();
while (iterator.hasNext())
{
Node node2 = (Node)iterator.next();
Iterator iterator2 = node2.getLeavingEdges().iterator();
while (iterator2.hasNext())
{
Edge edge = (Edge)iterator2.next();
if (node2.getEndTime() < 0L || node2.getEndTime() > edge.getToNode().getBeginTime())
{
node2.setEndTime(Math.max(edge.getToNode().getBeginTime(), node2.getBeginTime()));
}
}
}
return(lattice);
}
protected override GrammarNode CreateGrammar()
{
// TODO: FlatLinguist requires the initial grammar node
// to contain a single silence. We'll do that for now,
// but once the FlatLinguist is fixed, this should be
// returned to its former method of creating an empty
// initial grammar node
// initialNode = createGrammarNode(initialID, false);
InitialNode = null;
FinalNode = CreateGrammarNode(true);
try
{
var @in = new LineNumberReader(new StreamReader(RefFile));
String line;
while (true)
{
line = @in.ReadLine();
if (line == null || line.IsEmpty())
{
break;
}
int uttNameStart = line.IndexOf('(') + 1;
int uttNameEnd = line.IndexOf(')');
if (uttNameStart < 0 || uttNameStart > uttNameEnd)
{
continue;
}
String uttName = line.JSubString(uttNameStart, uttNameEnd);
String transcript = line.JSubString(0, uttNameStart - 1).Trim();
if (transcript.IsEmpty())
{
continue;
}
InitialNode = CreateGrammarNode(IDictionary.SilenceSpelling);
CreateForcedAlignerGrammar(InitialNode, FinalNode, transcript);
Grammars.Put(uttName, InitialNode);
CurrentUttName = uttName;
}
@in.Close();
}
catch (FileNotFoundException e)
{
throw new Error(e);
}
catch (IOException e)
{
throw new Error(e);
}
catch (NoSuchMethodException e)
{
throw new Error(e);
}
return(InitialNode);
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: private static String readLine(java.io.LineNumberReader r) throws java.io.IOException
private static string readLine(LineNumberReader r)
{
string line = null;
while ((line = r.readLine()) != null)
{
line = line.Trim();
if (line.Length > 0 && line[0] != '#')
{
return line;
}
}
return line;
}
public Lexer(TextReader textReader)
{
tokens = new Queue <Token>();
reader = new LineNumberReader(textReader);
hasMore = true;
}
/// <summary>Read in typed dependencies.</summary>
/// <remarks>
/// Read in typed dependencies. Warning created typed dependencies are not
/// backed by any sort of a tree structure.
/// </remarks>
/// <param name="filename"/>
/// <exception cref="System.IO.IOException"/>
protected internal static IList <ICollection <TypedDependency> > ReadDeps(string filename)
{
LineNumberReader breader = new LineNumberReader(new FileReader(filename));
IList <ICollection <TypedDependency> > readDeps = new List <ICollection <TypedDependency> >();
ICollection <TypedDependency> deps = new List <TypedDependency>();
for (string line = breader.ReadLine(); line != null; line = breader.ReadLine())
{
if (line.Equals("null(-0,-0)") || line.Equals("null(-1,-1)"))
{
readDeps.Add(deps);
deps = new List <TypedDependency>();
continue;
}
// relex parse error
try
{
if (line.Equals(string.Empty))
{
if (deps.Count != 0)
{
//System.out.println(deps);
readDeps.Add(deps);
deps = new List <TypedDependency>();
}
continue;
}
int firstParen = line.IndexOf("(");
int commaSpace = line.IndexOf(", ");
string depName = Sharpen.Runtime.Substring(line, 0, firstParen);
string govName = Sharpen.Runtime.Substring(line, firstParen + 1, commaSpace);
string childName = Sharpen.Runtime.Substring(line, commaSpace + 2, line.Length - 1);
GrammaticalRelation grel = GrammaticalRelation.ValueOf(depName);
if (depName.StartsWith("prep_"))
{
string prep = Sharpen.Runtime.Substring(depName, 5);
grel = EnglishGrammaticalRelations.GetPrep(prep);
}
if (depName.StartsWith("prepc_"))
{
string prepc = Sharpen.Runtime.Substring(depName, 6);
grel = EnglishGrammaticalRelations.GetPrepC(prepc);
}
if (depName.StartsWith("conj_"))
{
string conj = Sharpen.Runtime.Substring(depName, 5);
grel = EnglishGrammaticalRelations.GetConj(conj);
}
if (grel == null)
{
throw new Exception("Unknown grammatical relation '" + depName + "'");
}
//Word govWord = new Word(govName.substring(0, govDash));
IndexedWord govWord = new IndexedWord();
govWord.SetValue(NormalizeNumbers(govName));
govWord.SetWord(govWord.Value());
//Word childWord = new Word(childName.substring(0, childDash));
IndexedWord childWord = new IndexedWord();
childWord.SetValue(NormalizeNumbers(childName));
childWord.SetWord(childWord.Value());
TypedDependency dep = new DependencyScoring.TypedDependencyStringEquality(grel, govWord, childWord);
deps.Add(dep);
}
catch (Exception e)
{
breader.Close();
throw new Exception("Error on line " + breader.GetLineNumber() + ":\n\n" + e);
}
}
if (deps.Count != 0)
{
readDeps.Add(deps);
}
//log.info("last: "+readDeps.get(readDeps.size()-1));
breader.Close();
return(readDeps);
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: private static Step parseStep(java.io.LineNumberReader r, String header) throws java.io.IOException
private static Step parseStep(LineNumberReader r, string header)
{
Matcher matcher = headerPattern.matcher(header);
if (!matcher.find())
{
throw new Exception("Illegal Step header specified at line " + r.LineNumber);
}
Debug.Assert(matcher.groupCount() == 4);
string name = matcher.group(1);
int min = int.Parse(matcher.group(2));
int type = int.Parse(matcher.group(3));
string[] suffixes = parseList(matcher.group(4));
Rule[] rules = parseRules(r, type);
return new Step(name, rules, min, suffixes);
}
