Esempio n. 1
0
        public virtual void TestJobMonitorAndPrint()
        {
            JobStatus jobStatus_1 = new JobStatus(new JobID("job_000", 1), 1f, 0.1f, 0.1f, 0f
                                                  , JobStatus.State.Running, JobPriority.High, "tmp-user", "tmp-jobname", "tmp-queue"
                                                  , "tmp-jobfile", "tmp-url", true);
            JobStatus jobStatus_2 = new JobStatus(new JobID("job_000", 1), 1f, 1f, 1f, 1f, JobStatus.State
                                                  .Succeeded, JobPriority.High, "tmp-user", "tmp-jobname", "tmp-queue", "tmp-jobfile"
                                                  , "tmp-url", true);

            Org.Mockito.Mockito.DoAnswer(new _Answer_85()).When(job).GetTaskCompletionEvents(
                Matchers.AnyInt(), Matchers.AnyInt());
            Org.Mockito.Mockito.DoReturn(new TaskReport[5]).When(job).GetTaskReports(Matchers.IsA
                                                                                     <TaskType>());
            Org.Mockito.Mockito.When(clientProtocol.GetJobStatus(Matchers.Any <JobID>())).ThenReturn
                (jobStatus_1, jobStatus_2);
            // setup the logger to capture all logs
            Layout layout                  = Logger.GetRootLogger().GetAppender("stdout").GetLayout();
            ByteArrayOutputStream os       = new ByteArrayOutputStream();
            WriterAppender        appender = new WriterAppender(layout, os);

            appender.SetThreshold(Level.All);
            Logger qlogger = Logger.GetLogger(typeof(Job));

            qlogger.AddAppender(appender);
            job.MonitorAndPrintJob();
            qlogger.RemoveAppender(appender);
            LineNumberReader r = new LineNumberReader(new StringReader(os.ToString()));
            string           line;
            bool             foundHundred    = false;
            bool             foundComplete   = false;
            bool             foundUber       = false;
            string           uberModeMatch   = "uber mode : true";
            string           progressMatch   = "map 100% reduce 100%";
            string           completionMatch = "completed successfully";

            while ((line = r.ReadLine()) != null)
            {
                if (line.Contains(uberModeMatch))
                {
                    foundUber = true;
                }
                foundHundred = line.Contains(progressMatch);
                if (foundHundred)
                {
                    break;
                }
            }
            line          = r.ReadLine();
            foundComplete = line.Contains(completionMatch);
            NUnit.Framework.Assert.IsTrue(foundUber);
            NUnit.Framework.Assert.IsTrue(foundHundred);
            NUnit.Framework.Assert.IsTrue(foundComplete);
            System.Console.Out.WriteLine("The output of job.toString() is : \n" + job.ToString
                                             ());
            NUnit.Framework.Assert.IsTrue(job.ToString().Contains("Number of maps: 5\n"));
            NUnit.Framework.Assert.IsTrue(job.ToString().Contains("Number of reduces: 5\n"));
        }
Esempio n. 2
0
        protected internal virtual ICollection <string> BuildSplitMap(string path)
        {
            path = DataFilePaths.Convert(path);
            ICollection <string> fileSet = Generics.NewHashSet();
            LineNumberReader     reader  = null;

            try
            {
                reader = new LineNumberReader(new FileReader(path));
                while (reader.Ready())
                {
                    string line = reader.ReadLine();
                    fileSet.Add(line.Trim());
                }
                reader.Close();
            }
            catch (FileNotFoundException)
            {
                System.Console.Error.Printf("%s: Could not open split file %s\n", this.GetType().FullName, path);
            }
            catch (IOException)
            {
                System.Console.Error.Printf("%s: Error reading split file %s (line %d)\n", this.GetType().FullName, path, reader.GetLineNumber());
            }
            return(fileSet);
        }
Esempio n. 3
0
        private ICollection <string> MakeSplitSet(string splitFileName)
        {
            splitFileName = DataFilePaths.Convert(splitFileName);
            ICollection <string> splitSet = Generics.NewHashSet();
            LineNumberReader     reader   = null;

            try
            {
                reader = new LineNumberReader(new FileReader(splitFileName));
                for (string line; (line = reader.ReadLine()) != null;)
                {
                    splitSet.Add(line.Trim());
                }
                reader.Close();
            }
            catch (FileNotFoundException e)
            {
                Sharpen.Runtime.PrintStackTrace(e);
            }
            catch (IOException e)
            {
                System.Console.Error.Printf("%s: Error reading %s (line %d)%n", this.GetType().FullName, splitFileName, reader.GetLineNumber());
                Sharpen.Runtime.PrintStackTrace(e);
            }
            return(splitSet);
        }
Esempio n. 4
0
 List <Driver> getDrivers()
 {
     if (mDrivers == null)
     {
         mDrivers = new List <Driver>();
         LineNumberReader r = new LineNumberReader(new FileReader("/proc/tty/drivers"));
         string           l;
         while ((l = r.ReadLine()) != null)
         {
             // Issue 3:
             // Since driver name may contain spaces, we do not extract driver name with split()
             string   drivername = l.Substring(0, 0x15).Trim();
             string[] w          = System.Text.RegularExpressions.Regex.Split(l, @"\s{1,}");
             if ((w.Length >= 5) && (w[w.Length - 1].Equals("serial")))
             {
                 Log.Debug(TAG, "Found new driver " + drivername + " on " + w[w.Length - 4]);
                 mDrivers.Add(new Driver {
                     mDriverName = drivername, mDeviceRoot = w[w.Length - 4]
                 });
             }
         }
         r.Close();
     }
     return(mDrivers);
 }
Esempio n. 5
0
        private void LoadUniversalMap(string path)
        {
            LineNumberReader reader = null;

            try
            {
                reader = new LineNumberReader(new FileReader(path));
                for (string line; (line = reader.ReadLine()) != null;)
                {
                    if (line.Trim().Equals(string.Empty))
                    {
                        continue;
                    }
                    string[] toks = line.Trim().Split("\\s+");
                    if (toks.Length != 2)
                    {
                        throw new Exception("Invalid mapping line: " + line);
                    }
                    universalMap[toks[0]] = toks[1];
                }
                reader.Close();
            }
            catch (FileNotFoundException)
            {
                System.Console.Error.Printf("%s: File not found %s%n", this.GetType().FullName, path);
            }
            catch (IOException e)
            {
                int lineId = (reader == null) ? -1 : reader.GetLineNumber();
                System.Console.Error.Printf("%s: Error at line %d%n", this.GetType().FullName, lineId);
                Sharpen.Runtime.PrintStackTrace(e);
            }
        }
Esempio n. 6
0
        private void ReadLine()
        {
            string line = reader.ReadLine();

            if (line == null)
            {
                hasMore = false;
                return;
            }

            int lineNumber = reader.LineNumber;
            int pos        = 0;
            int endPos     = line.Length;

            while (pos < endPos)
            {
                Match match = Pattern.Match(line, pos, endPos - pos);
                if (match.Success)
                {
                    AddToken(lineNumber, match);
                    pos = match.Index + match.Length;
                }
                else
                {
                    throw new ToysException($"Invalid token at line {lineNumber}, col {pos}.");
                }
            }
            tokens.Enqueue(new IdentityToken(lineNumber, endPos, endPos + Token.EOL.Length - 1, Token.EOL));
        }
        public virtual void Build()
        {
            LineNumberReader infile        = null;
            PrintWriter      outfile       = null;
            string           currentInfile = string.Empty;

            try
            {
                outfile = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outFileName), "UTF-8")));
                foreach (File path in pathsToData)
                {
                    infile        = new LineNumberReader(new BufferedReader(new InputStreamReader(new FileInputStream(path), "UTF-8")));
                    currentInfile = path.GetPath();
                    while (infile.Ready())
                    {
                        List <Word> sent = SentenceUtils.ToUntaggedList(infile.ReadLine().Split("\\s+"));
                        foreach (Word token in sent)
                        {
                            Matcher hasArabic = utf8ArabicChart.Matcher(token.Word());
                            if (hasArabic.Find())
                            {
                                token.SetWord(escaper.Apply(token.Word()));
                                token.SetWord(lexMapper.Map(null, token.Word()));
                            }
                        }
                        outfile.Println(SentenceUtils.ListToString(sent));
                    }
                    toStringBuffer.Append(string.Format(" Read %d input lines from %s", infile.GetLineNumber(), path.GetPath()));
                }
                infile.Close();
            }
            catch (UnsupportedEncodingException e)
            {
                System.Console.Error.Printf("%s: Filesystem does not support UTF-8 output\n", this.GetType().FullName);
                Sharpen.Runtime.PrintStackTrace(e);
            }
            catch (FileNotFoundException)
            {
                System.Console.Error.Printf("%s: Could not open %s for writing\n", this.GetType().FullName, outFileName);
            }
            catch (IOException)
            {
                System.Console.Error.Printf("%s: Error reading from %s (line %d)\n", this.GetType().FullName, currentInfile, infile.GetLineNumber());
            }
            catch (Exception e)
            {
                System.Console.Error.Printf("%s: Input sentence from %s contains token mapped to null (line %d)\n", this.GetType().FullName, currentInfile, infile.GetLineNumber());
                Sharpen.Runtime.PrintStackTrace(e);
            }
            finally
            {
                if (outfile != null)
                {
                    outfile.Close();
                }
            }
        }
Esempio n. 8
0
        public virtual void Setup(File path, params string[] options)
        {
            if (path == null || !path.Exists())
            {
                return;
            }
            LineNumberReader reader = null;

            try
            {
                reader = new LineNumberReader(new FileReader(path));
                bool insideTagMap = false;
                for (string line; (line = reader.ReadLine()) != null;)
                {
                    line = line.Trim();
                    Matcher isStartSymbol = startOfTagMap.Matcher(line);
                    insideTagMap = (isStartSymbol.Matches() || insideTagMap);
                    if (insideTagMap)
                    {
                        //Comment line
                        if (line.StartsWith(";"))
                        {
                            continue;
                        }
                        Matcher mappingLine = mapping.Matcher(line);
                        if (mappingLine.Find())
                        {
                            if (mappingLine.GroupCount() == numExpectedTokens)
                            {
                                string finalShortTag = ProcessShortTag(mappingLine.Group(1), mappingLine.Group(2));
                                tagMap[mappingLine.Group(1)] = finalShortTag;
                            }
                            else
                            {
                                System.Console.Error.Printf("%s: Skipping bad mapping in %s (line %d)%n", this.GetType().FullName, path.GetPath(), reader.GetLineNumber());
                            }
                        }
                        Matcher isEndSymbol = endOfTagMap.Matcher(line);
                        if (isEndSymbol.Matches())
                        {
                            break;
                        }
                    }
                }
                reader.Close();
            }
            catch (FileNotFoundException)
            {
                System.Console.Error.Printf("%s: Could not open mapping file %s%n", this.GetType().FullName, path.GetPath());
            }
            catch (IOException)
            {
                int lineNum = (reader == null) ? -1 : reader.GetLineNumber();
                System.Console.Error.Printf("%s: Error reading %s (line %d)%n", this.GetType().FullName, path.GetPath(), lineNum);
            }
        }
        /// <exception cref="System.IO.IOException"/>
        private IList <string> ReadMockParams()
        {
            List <string>    ret    = new List <string>();
            LineNumberReader reader = new LineNumberReader(new FileReader(mockParamFile));
            string           line;

            while ((line = reader.ReadLine()) != null)
            {
                ret.AddItem(line);
            }
            reader.Close();
            return(ret);
        }
Esempio n. 10
0
        /**
         * /// Create a Lattice from a LAT file.  LAT files are created by the method Lattice.dump()
         *
         * /// @param fileName
         */
        public Lattice(String fileName)
        {
            try
            {
                Console.WriteLine(@"Loading from " + fileName);

                // load the nodes
                var    numberReader = new LineNumberReader(new StreamReader(fileName));
                string line;
                while ((line = numberReader.ReadLine()) != null)
                {
                    var tokens = new StringTokenizer(line);
                    if (tokens.hasMoreTokens())
                    {
                        var type = tokens.nextToken();

                        if (type.Equals("edge:"))
                        {
                            Edge.Load(this, tokens);
                        }
                        else if (type.Equals("node:"))
                        {
                            Node.Load(this, tokens);
                        }
                        else if (type.Equals("initialNode:"))
                        {
                            InitialNode = GetNode(tokens.nextToken());
                        }
                        else if (type.Equals("terminalNode:"))
                        {
                            TerminalNode = GetNode(tokens.nextToken());
                        }
                        else if (type.Equals("logBase:"))
                        {
                            LogBase = Double.Parse(tokens.nextToken(), CultureInfo.InvariantCulture.NumberFormat);
                        }
                        else
                        {
                            numberReader.Close();
                            throw new Exception("SYNTAX ERROR: " + fileName +
                                                '[' + numberReader.LineNumber + "] " + line);
                        }
                    }
                }
                numberReader.Close();
            }
            catch (Exception e)
            {
                throw new Exception(e.ToString());
            }
        }
Esempio n. 11
0
        public virtual void Parse()
        {
            int lineNum = 0;

            try
            {
                LineNumberReader reader           = new LineNumberReader(new FileReader(configFile));
                Properties       paramsForDataset = null;
                while (reader.Ready())
                {
                    string line = reader.ReadLine();
                    lineNum = reader.GetLineNumber();
                    //For exception handling
                    Matcher m = skipLine.Matcher(line);
                    if (m.LookingAt())
                    {
                        continue;
                    }
                    m = setDelim.Matcher(line);
                    if (m.Matches() && paramsForDataset != null)
                    {
                        datasetList.Add(paramsForDataset);
                        paramsForDataset = null;
                        continue;
                    }
                    else
                    {
                        if (paramsForDataset == null)
                        {
                            paramsForDataset = new Properties();
                        }
                    }
                    bool matched = false;
                    foreach (string param in patternsMap.Keys)
                    {
                        Pair <Pattern, Pattern> paramTemplate = patternsMap[param];
                        Matcher paramToken = paramTemplate.first.Matcher(line);
                        if (paramToken.LookingAt())
                        {
                            matched = true;
                            string[] tokens = line.Split(Delim);
                            if (tokens.Length != 2)
                            {
                                System.Console.Error.Printf("%s: Skipping malformed parameter in %s (line %d)%n", this.GetType().FullName, configFile, reader.GetLineNumber());
                                break;
                            }
                            string actualParam = tokens[0].Trim();
                            string paramValue  = tokens[1].Trim();
                            if (paramTemplate.second != null)
                            {
                                paramToken = paramTemplate.second.Matcher(paramValue);
                                if (paramToken.Matches())
                                {
                                    paramsForDataset.SetProperty(actualParam, paramValue);
                                }
                                else
                                {
                                    System.Console.Error.Printf("%s: Skipping illegal parameter value in %s (line %d)%n", this.GetType().FullName, configFile, reader.GetLineNumber());
                                    break;
                                }
                            }
                            else
                            {
                                paramsForDataset.SetProperty(actualParam, paramValue);
                            }
                        }
                    }
                    if (!matched)
                    {
                        string error = this.GetType().FullName + ": Unknown token in " + configFile + " (line " + reader.GetLineNumber() + ")%n";
                        System.Console.Error.Printf(error);
                        throw new ArgumentException(error);
                    }
                }
                if (paramsForDataset != null)
                {
                    datasetList.Add(paramsForDataset);
                }
                reader.Close();
            }
            catch (FileNotFoundException)
            {
                System.Console.Error.Printf("%s: Cannot open file %s%n", this.GetType().FullName, configFile);
            }
            catch (IOException)
            {
                System.Console.Error.Printf("%s: Error reading %s (line %d)%n", this.GetType().FullName, configFile, lineNum);
            }
        }
        protected override GrammarNode CreateGrammar()
        {
            // TODO: FlatLinguist requires the initial grammar node
            // to contain a single silence. We'll do that for now,
            // but once the FlatLinguist is fixed, this should be
            // returned to its former method of creating an empty
            // initial grammar node
            //   initialNode = createGrammarNode(initialID, false);

            InitialNode = null;
            FinalNode   = CreateGrammarNode(true);
            try
            {
                var    @in = new LineNumberReader(new StreamReader(RefFile));
                String line;
                while (true)
                {
                    line = @in.ReadLine();

                    if (line == null || line.IsEmpty())
                    {
                        break;
                    }

                    int uttNameStart = line.IndexOf('(') + 1;
                    int uttNameEnd   = line.IndexOf(')');

                    if (uttNameStart < 0 || uttNameStart > uttNameEnd)
                    {
                        continue;
                    }

                    String uttName    = line.JSubString(uttNameStart, uttNameEnd);
                    String transcript = line.JSubString(0, uttNameStart - 1).Trim();

                    if (transcript.IsEmpty())
                    {
                        continue;
                    }

                    InitialNode = CreateGrammarNode(IDictionary.SilenceSpelling);
                    CreateForcedAlignerGrammar(InitialNode, FinalNode, transcript);
                    Grammars.Put(uttName, InitialNode);
                    CurrentUttName = uttName;
                }
                @in.Close();
            }
            catch (FileNotFoundException e)
            {
                throw new Error(e);
            }
            catch (IOException e)
            {
                throw new Error(e);
            }
            catch (NoSuchMethodException e)
            {
                throw new Error(e);
            }
            return(InitialNode);
        }
        /// <summary>Read in typed dependencies.</summary>
        /// <remarks>
        /// Read in typed dependencies. Warning created typed dependencies are not
        /// backed by any sort of a tree structure.
        /// </remarks>
        /// <param name="filename"/>
        /// <exception cref="System.IO.IOException"/>
        protected internal static IList <ICollection <TypedDependency> > ReadDeps(string filename)
        {
            LineNumberReader breader = new LineNumberReader(new FileReader(filename));
            IList <ICollection <TypedDependency> > readDeps = new List <ICollection <TypedDependency> >();
            ICollection <TypedDependency>          deps     = new List <TypedDependency>();

            for (string line = breader.ReadLine(); line != null; line = breader.ReadLine())
            {
                if (line.Equals("null(-0,-0)") || line.Equals("null(-1,-1)"))
                {
                    readDeps.Add(deps);
                    deps = new List <TypedDependency>();
                    continue;
                }
                // relex parse error
                try
                {
                    if (line.Equals(string.Empty))
                    {
                        if (deps.Count != 0)
                        {
                            //System.out.println(deps);
                            readDeps.Add(deps);
                            deps = new List <TypedDependency>();
                        }
                        continue;
                    }
                    int    firstParen        = line.IndexOf("(");
                    int    commaSpace        = line.IndexOf(", ");
                    string depName           = Sharpen.Runtime.Substring(line, 0, firstParen);
                    string govName           = Sharpen.Runtime.Substring(line, firstParen + 1, commaSpace);
                    string childName         = Sharpen.Runtime.Substring(line, commaSpace + 2, line.Length - 1);
                    GrammaticalRelation grel = GrammaticalRelation.ValueOf(depName);
                    if (depName.StartsWith("prep_"))
                    {
                        string prep = Sharpen.Runtime.Substring(depName, 5);
                        grel = EnglishGrammaticalRelations.GetPrep(prep);
                    }
                    if (depName.StartsWith("prepc_"))
                    {
                        string prepc = Sharpen.Runtime.Substring(depName, 6);
                        grel = EnglishGrammaticalRelations.GetPrepC(prepc);
                    }
                    if (depName.StartsWith("conj_"))
                    {
                        string conj = Sharpen.Runtime.Substring(depName, 5);
                        grel = EnglishGrammaticalRelations.GetConj(conj);
                    }
                    if (grel == null)
                    {
                        throw new Exception("Unknown grammatical relation '" + depName + "'");
                    }
                    //Word govWord = new Word(govName.substring(0, govDash));
                    IndexedWord govWord = new IndexedWord();
                    govWord.SetValue(NormalizeNumbers(govName));
                    govWord.SetWord(govWord.Value());
                    //Word childWord = new Word(childName.substring(0, childDash));
                    IndexedWord childWord = new IndexedWord();
                    childWord.SetValue(NormalizeNumbers(childName));
                    childWord.SetWord(childWord.Value());
                    TypedDependency dep = new DependencyScoring.TypedDependencyStringEquality(grel, govWord, childWord);
                    deps.Add(dep);
                }
                catch (Exception e)
                {
                    breader.Close();
                    throw new Exception("Error on line " + breader.GetLineNumber() + ":\n\n" + e);
                }
            }
            if (deps.Count != 0)
            {
                readDeps.Add(deps);
            }
            //log.info("last: "+readDeps.get(readDeps.size()-1));
            breader.Close();
            return(readDeps);
        }
        public virtual void TestContainerLaunch()
        {
            string appSubmitter = "nobody";
            string appId        = "APP_ID";
            string containerId  = "CONTAINER_ID";
            string testImage    = "\"sequenceiq/hadoop-docker:2.4.1\"";

            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
                = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
                                            >(Org.Mockito.Mockito.ReturnsDeepStubs);
            ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(Org.Mockito.Mockito.ReturnsDeepStubs
                                                                     );
            ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
                                                 ();
            Dictionary <string, string> env = new Dictionary <string, string>();

            Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
            Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
            Org.Mockito.Mockito.When(cId.GetApplicationAttemptId().GetApplicationId().ToString
                                         ()).ThenReturn(appId);
            Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
            Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
            env[YarnConfiguration.NmDockerContainerExecutorImageName] = testImage;
            Path scriptPath = new Path("file:///bin/echo");
            Path tokensPath = new Path("file:///dev/null");
            Path pidFile    = new Path(workDir, "pid");

            dockerContainerExecutor.ActivateContainer(cId, pidFile);
            int ret = dockerContainerExecutor.LaunchContainer(container, scriptPath, tokensPath
                                                              , appSubmitter, appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs
                                                                  ());

            NUnit.Framework.Assert.AreEqual(0, ret);
            //get the script
            Path sessionScriptPath = new Path(workDir, Shell.AppendScriptExtension(DockerContainerExecutor
                                                                                   .DockerContainerExecutorSessionScript));
            LineNumberReader lnr = new LineNumberReader(new FileReader(sessionScriptPath.ToString
                                                                           ()));
            bool           cmdFound     = false;
            IList <string> localDirs    = DirsToMount(dirsHandler.GetLocalDirs());
            IList <string> logDirs      = DirsToMount(dirsHandler.GetLogDirs());
            IList <string> workDirMount = DirsToMount(Sharpen.Collections.SingletonList(workDir
                                                                                        .ToUri().GetPath()));
            IList <string> expectedCommands = new AList <string>(Arrays.AsList(DockerLaunchCommand
                                                                               , "run", "--rm", "--net=host", "--name", containerId));

            Sharpen.Collections.AddAll(expectedCommands, localDirs);
            Sharpen.Collections.AddAll(expectedCommands, logDirs);
            Sharpen.Collections.AddAll(expectedCommands, workDirMount);
            string shellScript = workDir + "/launch_container.sh";

            Sharpen.Collections.AddAll(expectedCommands, Arrays.AsList(testImage.ReplaceAll("['\"]"
                                                                                            , string.Empty), "bash", "\"" + shellScript + "\""));
            string expectedPidString = "echo `/bin/true inspect --format {{.State.Pid}} " + containerId
                                       + "` > " + pidFile.ToString() + ".tmp";
            bool pidSetterFound = false;

            while (lnr.Ready())
            {
                string line = lnr.ReadLine();
                Log.Debug("line: " + line);
                if (line.StartsWith(DockerLaunchCommand))
                {
                    IList <string> command = new AList <string>();
                    foreach (string s in line.Split("\\s+"))
                    {
                        command.AddItem(s.Trim());
                    }
                    NUnit.Framework.Assert.AreEqual(expectedCommands, command);
                    cmdFound = true;
                }
                else
                {
                    if (line.StartsWith("echo"))
                    {
                        NUnit.Framework.Assert.AreEqual(expectedPidString, line);
                        pidSetterFound = true;
                    }
                }
            }
            NUnit.Framework.Assert.IsTrue(cmdFound);
            NUnit.Framework.Assert.IsTrue(pidSetterFound);
        }
Esempio n. 15
0
        public static Lattice ReadSlf(String fileName)
        {
            var    lattice = new Lattice();
            var    @in     = new LineNumberReader(new StreamReader(fileName));
            String line;
            var    readingNodes = false;
            var    readingEdges = false;
            var    startIdx     = 0;
            var    endIdx       = 1;
            var    lmscale      = 9.5;

            while ((line = @in.ReadLine()) != null)
            {
                if (line.Contains("Node definitions"))
                {
                    readingEdges = false;
                    readingNodes = true;
                    continue;
                }
                if (line.Contains("Link definitions"))
                {
                    readingEdges = true;
                    readingNodes = false;
                    continue;
                }
                if (line.StartsWith("#"))
                {
                    //skip commented line
                    continue;
                }
                if (readingNodes)
                {
                    //reading node info, format:
                    //I=id   t=start_time_sec   W=word_transcription
                    var parts = line.Split("\\s+");
                    if (parts.Length != 3 || !parts[0].StartsWith("I=") || !parts[1].StartsWith("t=") || !parts[2].StartsWith("W="))
                    {
                        @in.Close();
                        throw new IOException("Unknown node definition: " + line);
                    }
                    var idx = Convert.ToInt32(parts[0].Substring(2), CultureInfo.InvariantCulture.NumberFormat);
                    //convert to milliseconds inplace
                    var beginTime = (long)(Convert.ToDouble(parts[1].Substring(2), CultureInfo.InvariantCulture.NumberFormat) * 1000);
                    var wordStr   = parts[2].Substring(2);
                    var isFiller  = false;
                    if (idx == startIdx || wordStr.Equals("!ENTER"))
                    {
                        wordStr  = "<s>";
                        isFiller = true;
                    }
                    if (idx == endIdx || wordStr.Equals("!EXIT"))
                    {
                        wordStr  = "</s>";
                        isFiller = true;
                    }
                    if (wordStr.Equals("!NULL"))
                    {
                        wordStr  = "<sil>";
                        isFiller = true;
                    }
                    if (wordStr.StartsWith("["))
                    {
                        isFiller = true;
                    }
                    var word = new Word(wordStr, new Pronunciation[0], isFiller);
                    var node = lattice.AddNode(idx.ToString(CultureInfo.InvariantCulture), word, beginTime, -1);
                    if (wordStr.Equals("<s>"))
                    {
                        lattice.InitialNode = node;
                    }
                    if (wordStr.Equals("</s>"))
                    {
                        lattice.TerminalNode = node;
                    }
                }
                else if (readingEdges)
                {
                    //reading edge info, format:
                    //J=id   S=from_node   E=to_node   a=acoustic_score   l=language_score
                    var parts = line.Split("\\s+");
                    if (parts.Length != 5 || !parts[1].StartsWith("S=") || !parts[2].StartsWith("E=") ||
                        !parts[3].StartsWith("a=") || !parts[4].StartsWith("l="))
                    {
                        @in.Close();
                        throw new IOException("Unknown edge definition: " + line);
                    }
                    var fromId = parts[1].Substring(2);
                    var toId   = parts[2].Substring(2);
                    var ascore = Convert.ToDouble(parts[3].Substring(2), CultureInfo.InvariantCulture.NumberFormat);
                    var lscore = Convert.ToDouble(parts[4].Substring(2), CultureInfo.InvariantCulture.NumberFormat) * lmscale;
                    lattice.AddEdge(lattice.Nodes.Get(fromId), lattice.Nodes.Get(toId), ascore, lscore);
                }
                else
                {
                    //reading header here if needed
                    if (line.StartsWith("start="))
                    {
                        startIdx = Convert.ToInt32(line.Replace("start=", ""), CultureInfo.InvariantCulture.NumberFormat);
                    }
                    if (line.StartsWith("end="))
                    {
                        endIdx = Convert.ToInt32(line.Replace("end=", ""), CultureInfo.InvariantCulture.NumberFormat);
                    }
                    if (line.StartsWith("lmscale="))
                    {
                        lmscale = Convert.ToDouble(line.Replace("lmscale=", ""), CultureInfo.InvariantCulture.NumberFormat);
                    }
                }
            }
            foreach (var node in lattice.Nodes.Values)
            {
                //calculate end time of nodes depending successors begin time
                foreach (var edge in node.LeavingEdges)
                {
                    if (node.EndTime < 0 || node.EndTime > edge.ToNode.BeginTime)
                    {
                        node.EndTime = edge.ToNode.BeginTime;
                    }
                }
            }
            @in.Close();
            return(lattice);
        }