Exemplo n.º 1
0
        //Output raw result with probability
        private void OutputRawResultToFile(List <List <string> > inbuf, crf_term_out[] crf_out, SegDecoderTagger tagger, StreamWriter sw)
        {
            for (var k = 0; k < crf_out.Length; k++)
            {
                if (crf_out[k] == null)
                {
                    //No more result
                    break;
                }

                var sb = new StringBuilder();

                var crf_seg_out = crf_out[k];
                //Show the entire sequence probability
                //For each token
                for (var i = 0; i < inbuf.Count; i++)
                {
                    //Show all features
                    for (var j = 0; j < inbuf[i].Count; j++)
                    {
                        sb.Append(inbuf[i][j]);
                        sb.Append("\t");
                    }

                    //Show the best result and its probability
                    sb.Append(crf_seg_out.result_[i]);

                    if (tagger.vlevel_ > 1)
                    {
                        sb.Append("\t");
                        sb.Append(crf_seg_out.weight_[i]);

                        //Show the probability of all tags
                        sb.Append("\t");
                        for (var j = 0; j < tagger.ysize_; j++)
                        {
                            sb.Append(tagger.yname(j));
                            sb.Append("/");
                            sb.Append(tagger.prob(i, j));

                            if (j < tagger.ysize_ - 1)
                            {
                                sb.Append("\t");
                            }
                        }
                    }
                    sb.AppendLine();
                }
                if (tagger.vlevel_ > 0)
                {
                    sw.WriteLine("#{0}", crf_seg_out.prob);
                }
                sw.WriteLine(sb.ToString().Trim());
                sw.WriteLine();
            }
        }
Exemplo n.º 2
0
        bool Decode(CRFSharpWrapper.DecoderArgs options)
        {
            ParallelOptions parallelOption = new ParallelOptions();

            if (File.Exists(options.strInputFileName) == false)
            {
                Console.WriteLine("FAILED: Open {0} file failed.", options.strInputFileName);
                return(false);
            }

            if (File.Exists(options.strModelFileName) == false)
            {
                Console.WriteLine("FAILED: Open {0} file failed.", options.strModelFileName);
                return(false);
            }

            StreamReader sr = new StreamReader(options.strInputFileName);
            StreamWriter sw = null, swSeg = null;

            if (options.strOutputFileName != null && options.strOutputFileName.Length > 0)
            {
                sw = new StreamWriter(options.strOutputFileName);
            }
            if (options.strOutputSegFileName != null && options.strOutputSegFileName.Length > 0)
            {
                swSeg = new StreamWriter(options.strOutputSegFileName);
            }

            //Create CRFSharp wrapper instance. It's a global instance
            CRFSharpWrapper.Decoder crfWrapper = new CRFSharpWrapper.Decoder();
            //Load model from file
            if (crfWrapper.LoadModel(options.strModelFileName) == false)
            {
                return(false);
            }

            ConcurrentQueue <List <List <string> > > queueRecords = new ConcurrentQueue <List <List <string> > >();
            ConcurrentQueue <List <List <string> > > queueSegRecords = new ConcurrentQueue <List <List <string> > >();

            parallelOption.MaxDegreeOfParallelism = options.thread;
            Parallel.For(0, options.thread, parallelOption, t =>
            {
                //Create decoder tagger instance. If the running environment is multi-threads, each thread needs a separated instance
                SegDecoderTagger tagger = crfWrapper.CreateTagger(options.nBest, options.maxword);
                tagger.set_vlevel(options.probLevel);

                //Initialize result
                crf_seg_out[] crf_out = new crf_seg_out[options.nBest];
                for (int i = 0; i < options.nBest; i++)
                {
                    crf_out[i] = new crf_seg_out(tagger.crf_max_word_num);
                }

                List <List <string> > inbuf = new List <List <string> >();
                while (true)
                {
                    lock (rdLocker)
                    {
                        if (ReadRecord(inbuf, sr) == false)
                        {
                            break;
                        }

                        queueRecords.Enqueue(inbuf);
                        queueSegRecords.Enqueue(inbuf);
                    }

                    //Call CRFSharp wrapper to predict given string's tags
                    if (swSeg != null)
                    {
                        crfWrapper.Segment(crf_out, tagger, inbuf);
                    }
                    else
                    {
                        crfWrapper.Segment((crf_term_out[])crf_out, (DecoderTagger)tagger, inbuf);
                    }

                    List <List <string> > peek = null;
                    //Save segmented tagged result into file
                    if (swSeg != null)
                    {
                        List <string> rstList = ConvertCRFTermOutToStringList(inbuf, crf_out);
                        while (peek != inbuf)
                        {
                            queueSegRecords.TryPeek(out peek);
                        }
                        foreach (string item in rstList)
                        {
                            swSeg.WriteLine(item);
                        }
                        queueSegRecords.TryDequeue(out peek);
                        peek = null;
                    }

                    //Save raw tagged result (with probability) into file
                    if (sw != null)
                    {
                        while (peek != inbuf)
                        {
                            queueRecords.TryPeek(out peek);
                        }
                        OutputRawResultToFile(inbuf, crf_out, tagger, sw);
                        queueRecords.TryDequeue(out peek);
                    }
                }
            });


            sr.Close();

            if (sw != null)
            {
                sw.Close();
            }
            if (swSeg != null)
            {
                swSeg.Close();
            }

            return(true);
        }