コード例 #1
0
        private static SentenceDetectorME PrepareSentenceDetector()
        {
            var sentModelStream = new FileInputStream(SentenceModelPath); //load the sentence model into a stream
            var sentModel       = new SentenceModel(sentModelStream);     // load the model

            return(new SentenceDetectorME(sentModel));                    //create sentence detector
        }
コード例 #2
0
 public void TestWithOpenNLPModel()
 {
     using (var file = Tests.OpenFile("/opennlp/models/en-sent.bin")) {
         var model = new SentenceModel(file);
         EvalSentences(new SentenceDetectorME(model));
     }
 }
コード例 #3
0
        public void TestNullDict()
        {
            char[] eos     = { '.', '?' };
            var    sdModel = Train(new SentenceDetectorFactory("en", true, null, eos));

            Assert.NotNull(sdModel);

            SentenceDetectorFactory factory = sdModel.Factory;

            Assert.Null(factory.AbbreviationDictionary);
            Assert.True(factory.GetContextGenerator() is DefaultSentenceContextGenerator);
            Assert.True(factory.GetEndOfSentenceScanner() is DefaultEndOfSentenceScanner);
            Assert.True(eos.SequenceEqual(factory.EOSCharacters));

            var o = new MemoryStream();

            sdModel.Serialize(new UnclosableStream(o));

            o.Seek(0, SeekOrigin.Begin);

            var fromSerialized = new SentenceModel(o);

            factory = fromSerialized.Factory;
            Assert.Null(factory.AbbreviationDictionary);
            Assert.True(factory.GetContextGenerator() is DefaultSentenceContextGenerator);
            Assert.True(factory.GetEndOfSentenceScanner() is DefaultEndOfSentenceScanner);
            Assert.True(eos.SequenceEqual(factory.EOSCharacters));
        }
コード例 #4
0
        static Sentenizer()
        {
            var modelFile = HttpContext.Current.Server.MapPath("~/Files/TextAnalytics/en-sentence-model-dragon.bin");

            ModelIn = new java.io.FileInputStream(modelFile);
            Model   = new SentenceModel(ModelIn);
        }
コード例 #5
0
        static Sentenizer()
        {
            var modelFile = ConfigurationManager.AppSettings["ModelSentenizer"] ?? string.Empty;

            ModelIn = new java.io.FileInputStream(modelFile);
            Model   = new SentenceModel(ModelIn);
        }
コード例 #6
0
        static void Main(string[] args)
        {
            String[]     file = Directory.GetFiles(@"..\..\..\", "*.txt");
            StreamWriter sw   = new StreamWriter(@"..\..\..\output\Html.txt");

            foreach (String files in file)
            {
                using (StreamReader sr = new StreamReader(files))
                {
                    while (sr.Peek() != -1)
                    {
                        string line = sr.ReadLine();
                        int    i;
                        java.io.InputStream modelin  = new java.io.FileInputStream(string.Format(@"{0}\en-sent.bin", @"..\Debug"));
                        SentenceModel       model    = new SentenceModel(modelin);
                        SentenceDetector    detector = new SentenceDetectorME(model);
                        string[]            sents    = detector.sentDetect(line);
                        foreach (var sent in sents)
                        {
                            sw.WriteLine(sent);
                        }
                    }
                }
                sw.Flush();
            }
            sw.Close();
        }
コード例 #7
0
        static Sentenizer()
        {
            // var modelFile = ConfigurationManager.AppSettings["ModelSentenizer"] ?? string.Empty;

            var modelFile = AppDomain.CurrentDomain.BaseDirectory + "Repository\\en-sent.bin";

            ModelIn = new FileInputStream(modelFile);
            Model   = new SentenceModel(ModelIn);
        }
コード例 #8
0
        public IActionResult ParsedSentence(SentenceModel model)
        {
            try
            {
                string parsedSentence = string.Empty;
                string word           = string.Empty;

                //If user didn't input string, simply return "0" for no distinct characters and no start/end letters"
                if (model.OriginalSentence == null || model.OriginalSentence?.Length == 0)
                {
                    model.ParsedSentence = "0";
                    return(View(model));
                }

                //Iterate through the sentence, if the character is a space or non-alphabetic character, then this is a separator.
                //Separators are simply appended to the result to maintain their original form and location.
                //For each letter, we start a new word, until we reach another separator, then we process that word and append that word to the result.
                for (int i = 0; i < model.OriginalSentence.Length; i++)
                {
                    //If character is alphabetic
                    if (char.IsLetter(model.OriginalSentence[i]))
                    {
                        //Add the letter to the current word.
                        word += model.OriginalSentence[i];
                        //If this is the last character in the sentence and we still have a word to parse
                        if (i == model.OriginalSentence.Length - 1 && word != "")
                        {
                            //Append the final parsed word to the result.
                            parsedSentence += ParsedWord(word);
                        }
                    }
                    //This is a separator (space or non-alphabetic character), but have to append the current word to the result.
                    else if (word != "")
                    {
                        //Add the parsed word and the next separator.
                        parsedSentence += ParsedWord(word) + model.OriginalSentence[i];
                        word            = "";
                    }
                    //This is a separator (space or non-alphabetic character), append it to the result.
                    else
                    {
                        parsedSentence += model.OriginalSentence[i];
                        word            = "";
                    }
                }
                model.ParsedSentence = parsedSentence;

                return(View(model));
            }
            catch (Exception)
            {
                return(View(new ErrorViewModel {
                    RequestId = Activity.Current?.Id ?? HttpContext.TraceIdentifier
                }));
            }
        }
コード例 #9
0
        public void ParsingTest_Empty()
        {
            var controller = new SentenceController();
            var model      = new SentenceModel
            {
                OriginalSentence = string.Empty
            };
            var result = controller.ParsedSentence(model) as ViewResult;

            Assert.AreEqual("0", model.ParsedSentence);
        }
コード例 #10
0
        public void ParsingTest_RemoveNonAlphabetic()
        {
            var controller = new SentenceController();
            var model      = new SentenceModel
            {
                OriginalSentence = "1~@!$%1This1()*{}[]1is1~`&#&*&^%1a1:;?><9TEST!!!!"
            };
            var result = controller.ParsedSentence(model) as ViewResult;

            Assert.AreEqual("1~@!$%1T2s1()*{}[]1i0s1~`&#&*&^%1a0a1:;?><9T2T!!!!", model.ParsedSentence);
        }
コード例 #11
0
        public void ParsingTest_1234()
        {
            var controller = new SentenceController();
            var model      = new SentenceModel
            {
                OriginalSentence = "1a1_2bc2_3def3_4ghji4"
            };
            var result = controller.ParsedSentence(model) as ViewResult;

            Assert.AreEqual("1a0a1_2b0c2_3d1f3_4g2i4", model.ParsedSentence);
        }
コード例 #12
0
        public void ParsingTest_ABCDE()
        {
            var controller = new SentenceController();
            var model      = new SentenceModel
            {
                OriginalSentence = "Aa$BBb_CCCc@DdDDd^eEeEEE"
            };
            var result = controller.ParsedSentence(model) as ViewResult;

            Assert.AreEqual("A0a$B1b_C1c@D2d^e2E", model.ParsedSentence);
        }
コード例 #13
0
        public void ParsingTest_Smooth()
        {
            var controller = new SentenceController();
            var model      = new SentenceModel
            {
                OriginalSentence = "Smooth"
            };
            var result = controller.ParsedSentence(model) as ViewResult;

            Assert.AreEqual("S3h", model.ParsedSentence);
        }
コード例 #14
0
        private void LoadSentenceDetector()
        {
            if (!alreadyLoadSentenceDetector)
            {
                java.io.FileInputStream modelInpStream = new java.io.FileInputStream("Resources\\en-sent.bin");
                SentenceModel           sentenceModel  = new SentenceModel(modelInpStream);
                sentenceDetector = new SentenceDetectorME(sentenceModel);

                alreadyLoadSentenceDetector = true;
            }
        }
コード例 #15
0
        static void Main(string[] args)
        {
            java.io.InputStream model_src = new java.io.FileInputStream(@"..\..\..\..\en-sent.bin");
            SentenceModel       smodel    = new SentenceModel(model_src);

            detector = new SentenceDetectorME(smodel);

            foreach (String src in Directory.GetFiles(@"..\..\..\..\Dataset"))
            {
                lab5(src);
            }
        }
コード例 #16
0
 public static SentenceModel GetSentenceModel(string modelName, IResourceLoader loader)
 {
     if (!sentenceModels.TryGetValue(modelName, out SentenceModel model) || model == null)
     {
         using (Stream resource = loader.OpenResource(modelName))
         {
             model = new SentenceModel(new ikvm.io.InputStreamWrapper(resource));
         }
         sentenceModels[modelName] = model;
     }
     return(model);
 }
コード例 #17
0
        public Span[] SentPosDetect(string paragraph)
        {
            var                bin       = GetFileStream("en-sent.bin");
            SentenceModel      model     = new SentenceModel(bin);
            SentenceDetectorME sdetector = new SentenceDetectorME(model);

            Span[] sentences = sdetector.sentPosDetect(paragraph);

            bin.close();

            return(sentences);
        }
コード例 #18
0
ファイル: Training.cs プロジェクト: Yucked/Witty
        /// <summary>Returns the meaning of a sentence.</summary>
        /// <param name="sentence"><see cref="SentenceModel"/></param>
        /// <exception cref="ArgumentNullException"><param name="sentence"/> was <c>null</c></exception>
        public async Task <MeaningObject> SentenceMeaningAsync(SentenceModel sentence)
        {
            if (sentence == null)
            {
                throw new ArgumentNullException(nameof(sentence), "Sentence is a required argument.");
            }
            var get = await RestClient.GetAsync(
                $"message?q={sentence.Message}&context={JsonConvert.SerializeObject(sentence.Context ?? DefaultContext)}" +
                $"&msg_id={SnowFlake}&thread_id={SnowFlake}&n={sentence.MaxTraits}&verbose={sentence.Verbose}")
                      .ConfigureAwait(false);

            return(await ProcessAsync <MeaningObject>(get));
        }
コード例 #19
0
        public IActionResult CreateSentence([FromBody] SentenceModel sentence)
        {
            List <string> words = new List <string>();

            words = sentence.ReversedSentence.Split(" ").ToList();
            words.Reverse();
            sentence.ReversedSentence = "";
            foreach (var word in words)
            {
                sentence.ReversedSentence += word + " ";
            }
            _repository.Add(sentence);
            return(Created("api/Sentence/posts", sentence));
        }
コード例 #20
0
        static void Sent(string src)
        {
            StreamReader input  = new StreamReader(src);
            StreamWriter output = new StreamWriter(Regex.Replace(src, "(.*).txt", "$1_sent.txt"));

            while (input.Peek() != -1)
            {
                InputStream      modelIn  = new FileInputStream(@"..\..\..\en-sent.bin");
                SentenceModel    smodel   = new SentenceModel(modelIn);
                SentenceDetector detector = new SentenceDetectorME(smodel);
                string[]         sents    = detector.sentDetect(input.ReadLine());
                foreach (string sent in sents)
                {
                    output.WriteLine(sent);
                }
            }
        }
コード例 #21
0
        public override void run(string format, string[] args)
        {
            base.run(format, args);

            SentenceModel model = (new SentenceModelLoader()).load(@params.Model);

            SentenceDetectorEvaluationMonitor errorListener = null;

            if (@params.Misclassified.Value)
            {
                errorListener = new SentenceEvaluationErrorListener();
            }

            SentenceDetectorEvaluator evaluator = new SentenceDetectorEvaluator(new SentenceDetectorME(model), errorListener);

            Console.Write("Evaluating ... ");
            try
            {
                evaluator.evaluate(sampleStream);
            }
            catch (IOException e)
            {
                throw new TerminateToolException(-1, "IO error while reading training data or indexing data: " + e.Message, e);
            }
            finally
            {
                try
                {
                    sampleStream.close();
                }
                catch (IOException)
                {
                    // sorry that this can fail
                }
            }

            Console.Error.WriteLine("done");

            Console.WriteLine();

            Console.WriteLine(evaluator.FMeasure);
        }
コード例 #22
0
        public static void Main(string[] args)
        {
            DirectoryInfo folder = new DirectoryInfo(@"..\..\..\..\..\Dataset");

            foreach (var fname in folder.GetFiles())
            {
                String line = File.ReadAllText(fname.FullName);
                java.io.InputStream modelIn  = new java.io.FileInputStream(@"..\..\..\..\..\en-sent.bin");
                SentenceModel       smodel   = new SentenceModel(modelIn);
                SentenceDetector    detector = new SentenceDetectorME(smodel);
                string[]            sents    = detector.sentDetect(line);
                using (StreamWriter sw = new StreamWriter(fname.FullName.Replace(fname.FullName.Substring(fname.FullName.Length - 3), "rtf")))
                {
                    foreach (var sent in sents)
                    {
                        sw.WriteLine(sent);
                    }
                }
            }
        }
コード例 #23
0
ファイル: OpenNLP.cs プロジェクト: baio/d-mill
        public void SplitSentences()
        {
            var txt = File.ReadAllText(@"c:\dev\d-mill\uspe\Data\uspe-1.txt");

            txt = Regex.Replace(txt, "\\s+", " ");

            txt = Regex.Replace(txt, "\\r\\n", "");

            txt = Regex.Replace(txt, "MR.\\s+", "MR.");

            var modelStream = new java.io.FileInputStream("../../Models/en-sent.bin");

            var model = new SentenceModel(modelStream);

            var detector = new SentenceDetectorME(model);

            var sentences = detector.sentDetect(txt);

            File.WriteAllLines(@"c:\dev\d-mill\uspe\Data\uspe-sentenced.txt", sentences);
        }
コード例 #24
0
        /// <summary>
        /// Perform sentence detection the input stream.
        ///
        /// A newline will be treated as a paragraph boundary.
        /// </summary>
        public override void run(string[] args)
        {
            if (args.Length != 1)
            {
                Console.WriteLine(Help);
            }
            else
            {
                SentenceModel model = (new SentenceModelLoader()).load(new File(args[0]));

                SentenceDetectorME sdetector = new SentenceDetectorME(model);

                ObjectStream <string> paraStream = new ParagraphStream(new PlainTextByLineStream(new InputStreamReader(Console.OpenStandardInput)));

                PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "sent");
                perfMon.start();

                try
                {
                    string para;
                    while ((para = paraStream.read()) != null)
                    {
                        string[] sents = sdetector.sentDetect(para);
                        foreach (string sentence in sents)
                        {
                            Console.WriteLine(sentence);
                        }

                        perfMon.incrementCounter(sents.Length);

                        Console.WriteLine();
                    }
                }
                catch (IOException e)
                {
                    CmdLineUtil.handleStdinIoError(e);
                }

                perfMon.stopAndPrintFinalResult();
            }
        }
コード例 #25
0
        public NLP()
        {
            //loading sentence detector model
            java.io.FileInputStream modelInpStream = new java.io.FileInputStream("Resources\\en-sent.bin");
            SentenceModel           sentenceModel  = new SentenceModel(modelInpStream);

            sentenceDetector = new SentenceDetectorME(sentenceModel);

            //loading tokenizer model
            modelInpStream = new java.io.FileInputStream("Resources\\en-token.bin");
            TokenizerModel tokenizerModel = new TokenizerModel(modelInpStream);

            tokenizer = new TokenizerME(tokenizerModel);

            modelInpStream = new java.io.FileInputStream("Resources\\en-pos-maxent.bin");
            POSModel posModel = new POSModel(modelInpStream);

            tagger = new POSTaggerME(posModel);

            modelInpStream = new java.io.FileInputStream("Resources\\en-chunker.bin");
            ChunkerModel chunkerModel = new ChunkerModel(modelInpStream);

            chunker = new ChunkerME(chunkerModel);

            modelInpStream = new java.io.FileInputStream("Resources\\en-parser-chunking.bin");
            ParserModel parserModel = new ParserModel(modelInpStream);

            parser = ParserFactory.create(parserModel);

            //loading stop words list
            StreamReader sr = new StreamReader("Resources\\english.stop.txt");
            string       line;

            while ((line = sr.ReadLine()) != null)
            {
                stopwords.Add(Stemming(line));
                stopwords.Add(line);
            }
        }
コード例 #26
0
        public async Task <ActionResult> Post([FromBody] SentenceModel value)
        {
            var client = _httpClientFactory.CreateClient("sa-logic");

            var settings = new JsonSerializerSettings
            {
                ContractResolver = new CamelCasePropertyNamesContractResolver()
            };
            var camelCaseLoad = JsonConvert.SerializeObject(value, settings);
            var content       = new StringContent(camelCaseLoad, Encoding.UTF8, "application/json");

            var result = await client.PostAsync("/analyse/sentiment", content);

            if (!result.IsSuccessStatusCode)
            {
                return(BadRequest(value));
            }

            var sentiment = await result.Content.ReadAsAsync <SentimentModel>();

            return(Ok(sentiment));
        }
コード例 #27
0
        static void Main(string[] args)
        {
            StreamWriter sw = new StreamWriter(@"..\..\Data\result.txt");
            StreamReader sr = new StreamReader(@"..\..\Data\data.txt");

            while (sr.Peek() != -1)
            {
                string line = sr.ReadLine();
                java.io.InputStream modelIn  = new java.io.FileInputStream("en-sent.bin");
                SentenceModel       smodel   = new SentenceModel(modelIn);
                SentenceDetector    detector = new SentenceDetectorME(smodel);
                string[]            sents    = detector.sentDetect(line);
                foreach (var sent in sents)
                {
                    sw.WriteLine(sent);
                    sw.WriteLine();
                }
                sw.Flush();
            }
            sr.Close();
            sw.Close();
        }
コード例 #28
0
        static void Main(string[] args)
        {
            java.io.InputStream modelIn  = new java.io.FileInputStream(string.Format("en-sent.bin"));
            java.io.InputStream modelIn2 = new java.io.FileInputStream(string.Format("en-token.bin"));
            TokenizerModel      model    = new TokenizerModel(modelIn2);
            TokenizerME         mE       = new TokenizerME(model);
            SentenceModel       sM       = new SentenceModel(modelIn);
            SentenceDetector    detector = new SentenceDetectorME(sM);
            string folderName            = @"C:\Users\Administrator\Desktop\lab-6-opennlp-ju-zi-qie-fen-10411174\file";

            foreach (string fname in System.IO.Directory.GetFiles(folderName))
            {
                String       line  = null;
                String[]     name  = fname.Split('\\');
                StreamWriter sw    = new StreamWriter(@"C:\Users\Administrator\Desktop\lab-6-opennlp-ju-zi-qie-fen-10411174\answer\" + name[6]);
                StreamReader file2 = new StreamReader(fname);
                while ((line = file2.ReadLine()) != null)
                {
                    string   str   = null;
                    string[] sents = detector.sentDetect(line);
                    if (sents.Length.Equals(0))
                    {
                        continue;
                    }
                    foreach (var s in sents)
                    {
                        str = str + s;
                    }
                    var Tokens = mE.tokenize(str);
                    foreach (var s in Tokens)
                    {
                        sw.Write(s + " ");
                    }
                    sw.WriteLine();
                }
                sw.Close();
            }
        }
コード例 #29
0
        public static void ConvertToJson(string textFile, string word)
        {
            List <SentenceModel> sentenceModelsList = new List <SentenceModel>();

            foreach (var sentece in FindWord(textFile, word))
            {
                SentenceModel sentenceModel = new SentenceModel()
                {
                    Word     = word,
                    Sentence = sentece,
                    Number   = Numbers(sentece, word)
                };
                sentenceModelsList.Add(sentenceModel);
            }
            string add  = File.ReadAllText(HttpContext.Current.Server.MapPath("Data/jsonFile.json"));
            var    back = JsonConvert.DeserializeObject <List <SentenceModel> >(add);

            sentenceModelsList.AddRange(back);

            var json = JsonConvert.SerializeObject(sentenceModelsList);

            File.WriteAllText(HttpContext.Current.Server.MapPath("Data/jsonFile.json"), json);
        }
コード例 #30
0
 private static SentenceDetectorME PrepareSentenceDetector()
 {
     var sentModelStream = new FileInputStream(SentenceModelPath); //load the sentence model into a stream
     var sentModel = new SentenceModel(sentModelStream); // load the model
     return new SentenceDetectorME(sentModel); //create sentence detector
 }
コード例 #31
0
ファイル: Sentenizer.cs プロジェクト: MALLOCkol/Dragon
 static Sentenizer()
 {
     var modelFile = ConfigurationManager.AppSettings["ModelSentenizer"] ?? string.Empty;
     ModelIn = new java.io.FileInputStream(modelFile);
     Model = new SentenceModel(ModelIn);
 }
コード例 #32
0
ファイル: Menu.cs プロジェクト: JJColeman/jcoleman_Capstone
        public String chooseSentenceMenu()
        {
            int userInputNumber = 0;
            string userChoosenSentence = "";
            System.Console.WriteLine("Choose a sentence to use from your current text");
            System.Console.WriteLine("Must be a space between each sentence");
            try
            {
                java.io.File file = new java.io.File("C:\\en-sent.bin");
                java.io.InputStream modelIn = new FileInputStream("C:\\Users\\jcoleman\\Documents\\Capstone\\jcoleman_Capstone\\Code\\NEWCHATBOT\\ConsoleBot\\ConsoleBot\\en-sent.bin");
                SentenceModel model = new SentenceModel(modelIn);
                SentenceDetectorME sentenceDetector = new SentenceDetectorME(model);

                string text = "";
                FileText = System.IO.File.ReadAllLines(FilePath);

                for (int i = 0; i < FileText.Length; i++)
                {
                    text += FileText[i];
                }

                string[] sentences = sentenceDetector.sentDetect(text);

                for(int s = 0;s < sentences.Length;s++)
                {
                    System.Console.WriteLine((s+1) +" : " +sentences[s]);
                }

                string userInput = System.Console.ReadLine();
                userInputNumber = int.Parse(userInput);

                userChoosenSentence = sentences[userInputNumber - 1];
                modelIn.close();
            }
            catch(Exception e)
            {
                System.Console.WriteLine(e.Message);
            }

            return userChoosenSentence;
        }
コード例 #33
0
 private SentenceDetectorME PrepareSentenceDetector()
 {
     //TODO[danielcamargo]: we need to find/train the model in spanish
     var sentModelStream =
         new FileInputStream(string.Format(@"Models\{0}-sent.bin", _language == "es" ? "pt" : _language));
     var sentModel = new SentenceModel(sentModelStream);
     sentModelStream.close();
     return new SentenceDetectorME(sentModel);
 }
コード例 #34
0
 public SentenceDetector(SentenceModel model)
 {
     this.detector = new SentenceDetectorME(model);
 }
コード例 #35
0
        public SentenceDetector(FileStream modelStream)
        {
            SentenceModel model = new SentenceModel(modelStream);

            this.detector = new SentenceDetectorME(model);
        }
コード例 #36
0
ファイル: TextProcessor.cs プロジェクト: baio/d-mill
        public static string[] SplitSentences(string Text)
        {
            var modelStream = new java.io.ByteArrayInputStream(Resource.en_sent);

            var model = new SentenceModel(modelStream);

            var detector = new SentenceDetectorME(model);

            return detector.sentDetect(Text);
        }