private static SentenceDetectorME PrepareSentenceDetector()
{
var sentModelStream = new FileInputStream(SentenceModelPath); //load the sentence model into a stream
var sentModel = new SentenceModel(sentModelStream); // load the model
return(new SentenceDetectorME(sentModel)); //create sentence detector
}
public void TestWithOpenNLPModel()
{
using (var file = Tests.OpenFile("/opennlp/models/en-sent.bin")) {
var model = new SentenceModel(file);
EvalSentences(new SentenceDetectorME(model));
}
}
public void TestNullDict()
{
char[] eos = { '.', '?' };
var sdModel = Train(new SentenceDetectorFactory("en", true, null, eos));
Assert.NotNull(sdModel);
SentenceDetectorFactory factory = sdModel.Factory;
Assert.Null(factory.AbbreviationDictionary);
Assert.True(factory.GetContextGenerator() is DefaultSentenceContextGenerator);
Assert.True(factory.GetEndOfSentenceScanner() is DefaultEndOfSentenceScanner);
Assert.True(eos.SequenceEqual(factory.EOSCharacters));
var o = new MemoryStream();
sdModel.Serialize(new UnclosableStream(o));
o.Seek(0, SeekOrigin.Begin);
var fromSerialized = new SentenceModel(o);
factory = fromSerialized.Factory;
Assert.Null(factory.AbbreviationDictionary);
Assert.True(factory.GetContextGenerator() is DefaultSentenceContextGenerator);
Assert.True(factory.GetEndOfSentenceScanner() is DefaultEndOfSentenceScanner);
Assert.True(eos.SequenceEqual(factory.EOSCharacters));
}
static Sentenizer()
{
var modelFile = HttpContext.Current.Server.MapPath("~/Files/TextAnalytics/en-sentence-model-dragon.bin");
ModelIn = new java.io.FileInputStream(modelFile);
Model = new SentenceModel(ModelIn);
}
static Sentenizer()
{
var modelFile = ConfigurationManager.AppSettings["ModelSentenizer"] ?? string.Empty;
ModelIn = new java.io.FileInputStream(modelFile);
Model = new SentenceModel(ModelIn);
}
static void Main(string[] args)
{
String[] file = Directory.GetFiles(@"..\..\..\", "*.txt");
StreamWriter sw = new StreamWriter(@"..\..\..\output\Html.txt");
foreach (String files in file)
{
using (StreamReader sr = new StreamReader(files))
{
while (sr.Peek() != -1)
{
string line = sr.ReadLine();
int i;
java.io.InputStream modelin = new java.io.FileInputStream(string.Format(@"{0}\en-sent.bin", @"..\Debug"));
SentenceModel model = new SentenceModel(modelin);
SentenceDetector detector = new SentenceDetectorME(model);
string[] sents = detector.sentDetect(line);
foreach (var sent in sents)
{
sw.WriteLine(sent);
}
}
}
sw.Flush();
}
sw.Close();
}
static Sentenizer()
{
// var modelFile = ConfigurationManager.AppSettings["ModelSentenizer"] ?? string.Empty;
var modelFile = AppDomain.CurrentDomain.BaseDirectory + "Repository\\en-sent.bin";
ModelIn = new FileInputStream(modelFile);
Model = new SentenceModel(ModelIn);
}
public IActionResult ParsedSentence(SentenceModel model)
{
try
{
string parsedSentence = string.Empty;
string word = string.Empty;
//If user didn't input string, simply return "0" for no distinct characters and no start/end letters"
if (model.OriginalSentence == null || model.OriginalSentence?.Length == 0)
{
model.ParsedSentence = "0";
return(View(model));
}
//Iterate through the sentence, if the character is a space or non-alphabetic character, then this is a separator.
//Separators are simply appended to the result to maintain their original form and location.
//For each letter, we start a new word, until we reach another separator, then we process that word and append that word to the result.
for (int i = 0; i < model.OriginalSentence.Length; i++)
{
//If character is alphabetic
if (char.IsLetter(model.OriginalSentence[i]))
{
//Add the letter to the current word.
word += model.OriginalSentence[i];
//If this is the last character in the sentence and we still have a word to parse
if (i == model.OriginalSentence.Length - 1 && word != "")
{
//Append the final parsed word to the result.
parsedSentence += ParsedWord(word);
}
}
//This is a separator (space or non-alphabetic character), but have to append the current word to the result.
else if (word != "")
{
//Add the parsed word and the next separator.
parsedSentence += ParsedWord(word) + model.OriginalSentence[i];
word = "";
}
//This is a separator (space or non-alphabetic character), append it to the result.
else
{
parsedSentence += model.OriginalSentence[i];
word = "";
}
}
model.ParsedSentence = parsedSentence;
return(View(model));
}
catch (Exception)
{
return(View(new ErrorViewModel {
RequestId = Activity.Current?.Id ?? HttpContext.TraceIdentifier
}));
}
}
public void ParsingTest_Empty()
{
var controller = new SentenceController();
var model = new SentenceModel
{
OriginalSentence = string.Empty
};
var result = controller.ParsedSentence(model) as ViewResult;
Assert.AreEqual("0", model.ParsedSentence);
}
public void ParsingTest_RemoveNonAlphabetic()
{
var controller = new SentenceController();
var model = new SentenceModel
{
OriginalSentence = "1~@!$%1This1()*{}[]1is1~`&#&*&^%1a1:;?><9TEST!!!!"
};
var result = controller.ParsedSentence(model) as ViewResult;
Assert.AreEqual("1~@!$%1T2s1()*{}[]1i0s1~`&#&*&^%1a0a1:;?><9T2T!!!!", model.ParsedSentence);
}
public void ParsingTest_1234()
{
var controller = new SentenceController();
var model = new SentenceModel
{
OriginalSentence = "1a1_2bc2_3def3_4ghji4"
};
var result = controller.ParsedSentence(model) as ViewResult;
Assert.AreEqual("1a0a1_2b0c2_3d1f3_4g2i4", model.ParsedSentence);
}
public void ParsingTest_ABCDE()
{
var controller = new SentenceController();
var model = new SentenceModel
{
OriginalSentence = "Aa$BBb_CCCc@DdDDd^eEeEEE"
};
var result = controller.ParsedSentence(model) as ViewResult;
Assert.AreEqual("A0a$B1b_C1c@D2d^e2E", model.ParsedSentence);
}
public void ParsingTest_Smooth()
{
var controller = new SentenceController();
var model = new SentenceModel
{
OriginalSentence = "Smooth"
};
var result = controller.ParsedSentence(model) as ViewResult;
Assert.AreEqual("S3h", model.ParsedSentence);
}
private void LoadSentenceDetector()
{
if (!alreadyLoadSentenceDetector)
{
java.io.FileInputStream modelInpStream = new java.io.FileInputStream("Resources\\en-sent.bin");
SentenceModel sentenceModel = new SentenceModel(modelInpStream);
sentenceDetector = new SentenceDetectorME(sentenceModel);
alreadyLoadSentenceDetector = true;
}
}
static void Main(string[] args)
{
java.io.InputStream model_src = new java.io.FileInputStream(@"..\..\..\..\en-sent.bin");
SentenceModel smodel = new SentenceModel(model_src);
detector = new SentenceDetectorME(smodel);
foreach (String src in Directory.GetFiles(@"..\..\..\..\Dataset"))
{
lab5(src);
}
}
public static SentenceModel GetSentenceModel(string modelName, IResourceLoader loader)
{
if (!sentenceModels.TryGetValue(modelName, out SentenceModel model) || model == null)
{
using (Stream resource = loader.OpenResource(modelName))
{
model = new SentenceModel(new ikvm.io.InputStreamWrapper(resource));
}
sentenceModels[modelName] = model;
}
return(model);
}
public Span[] SentPosDetect(string paragraph)
{
var bin = GetFileStream("en-sent.bin");
SentenceModel model = new SentenceModel(bin);
SentenceDetectorME sdetector = new SentenceDetectorME(model);
Span[] sentences = sdetector.sentPosDetect(paragraph);
bin.close();
return(sentences);
}
/// <summary>Returns the meaning of a sentence.</summary>
/// <param name="sentence"><see cref="SentenceModel"/></param>
/// <exception cref="ArgumentNullException"><param name="sentence"/> was <c>null</c></exception>
public async Task <MeaningObject> SentenceMeaningAsync(SentenceModel sentence)
{
if (sentence == null)
{
throw new ArgumentNullException(nameof(sentence), "Sentence is a required argument.");
}
var get = await RestClient.GetAsync(
$"message?q={sentence.Message}&context={JsonConvert.SerializeObject(sentence.Context ?? DefaultContext)}" +
$"&msg_id={SnowFlake}&thread_id={SnowFlake}&n={sentence.MaxTraits}&verbose={sentence.Verbose}")
.ConfigureAwait(false);
return(await ProcessAsync <MeaningObject>(get));
}
public IActionResult CreateSentence([FromBody] SentenceModel sentence)
{
List <string> words = new List <string>();
words = sentence.ReversedSentence.Split(" ").ToList();
words.Reverse();
sentence.ReversedSentence = "";
foreach (var word in words)
{
sentence.ReversedSentence += word + " ";
}
_repository.Add(sentence);
return(Created("api/Sentence/posts", sentence));
}
static void Sent(string src)
{
StreamReader input = new StreamReader(src);
StreamWriter output = new StreamWriter(Regex.Replace(src, "(.*).txt", "$1_sent.txt"));
while (input.Peek() != -1)
{
InputStream modelIn = new FileInputStream(@"..\..\..\en-sent.bin");
SentenceModel smodel = new SentenceModel(modelIn);
SentenceDetector detector = new SentenceDetectorME(smodel);
string[] sents = detector.sentDetect(input.ReadLine());
foreach (string sent in sents)
{
output.WriteLine(sent);
}
}
}
public override void run(string format, string[] args)
{
base.run(format, args);
SentenceModel model = (new SentenceModelLoader()).load(@params.Model);
SentenceDetectorEvaluationMonitor errorListener = null;
if (@params.Misclassified.Value)
{
errorListener = new SentenceEvaluationErrorListener();
}
SentenceDetectorEvaluator evaluator = new SentenceDetectorEvaluator(new SentenceDetectorME(model), errorListener);
Console.Write("Evaluating ... ");
try
{
evaluator.evaluate(sampleStream);
}
catch (IOException e)
{
throw new TerminateToolException(-1, "IO error while reading training data or indexing data: " + e.Message, e);
}
finally
{
try
{
sampleStream.close();
}
catch (IOException)
{
// sorry that this can fail
}
}
Console.Error.WriteLine("done");
Console.WriteLine();
Console.WriteLine(evaluator.FMeasure);
}
public static void Main(string[] args)
{
DirectoryInfo folder = new DirectoryInfo(@"..\..\..\..\..\Dataset");
foreach (var fname in folder.GetFiles())
{
String line = File.ReadAllText(fname.FullName);
java.io.InputStream modelIn = new java.io.FileInputStream(@"..\..\..\..\..\en-sent.bin");
SentenceModel smodel = new SentenceModel(modelIn);
SentenceDetector detector = new SentenceDetectorME(smodel);
string[] sents = detector.sentDetect(line);
using (StreamWriter sw = new StreamWriter(fname.FullName.Replace(fname.FullName.Substring(fname.FullName.Length - 3), "rtf")))
{
foreach (var sent in sents)
{
sw.WriteLine(sent);
}
}
}
}
public void SplitSentences()
{
var txt = File.ReadAllText(@"c:\dev\d-mill\uspe\Data\uspe-1.txt");
txt = Regex.Replace(txt, "\\s+", " ");
txt = Regex.Replace(txt, "\\r\\n", "");
txt = Regex.Replace(txt, "MR.\\s+", "MR.");
var modelStream = new java.io.FileInputStream("../../Models/en-sent.bin");
var model = new SentenceModel(modelStream);
var detector = new SentenceDetectorME(model);
var sentences = detector.sentDetect(txt);
File.WriteAllLines(@"c:\dev\d-mill\uspe\Data\uspe-sentenced.txt", sentences);
}
/// <summary>
/// Perform sentence detection the input stream.
///
/// A newline will be treated as a paragraph boundary.
/// </summary>
public override void run(string[] args)
{
if (args.Length != 1)
{
Console.WriteLine(Help);
}
else
{
SentenceModel model = (new SentenceModelLoader()).load(new File(args[0]));
SentenceDetectorME sdetector = new SentenceDetectorME(model);
ObjectStream <string> paraStream = new ParagraphStream(new PlainTextByLineStream(new InputStreamReader(Console.OpenStandardInput)));
PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "sent");
perfMon.start();
try
{
string para;
while ((para = paraStream.read()) != null)
{
string[] sents = sdetector.sentDetect(para);
foreach (string sentence in sents)
{
Console.WriteLine(sentence);
}
perfMon.incrementCounter(sents.Length);
Console.WriteLine();
}
}
catch (IOException e)
{
CmdLineUtil.handleStdinIoError(e);
}
perfMon.stopAndPrintFinalResult();
}
}
public NLP()
{
//loading sentence detector model
java.io.FileInputStream modelInpStream = new java.io.FileInputStream("Resources\\en-sent.bin");
SentenceModel sentenceModel = new SentenceModel(modelInpStream);
sentenceDetector = new SentenceDetectorME(sentenceModel);
//loading tokenizer model
modelInpStream = new java.io.FileInputStream("Resources\\en-token.bin");
TokenizerModel tokenizerModel = new TokenizerModel(modelInpStream);
tokenizer = new TokenizerME(tokenizerModel);
modelInpStream = new java.io.FileInputStream("Resources\\en-pos-maxent.bin");
POSModel posModel = new POSModel(modelInpStream);
tagger = new POSTaggerME(posModel);
modelInpStream = new java.io.FileInputStream("Resources\\en-chunker.bin");
ChunkerModel chunkerModel = new ChunkerModel(modelInpStream);
chunker = new ChunkerME(chunkerModel);
modelInpStream = new java.io.FileInputStream("Resources\\en-parser-chunking.bin");
ParserModel parserModel = new ParserModel(modelInpStream);
parser = ParserFactory.create(parserModel);
//loading stop words list
StreamReader sr = new StreamReader("Resources\\english.stop.txt");
string line;
while ((line = sr.ReadLine()) != null)
{
stopwords.Add(Stemming(line));
stopwords.Add(line);
}
}
public async Task <ActionResult> Post([FromBody] SentenceModel value)
{
var client = _httpClientFactory.CreateClient("sa-logic");
var settings = new JsonSerializerSettings
{
ContractResolver = new CamelCasePropertyNamesContractResolver()
};
var camelCaseLoad = JsonConvert.SerializeObject(value, settings);
var content = new StringContent(camelCaseLoad, Encoding.UTF8, "application/json");
var result = await client.PostAsync("/analyse/sentiment", content);
if (!result.IsSuccessStatusCode)
{
return(BadRequest(value));
}
var sentiment = await result.Content.ReadAsAsync <SentimentModel>();
return(Ok(sentiment));
}
static void Main(string[] args)
{
StreamWriter sw = new StreamWriter(@"..\..\Data\result.txt");
StreamReader sr = new StreamReader(@"..\..\Data\data.txt");
while (sr.Peek() != -1)
{
string line = sr.ReadLine();
java.io.InputStream modelIn = new java.io.FileInputStream("en-sent.bin");
SentenceModel smodel = new SentenceModel(modelIn);
SentenceDetector detector = new SentenceDetectorME(smodel);
string[] sents = detector.sentDetect(line);
foreach (var sent in sents)
{
sw.WriteLine(sent);
sw.WriteLine();
}
sw.Flush();
}
sr.Close();
sw.Close();
}
static void Main(string[] args)
{
java.io.InputStream modelIn = new java.io.FileInputStream(string.Format("en-sent.bin"));
java.io.InputStream modelIn2 = new java.io.FileInputStream(string.Format("en-token.bin"));
TokenizerModel model = new TokenizerModel(modelIn2);
TokenizerME mE = new TokenizerME(model);
SentenceModel sM = new SentenceModel(modelIn);
SentenceDetector detector = new SentenceDetectorME(sM);
string folderName = @"C:\Users\Administrator\Desktop\lab-6-opennlp-ju-zi-qie-fen-10411174\file";
foreach (string fname in System.IO.Directory.GetFiles(folderName))
{
String line = null;
String[] name = fname.Split('\\');
StreamWriter sw = new StreamWriter(@"C:\Users\Administrator\Desktop\lab-6-opennlp-ju-zi-qie-fen-10411174\answer\" + name[6]);
StreamReader file2 = new StreamReader(fname);
while ((line = file2.ReadLine()) != null)
{
string str = null;
string[] sents = detector.sentDetect(line);
if (sents.Length.Equals(0))
{
continue;
}
foreach (var s in sents)
{
str = str + s;
}
var Tokens = mE.tokenize(str);
foreach (var s in Tokens)
{
sw.Write(s + " ");
}
sw.WriteLine();
}
sw.Close();
}
}
public static void ConvertToJson(string textFile, string word)
{
List <SentenceModel> sentenceModelsList = new List <SentenceModel>();
foreach (var sentece in FindWord(textFile, word))
{
SentenceModel sentenceModel = new SentenceModel()
{
Word = word,
Sentence = sentece,
Number = Numbers(sentece, word)
};
sentenceModelsList.Add(sentenceModel);
}
string add = File.ReadAllText(HttpContext.Current.Server.MapPath("Data/jsonFile.json"));
var back = JsonConvert.DeserializeObject <List <SentenceModel> >(add);
sentenceModelsList.AddRange(back);
var json = JsonConvert.SerializeObject(sentenceModelsList);
File.WriteAllText(HttpContext.Current.Server.MapPath("Data/jsonFile.json"), json);
}
private static SentenceDetectorME PrepareSentenceDetector()
{
var sentModelStream = new FileInputStream(SentenceModelPath); //load the sentence model into a stream
var sentModel = new SentenceModel(sentModelStream); // load the model
return new SentenceDetectorME(sentModel); //create sentence detector
}
static Sentenizer()
{
var modelFile = ConfigurationManager.AppSettings["ModelSentenizer"] ?? string.Empty;
ModelIn = new java.io.FileInputStream(modelFile);
Model = new SentenceModel(ModelIn);
}
public String chooseSentenceMenu()
{
int userInputNumber = 0;
string userChoosenSentence = "";
System.Console.WriteLine("Choose a sentence to use from your current text");
System.Console.WriteLine("Must be a space between each sentence");
try
{
java.io.File file = new java.io.File("C:\\en-sent.bin");
java.io.InputStream modelIn = new FileInputStream("C:\\Users\\jcoleman\\Documents\\Capstone\\jcoleman_Capstone\\Code\\NEWCHATBOT\\ConsoleBot\\ConsoleBot\\en-sent.bin");
SentenceModel model = new SentenceModel(modelIn);
SentenceDetectorME sentenceDetector = new SentenceDetectorME(model);
string text = "";
FileText = System.IO.File.ReadAllLines(FilePath);
for (int i = 0; i < FileText.Length; i++)
{
text += FileText[i];
}
string[] sentences = sentenceDetector.sentDetect(text);
for(int s = 0;s < sentences.Length;s++)
{
System.Console.WriteLine((s+1) +" : " +sentences[s]);
}
string userInput = System.Console.ReadLine();
userInputNumber = int.Parse(userInput);
userChoosenSentence = sentences[userInputNumber - 1];
modelIn.close();
}
catch(Exception e)
{
System.Console.WriteLine(e.Message);
}
return userChoosenSentence;
}
private SentenceDetectorME PrepareSentenceDetector()
{
//TODO[danielcamargo]: we need to find/train the model in spanish
var sentModelStream =
new FileInputStream(string.Format(@"Models\{0}-sent.bin", _language == "es" ? "pt" : _language));
var sentModel = new SentenceModel(sentModelStream);
sentModelStream.close();
return new SentenceDetectorME(sentModel);
}
public SentenceDetector(SentenceModel model)
{
this.detector = new SentenceDetectorME(model);
}
public SentenceDetector(FileStream modelStream)
{
SentenceModel model = new SentenceModel(modelStream);
this.detector = new SentenceDetectorME(model);
}
public static string[] SplitSentences(string Text)
{
var modelStream = new java.io.ByteArrayInputStream(Resource.en_sent);
var model = new SentenceModel(modelStream);
var detector = new SentenceDetectorME(model);
return detector.sentDetect(Text);
}
