public async Task <IActionResult> Create()
{
var enumexcel = EnumerateExcel();
LocalList ll = new LocalList();
foreach (var e in enumexcel)
{
ll.LocalWagonList.Add(new ParserModel()
{
TCNumber = e.TCNumber, WaybillNumber = e.WaybillNumber
});
}
for (int i = 0; i < ll.LocalWagonList.Count; i++)
{
var wagon = new ParserModel()
{
TCNumber = ll.LocalWagonList[i].TCNumber,
WaybillNumber = ll.LocalWagonList[i].WaybillNumber
};
db.Wagons.Add(wagon);
}
await db.SaveChangesAsync();
return(RedirectToAction("Index"));
}
public void TestChunkingParserTraining()
{
var parseSamples = ParserTestUtil.CreateParseTestStream();
//var testSamples = ParserTestUtil.CreateParseTestStream();
var headRules = ParserTestUtil.CreateTestHeadRules();
var model = SharpNL.Parser.Chunking.Parser.Train("en", parseSamples, headRules, 100, 0);
Assert.NotNull(model);
var parser = ParserFactory.Create(model);
Assert.NotNull(parser);
ParserModel deserialized;
using (var stream = new MemoryStream()) {
model.Serialize(new UnclosableStream(stream));
stream.Seek(0, SeekOrigin.Begin);
deserialized = new ParserModel(stream);
}
Assert.NotNull(deserialized);
// TODO: compare both models
}
/// <summary>
/// Export datas in a CSV file.
/// </summary>
/// <param name="Filename">Path of the CSV file.</param>
public void Export(string Filename)
{
List <ParserModel> ListToWrite = new List <ParserModel>();
/* Creating data objects */
Console.WriteLine("[Debug: Bacchus.view.CSVManagerViewModel.readCSV] Getting data objects.");
List <ArticlesModel> Articles = ArticlesDao.SelectAll();
foreach (ArticlesModel Article in Articles)
{
ParserModel ElementToAdd = new ParserModel();
ElementToAdd.Description = Article.Description;
ElementToAdd.Ref = Article.RefArticle;
ElementToAdd.Marque = Article.Marque.Nom;
ElementToAdd.Famille = Article.SousFamille.Famille.Nom;
ElementToAdd.SousFamille = Article.SousFamille.Nom;
ElementToAdd.Prix = Article.PrixHT;
ListToWrite.Add(ElementToAdd);
}
if (WriteCSV(ListToWrite, Filename))
{
AdvertBox("Le transfert a été effectué avec succès.", "information");
ExportWindow.Close();
}
}
public void TestTreeInsertParserTraining()
{
var parseSamples = ParserTestUtil.CreateParseTrainStream();
var headRules = ParserTestUtil.CreateTestHeadRules();
var model = SharpNL.Parser.TreeInsert.Parser.Train("en", parseSamples, headRules, 100, 0);
var parser = ParserFactory.Create(model);
// Tests parsing to make sure the code does not has
// a bug which fails always with a runtime exception
var p = parser.Parse(Parse.ParseParse("She was just another freighter from the " +
"States and she seemed as commonplace as her name ."));
ParserModel deserialized;
using (var data = new MemoryStream()) {
model.Serialize(new UnclosableStream(data));
data.Seek(0, SeekOrigin.Begin);
deserialized = new ParserModel(data);
}
Assert.NotNull(deserialized);
// TODO: compare both models
}
public MainWindow()
{
InitializeComponent();
var model = new ParserModel(Browser);
DataContext = model;
}
void MainWindow_Loaded(object sender, RoutedEventArgs e)
{
ParserNodeReporter.Current.NodeAdded += Current_NodeAdded;
ParserNodeReporter.Current.NodeRemoved += Current_NodeRemoved;
ParserModel parserModel = new ParserModel(new Parser());
DataContext = parserModel;
}
public Parser(ParserModel model, int beamSize, double advancePercentage) :
this(model.BuildModel, model.AttachModel, model.CheckModel,
new POSTaggerME(model.ParserTaggerModel),
new ChunkerME(model.ParserChunkerModel, ChunkerME.DefaultBeamSize, new ParserChunkerSequenceValidator(model.ParserChunkerModel), new ChunkContextGenerator()),
model.HeadRules,
beamSize,
advancePercentage)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="Parser"/> class using the specified model with the specified beam size and advance percentage.
/// </summary>
/// <param name="model">The parser model.</param>
/// <param name="beamSize">The number of different parses kept during parsing.</param>
/// <param name="advancePercentage">The minimal amount of probability mass which advanced outcomes must represent.</param>
public Parser(ParserModel model, int beamSize, double advancePercentage) :
this(model.BuildModel, model.AttachModel, model.CheckModel,
new POSTaggerME(model.ParserTaggerModel),
new ChunkerME(model.ParserChunkerModel),
model.HeadRules,
beamSize,
advancePercentage)
{
}
public ActionResult Parse(ParserModel model)
{
if (!ModelState.IsValid)
{
return(View("Index", model));
}
var resultModel = _service.GetParsedResults(model.Url);
return(View("Result", resultModel));
}
private void LoadParser()
{
if (!alreadyLoadParser)
{
java.io.FileInputStream modelInpStream = new java.io.FileInputStream("Resources\\en-parser-chunking.bin");
ParserModel parserModel = new ParserModel(modelInpStream);
parser = ParserFactory.create(parserModel);
alreadyLoadParser = true;
}
}
public Parser(ParserModel model, int beamSize, double advancePercentage)
: this(
model.BuildModel,
model.CheckModel,
new POSTaggerME(model.ParserTaggerModel),
new ChunkerME(model.ParserChunkerModel),
model.HeadRules,
beamSize,
advancePercentage) {
}
protected internal override ParserModel trainAndUpdate(ParserModel originalModel, ObjectStream <Parse> parseSamples, ModelUpdaterParams parameters)
{
Dictionary mdict = ParserTrainerTool.buildDictionary(parseSamples, originalModel.HeadRules, parameters.Cutoff.Value);
parseSamples.reset();
// TODO: training individual models should be in the chunking parser, not here
// Training build
Console.WriteLine("Training builder");
opennlp.model.EventStream bes = new ParserEventStream(parseSamples, originalModel.HeadRules, ParserEventTypeEnum.BUILD, mdict);
AbstractModel buildModel = Parser.train(bes, parameters.Iterations.Value, parameters.Cutoff.Value);
parseSamples.close();
return(originalModel.updateBuildModel(buildModel));
}
/// <summary>
/// Method used to read a CSV defined by his filename.
/// </summary>
/// <param name="Filename">The path to the CSV file.</param>
/// <returns></returns>
private List <ParserModel> ReadCSV(string Filename)
{
var Records = new List <ParserModel>();
try
{
using (var Reader = new StreamReader(Filename, Encoding.Default))
using (var Csv = new CsvReader(Reader, CultureInfo.InvariantCulture))
{
/* Reading CSV File */
Console.WriteLine("[Debug: Bacchus.view.CSVManagerViewModel.readCSV] Reading CSV file.");
Csv.Configuration.Delimiter = ";";
Csv.Read();
Csv.ReadHeader();
while (Csv.Read())
{
try
{
var record = new ParserModel
{
Description = Csv.GetField <string>("Description"),
Ref = Csv.GetField <string>("Ref"),
Marque = Csv.GetField <string>("Marque"),
Famille = Csv.GetField <string>("Famille"),
SousFamille = Csv.GetField <string>("Sous-Famille"),
Prix = Csv.GetField <double>("Prix H.T.")
};
Records.Add(record);
}
catch (Exception)
{
AdvertBox("L'importation a échouée : un champ ou une colonne est manquant dans le fichier CSV.", "erreur");
return(null);
}
}
}
}
catch
{
AdvertBox("L'importation a échouée : le fichier CSV est ouvert par un autre programme.", "erreur");
return(null);
}
return(Records);
}
public override void run(string[] args)
{
if (args.Length != 2)
{
Console.WriteLine(Help);
}
else
{
File parserModelInFile = new File(args[0]);
ParserModel parserModel = (new ParserModelLoader()).load(parserModelInFile);
File taggerModelInFile = new File(args[1]);
POSModel taggerModel = (new POSModelLoader()).load(taggerModelInFile);
ParserModel updatedParserModel = parserModel.updateTaggerModel(taggerModel);
CmdLineUtil.writeModel("parser", parserModelInFile, updatedParserModel);
}
}
static IEnumerable <ParserModel> EnumerateExcel()
{
string xlsxpath = "WAGON_LIST 150520.xlsx";
using (var workbook = new XLWorkbook(xlsxpath))
{
var worksheet = workbook.Worksheets.Worksheet(1);
int rows = worksheet.RowsUsed().Count();
for (int row = 2; row <= rows; ++row)
{
var metric = new ParserModel
{
TCNumber = worksheet.Cell(row, 2).GetValue <int>(),
WaybillNumber = worksheet.Cell(row, 3).GetValue <string>()
};
yield return(metric);
}
}
}
public NLP()
{
//loading sentence detector model
java.io.FileInputStream modelInpStream = new java.io.FileInputStream("Resources\\en-sent.bin");
SentenceModel sentenceModel = new SentenceModel(modelInpStream);
sentenceDetector = new SentenceDetectorME(sentenceModel);
//loading tokenizer model
modelInpStream = new java.io.FileInputStream("Resources\\en-token.bin");
TokenizerModel tokenizerModel = new TokenizerModel(modelInpStream);
tokenizer = new TokenizerME(tokenizerModel);
modelInpStream = new java.io.FileInputStream("Resources\\en-pos-maxent.bin");
POSModel posModel = new POSModel(modelInpStream);
tagger = new POSTaggerME(posModel);
modelInpStream = new java.io.FileInputStream("Resources\\en-chunker.bin");
ChunkerModel chunkerModel = new ChunkerModel(modelInpStream);
chunker = new ChunkerME(chunkerModel);
modelInpStream = new java.io.FileInputStream("Resources\\en-parser-chunking.bin");
ParserModel parserModel = new ParserModel(modelInpStream);
parser = ParserFactory.create(parserModel);
//loading stop words list
StreamReader sr = new StreamReader("Resources\\english.stop.txt");
string line;
while ((line = sr.ReadLine()) != null)
{
stopwords.Add(Stemming(line));
stopwords.Add(line);
}
}
public sealed override void run(string format, string[] args)
{
ModelUpdaterParams @params = validateAndParseParams <ModelUpdaterParams>(ArgumentParser.filter(args, typeof(ModelUpdaterParams)), typeof(ModelUpdaterParams));
// Load model to be updated
Jfile modelFile = @params.Model;
ParserModel originalParserModel = (new ParserModelLoader()).load(modelFile);
ObjectStreamFactory factory = getStreamFactory(format);
string[] fargs = ArgumentParser.filter(args, factory.Parameters);
validateFactoryArgs(factory, fargs);
ObjectStream <Parse> sampleStream = factory.create <Parse>(fargs);
ParserModel updatedParserModel;
try
{
updatedParserModel = trainAndUpdate(originalParserModel, sampleStream, @params);
}
catch (IOException e)
{
throw new TerminateToolException(-1, "IO error while reading training data or indexing data: " + e.Message, e);
}
finally
{
try
{
sampleStream.close();
}
catch (IOException)
{
// sorry that this can fail
}
}
CmdLineUtil.writeModel("parser", modelFile, updatedParserModel);
}
public override ObjectStream <CorefSample> create(string[] args)
{
Parameters @params = ArgumentParser.parse(args, typeof(Parameters));
ParserModel parserModel = (new ParserModelLoader()).load(@params.ParserModel);
Parser parser = ParserFactory.create(parserModel);
TokenizerModel tokenizerModel = (new TokenizerModelLoader()).load(@params.TokenizerModel);
Tokenizer tokenizer = new TokenizerME(tokenizerModel);
ObjectStream <string> mucDocStream = new FileToStringSampleStream(new DirectorySampleStream(@params.Data, new FileFilterAnonymousInnerClassHelper(this), false), Charset.forName("UTF-8"));
ObjectStream <RawCorefSample> rawSamples = new MucCorefSampleStream(tokenizer, mucDocStream);
ObjectStream <RawCorefSample> parsedSamples = new FullParseCorefEnhancerStream(parser, rawSamples);
// How to load all these nameFinder models ?!
// Lets make a param per model, not that nice, but ok!
IDictionary <string, Jfile> modelFileTagMap = new Dictionary <string, Jfile>();
modelFileTagMap["person"] = @params.PersonModel;
modelFileTagMap["organization"] = @params.OrganizationModel;
IList <TokenNameFinder> nameFinders = new List <TokenNameFinder>();
IList <string> tags = new List <string>();
foreach (KeyValuePair <string, Jfile> entry in modelFileTagMap)
{
nameFinders.Add(new NameFinderME((new TokenNameFinderModelLoader()).load(entry.Value)));
tags.Add(entry.Key);
}
return(new MucMentionInserterStream(new NameFinderCorefEnhancerStream(nameFinders.ToArray(), tags.ToArray(), parsedSamples)));
}
/// <summary>
/// Initializes a new instance of the <see cref="Parser"/> class using the specified model with the default beam size and the default advance percentage.
/// </summary>
/// <param name="model">The parser model.</param>
public Parser(ParserModel model) : this(model, DefaultBeamSize, DefaultAdvancePercentage) {}
public void setUpParser()
{
string modelPath = "C:\\Users\\jcoleman\\Documents\\Capstone\\jcoleman_Capstone\\Code\\NEWCHATBOT\\ConsoleBot\\ConsoleBot\\en-parser-chunking.bin";
java.io.FileInputStream modelInpstream = new java.io.FileInputStream(modelPath);
parserModel = new ParserModel(modelInpstream);
parser = ParserFactory.create(parserModel);
parse = ParserTool.parseLine(sentence, parser, 1);
}
public override void run(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine(Help);
}
else
{
ParserModel model = (new ParserModelLoader()).load(new File(args[args.Length - 1]));
int?beamSize = CmdLineUtil.getIntParameter("-bs", args);
if (beamSize == null)
{
beamSize = AbstractBottomUpParser.defaultBeamSize;
}
int? numParses = CmdLineUtil.getIntParameter("-k", args);
bool showTopK;
if (numParses == null)
{
numParses = 1;
showTopK = false;
}
else
{
showTopK = true;
}
double?advancePercentage = CmdLineUtil.getDoubleParameter("-ap", args);
if (advancePercentage == null)
{
advancePercentage = AbstractBottomUpParser.defaultAdvancePercentage;
}
opennlp.tools.parser.Parser parser = ParserFactory.create(model, beamSize.Value, advancePercentage.Value);
ObjectStream <string> lineStream = new PlainTextByLineStream(new InputStreamReader(Console.OpenStandardInput));
PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "sent");
perfMon.start();
try
{
string line;
while ((line = lineStream.read()) != null)
{
if (line.Length == 0)
{
Console.WriteLine();
}
else
{
Parse[] parses = parseLine(line, parser, numParses.Value);
for (int pi = 0, pn = parses.Length; pi < pn; pi++)
{
if (showTopK)
{
Console.Write(pi + " " + parses[pi].Prob + " ");
}
parses[pi].show();
perfMon.incrementCounter();
}
}
}
}
catch (IOException e)
{
CmdLineUtil.handleStdinIoError(e);
}
perfMon.stopAndPrintFinalResult();
}
}
public Dictionary <string, List <string> > Main(string line)
{
//debug sentence
// line = "Show me the sales of Kean Cola .25ltr Bottle in Nicosia from January 2017 to October 2017 as a line chart.";
matchedWords?.Clear();
nounPhrases?.Clear();
nouns?.Clear();
adjectivePhrases?.Clear();
verbPhrases?.Clear();
InputStream modelIn = new FileInputStream(HttpRuntime.AppDomainAppPath + "\\Models\\en-parser-chunking.bin");
InputStream modelIn1 = new FileInputStream(HttpRuntime.AppDomainAppPath + "\\Models\\en-ner-date.bin");
InputStream modelIn2 = new FileInputStream(HttpRuntime.AppDomainAppPath + "\\Models\\en-token.bin");
ParserModel model = new ParserModel(modelIn);
var myParser = ParserFactory.create(model);
var topParses = ParserTool.parseLine(line, myParser, 1);
foreach (var p in topParses)
{
GetSentenceParts(p);
}
try
{
TokenizerModel model1 = new TokenizerModel(modelIn2);
TokenNameFinderModel model2 = new TokenNameFinderModel(modelIn1);
Tokenizer tokenizer = new TokenizerME(model1);
var nameFinder = new NameFinderME(model2);
var tokens = tokenizer.tokenize(line);
var nameSpans = nameFinder.find(tokens);
var array = Span.spansToStrings(nameSpans, tokens);
//
// foreach (var v in array)
// {
// System.Diagnostics.Debug.WriteLine(v);
// }
dates = new HashSet <string>(array);
PrintSets();
// System.Diagnostics.Debug.WriteLine("\nProcessing Presentation type");
//
// if (nouns.Contains("table"))
// {
// matchedWords.Add(new Tuple<string, string>("PRESENTATION_TYPE", "table"));
// }
// if (nounPhrases.Contains("bar chart"))
// {
// matchedWords.Add(new Tuple<string, string>("PRESENTATION_TYPE", "bar chart"));
// }
// if (nounPhrases.Contains("line chart"))
// {
// matchedWords.Add(new Tuple<string, string>("PRESENTATION_TYPE", "line chart"));
// }
//TODO IF NO OPTION IS FOUND ASK THE USER TO GIVE YOU ONE. IMPLEMENT IT IN THE WEB VERSION SOON
System.Diagnostics.Debug.WriteLine("\nProcessing Dates");
if (dates.Count == 2)
{
if (dates.ElementAt(0).contains("from"))
{
var a = dates.ElementAt(0).replace("from", "");
List <string> newList = new List <string>();
newList.Add("START_PERIOD");
matchedWords.Add(a, newList);
newList = new List <string>();
newList.Add("END_PERIOD");
//todo fix when the date is the same here
matchedWords.Add(dates.ElementAt(1), newList);
}
else
{
List <string> newList = new List <string>();
newList.Add("START_PERIOD");
matchedWords.Add(dates.ElementAt(0), newList);
newList = new List <string>();
newList.Add("END_PERIOD");
//todo fix when the date is the same here
matchedWords.Add(dates.ElementAt(1), newList);
}
}
if (dates.Count == 1)
{
if (dates.ElementAt(0).contains("from"))
{
var a = dates.ElementAt(0).replace("from", "");
var dts = a.Split(new[] { " to " }, StringSplitOptions.None);
List <string> newList = new List <string>();
newList.Add("START_PERIOD");
matchedWords.Add(dts[0], newList);
newList = new List <string>();
newList.Add("END_PERIOD");
//todo fix when the date is the same here
matchedWords.Add(dts[1], newList);
}
else
{
List <string> newList = new List <string>();
newList.Add("START_PERIOD");
newList.Add("END_PERIOD");
//todo fix when the date is the same here
matchedWords.Add(dates.ElementAt(0), newList);
}
}
System.Diagnostics.Debug.WriteLine("\nProcessing noun phrases");
// var manager = new Manager();
// var serializer = new XmlSerializer(typeof(Manager.language));
// var loadStream = new FileStream("file2.xml", FileMode.Open, FileAccess.Read);
// var loadedObject = (Manager.language) serializer.Deserialize(loadStream);
var doc = new XmlDocument();
// System.Diagnostics.Debug.WriteLine(HttpRuntime.AppDomainAppPath);
// System.Diagnostics.Debug.WriteLine(HttpRuntime.AppDomainAppPath);
// System.Diagnostics.Debug.WriteLine(HttpRuntime.AppDomainAppPath);
// System.Diagnostics.Debug.WriteLine(HttpRuntime.AppDomainAppPath);
doc.Load(HttpRuntime.AppDomainAppPath + "\\file2.xml");
var root = doc.SelectSingleNode("*");
FindMatchingNodesFromXml(root, nounPhrases);
foreach (var item in nouns.ToList())
{
foreach (var VARIABLE in matchedWords)
{
if (VARIABLE.Key.Contains(item))
{
nouns.Remove(item); //Will work!
}
}
}
FindMatchingNodesFromXml(root, verbPhrases);
// FindMatchingNodesFromXml(root, nouns);
System.Diagnostics.Debug.WriteLine("\nProcessing verb phrases ");
System.Diagnostics.Debug.WriteLine("\nProcessing nouns ");
// construct the dictionary object and open it
var directory = Directory.GetCurrentDirectory() + "\\wordnet\\";
foreach (var variable in matchedWords)
{
System.Diagnostics.Debug.WriteLine(variable.Value + "\t\t" + variable.Key);
}
foreach (var variable in matchedWords)
{
string a = variable.Key;
if (line.Contains(a))
{
line = line.replace(a, "");
}
}
foreach (var variable in stopWordsofwordnet)
{
string a = " " + variable.toLowerCase() + " ";
if (line.Contains(a))
{
line = line.replace(a, " ");
}
}
if (line.contains("."))
{
line = line.replace(".", "");
}
if (line.contains("-"))
{
line = line.replace("-", " ");
}
System.Diagnostics.Debug.WriteLine("/////////////");
System.Diagnostics.Debug.WriteLine("SECOND PARSE STRING " + line);
System.Diagnostics.Debug.WriteLine("/////////////");
line = line.Trim();
topParses = ParserTool.parseLine(line, myParser, 1);
nounPhrases?.Clear();
dates?.Clear();
verbPhrases?.Clear();
nouns?.Clear();
foreach (var p in topParses)
{
//p.show();
GetSentenceParts(p);
}
FindMatchingNodesFromXml(root, nounPhrases);
foreach (var item in nouns.ToList())
{
foreach (var VARIABLE in matchedWords)
{
if (VARIABLE.Key.Contains(item))
{
nouns.Remove(item); //Will work!
}
}
}
FindMatchingNodesFromXml(root, verbPhrases);
FindMatchingNodesFromXml(root, nouns);
tokens = tokenizer.tokenize(line);
nameSpans = nameFinder.find(tokens);
array = Span.spansToStrings(nameSpans, tokens);
dates = new HashSet <string>(array);
PrintSets();
System.Diagnostics.Debug.WriteLine("\nProcessing Dates");
if (dates.Count == 2)
{
if (dates.ElementAt(0).contains("from"))
{
var a = dates.ElementAt(0).replace("from", "");
List <string> newList = new List <string>();
newList.Add("START_PERIOD");
matchedWords.Add(a, newList);
newList = new List <string>();
newList.Add("END_PERIOD");
//todo fix when the date is the same here
matchedWords.Add(dates.ElementAt(1), newList);
}
else
{
List <string> newList = new List <string>();
newList.Add("START_PERIOD");
matchedWords.Add(dates.ElementAt(0), newList);
newList = new List <string>();
newList.Add("END_PERIOD");
//todo fix when the date is the same here
matchedWords.Add(dates.ElementAt(1), newList);
}
}
if (dates.Count == 1)
{
if (dates.ElementAt(0).contains("from"))
{
var a = dates.ElementAt(0).replace("from", "");
var dts = a.Split(new[] { " to " }, StringSplitOptions.None);
List <string> newList = new List <string>();
newList.Add("START_PERIOD");
matchedWords.Add(dts[0], newList);
newList = new List <string>();
newList.Add("END_PERIOD");
//todo fix when the date is the same here
matchedWords.Add(dts[1], newList);
}
else
{
List <string> newList = new List <string>();
newList.Add("START_PERIOD");
newList.Add("END_PERIOD");
//todo fix when the date is the same here
matchedWords.Add(dates.ElementAt(0), newList);
}
}
System.Diagnostics.Debug.WriteLine("\nProcessing noun phrases");
// var manager = new Manager();
// var serializer = new XmlSerializer(typeof(Manager.language));
// var loadStream = new FileStream("file2.xml", FileMode.Open, FileAccess.Read);
// var loadedObject = (Manager.language) serializer.Deserialize(loadStream);
FindMatchingNodesFromXml(root, nounPhrases);
FindMatchingNodesFromXml(root, verbPhrases);
FindMatchingNodesFromXml(root, nouns);
foreach (var variable in matchedWords)
{
System.Diagnostics.Debug.WriteLine(variable.Value + "\t\t" + variable.Key);
}
doc = null;
GC.Collect();
GC.WaitForPendingFinalizers();
//MATCHING WITH WORD NET
System.Diagnostics.Debug.WriteLine(directory);
// var wordNet = new WordNetEngine();
//
// wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.adj")), PartOfSpeech.Adjective);
// wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.adv")), PartOfSpeech.Adverb);
// wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.noun")), PartOfSpeech.Noun);
// wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.verb")), PartOfSpeech.Verb);
//
// wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adj")), PartOfSpeech.Adjective);
// wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adv")), PartOfSpeech.Adverb);
// wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.noun")), PartOfSpeech.Noun);
// wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.verb")), PartOfSpeech.Verb);
//
// System.Diagnostics.Debug.WriteLine("Loading database...");
// wordNet.Load();
// System.Diagnostics.Debug.WriteLine("Load completed.");
// while (true)
// {
// System.Diagnostics.Debug.WriteLine("\nType first word");
//
// var word = System.Diagnostics.Debug.ReadLine();
// var synSetList = wordNet.GetSynSets(word);
//
// if (synSetList.Count == 0) System.Diagnostics.Debug.WriteLine($"No SynSet found for '{word}'");
//
// foreach (var synSet in synSetList)
// {
// var words = string.Join(", ", synSet.Words);
//
// System.Diagnostics.Debug.WriteLine($"\nWords: {words}");
// }
// }
}
catch (IOException e)
{
e.printStackTrace();
}
finally
{
if (modelIn1 != null)
{
try
{
modelIn1.close();
}
catch (IOException e)
{
}
}
if (modelIn2 != null)
{
try
{
modelIn2.close();
}
catch (IOException e)
{
}
}
// truncateLists(ref nounPhrases);
// truncateLists(ref nouns);
// truncateLists(ref dates);
// truncateLists(ref verbPhrases);
}
return(matchedWords);
}
public Parser(ParserModel model) : this(model, DefaultBeamSize, DefaultAdvancePercentage)
{
}
// Constructors and finalizers:
private Repository()
{
_assemblyName = Regex.Match(_assemblyFullName, "^(.*?),.*$").Result("$1");
_rootDrive = ("/usr/project/xtmp/dp195/Poetix18/").Replace(@"\", Dsc);
_nlpFolder = ("rhetorica/nlp/").Replace(@"\", Dsc);
_openNlpModelsFolder = ("OpenNLP/models/").Replace(@"\", Dsc);
_openNlpModelsPath = RootDrive + _nlpFolder + _openNlpModelsFolder;
_wordNetFolder = ("WordNet_3/").Replace(@"\", Dsc);
_wordNetPath = RootDrive + _nlpFolder + _wordNetFolder;
_grammarFolder = ("StanfordParser/grammar/").Replace(@"\", Dsc);
_grammarPath = RootDrive + _nlpFolder + _grammarFolder;
_dataFolder = ("data/").Replace(@"\", Dsc);
_nlpTextsPath = RootDrive + _dataFolder;
string[] localTextDirectoryParts =
{
CurrentAssemblyDirectoryPath,
"..", "..","..", "data"
//"..", "..", "text"
};
_localTextPath = Path.Combine(localTextDirectoryParts) + "/"; // For development use
// WordNet engine:
Console.Write("Loading WordNet engine.... ");
_wordNetEngine = new WordNetEngine(WordNetPath, true);
Console.WriteLine("Done.");
// OpenNLP sentence detector:
Console.Write("Loading OpenNLP sentence detector.... ");
java.io.FileInputStream modelInputStream = new java.io.FileInputStream(OpenNlpModelsPath + "en-sent.bin");
_sentenceModel = new SentenceModel(modelInputStream);
modelInputStream.close();
_sentenceDetector = new SentenceDetectorME(_sentenceModel);
Console.WriteLine("Done.");
// OpenNLP tokenizer:
Console.Write("Loading OpenNLP tokenizer.... ");
modelInputStream = new java.io.FileInputStream(OpenNlpModelsPath + "en-token.bin");
_tokenizerModel = new opennlp.tools.tokenize.TokenizerModel(modelInputStream);
modelInputStream.close();
_tokenizer = new opennlp.tools.tokenize.TokenizerME(_tokenizerModel);
Console.WriteLine("Done.");
// OpenNLP name finder:
Console.Write("Loading OpenNLP name finder.... ");
modelInputStream = new java.io.FileInputStream(OpenNlpModelsPath + "en-ner-person.bin");
_tokenNameFinderModel = new TokenNameFinderModel(modelInputStream);
modelInputStream.close();
_nameFinder = new NameFinderME(_tokenNameFinderModel);
Console.WriteLine("Done.");
// OpenNLP POS tagger:
Console.Write("Loading OpenNLP POS tagger.... ");
modelInputStream = new java.io.FileInputStream(OpenNlpModelsPath + "en-pos-maxent.bin");
_posModel = new POSModel(modelInputStream);
modelInputStream.close();
_tagger = new POSTaggerME(_posModel);
Console.WriteLine("Done.");
// OpenNLP chunker:
Console.Write("Loading OpenNLP chunker.... ");
modelInputStream = new java.io.FileInputStream(OpenNlpModelsPath + "en-chunker.bin");
_chunkerModel = new ChunkerModel(modelInputStream);
modelInputStream.close();
_chunker = new ChunkerME(_chunkerModel);
Console.WriteLine("Done.");
// OpenNLP parser:
if (_loadParser)
{
Console.Write("Loading OpenNLP parser.... ");
modelInputStream = new java.io.FileInputStream(OpenNlpModelsPath + "en-parser-chunking.bin");
_parserModel = new ParserModel(modelInputStream);
modelInputStream.close();
_parser = ParserFactory.create(_parserModel);
Console.WriteLine("Done.");
}
// Stanford parser:
//_stanfordParser = new LexicalizedParser(GrammarPath + "englishPCFG.ser.gz"); // Obsolete method
_stanfordParser = LexicalizedParser.loadModel(GrammarPath + "englishPCFG.ser.gz");
// Porter stemmer:
_porterStemmer = new PorterStemmer();
}
protected internal abstract ParserModel trainAndUpdate(ParserModel originalModel, ObjectStream <Parse> parseSamples, ModelUpdaterParams parameters);
