public static Triple <string, IFileFilter, double> GetWeightedTreebankDescription(string[] args, int argIndex, string flag)
{
string path = null;
IFileFilter filter = null;
double weight = 1.0;
// the next arguments are the treebank path and maybe the range for testing
int numSubArgs = NumSubArgs(args, argIndex);
if (numSubArgs > 0 && numSubArgs < 4)
{
argIndex++;
path = args[argIndex++];
bool hasWeight = false;
if (numSubArgs > 1 && DoublePattern.Matcher(args[argIndex + numSubArgs - 2]).Matches())
{
weight = double.Parse(args[argIndex + numSubArgs - 2]);
hasWeight = true;
numSubArgs--;
}
if (numSubArgs == 2)
{
filter = new NumberRangesFileFilter(args[argIndex++], true);
}
else
{
if (numSubArgs == 3)
{
try
{
int low = System.Convert.ToInt32(args[argIndex]);
int high = System.Convert.ToInt32(args[argIndex + 1]);
filter = new NumberRangeFileFilter(low, high, true);
argIndex += 2;
}
catch (NumberFormatException)
{
// maybe it's a ranges expression?
filter = new NumberRangesFileFilter(args[argIndex++], true);
}
}
}
if (hasWeight)
{
argIndex++;
}
}
else
{
throw new ArgumentException("Bad arguments after " + flag);
}
return(Triple.MakeTriple(path, filter, weight));
}
/// <summary>
/// This method lets you train and test a segmenter relative to a
/// Treebank.
/// </summary>
/// <remarks>
/// This method lets you train and test a segmenter relative to a
/// Treebank.
/// <p>
/// <i>Implementation note:</i> This method is largely cloned from
/// LexicalizedParser's main method. Should we try to have it be able
/// to train segmenters to stop things going out of sync?
/// </remarks>
public static void Main(string[] args)
{
bool train = false;
bool saveToSerializedFile = false;
bool saveToTextFile = false;
string serializedInputFileOrUrl = null;
string textInputFileOrUrl = null;
string serializedOutputFileOrUrl = null;
string textOutputFileOrUrl = null;
string treebankPath = null;
Treebank testTreebank = null;
// Treebank tuneTreebank = null;
string testPath = null;
IFileFilter testFilter = null;
IFileFilter trainFilter = null;
string encoding = null;
// variables needed to process the files to be parsed
ITokenizerFactory <Word> tokenizerFactory = null;
// DocumentPreprocessor documentPreprocessor = new DocumentPreprocessor();
bool tokenized = false;
// whether or not the input file has already been tokenized
IFunction <IList <IHasWord>, IList <IHasWord> > escaper = new ChineseEscaper();
// int tagDelimiter = -1;
// String sentenceDelimiter = "\n";
// boolean fromXML = false;
int argIndex = 0;
if (args.Length < 1)
{
log.Info("usage: java edu.stanford.nlp.parser.lexparser." + "LexicalizedParser parserFileOrUrl filename*");
return;
}
Options op = new Options();
op.tlpParams = new ChineseTreebankParserParams();
// while loop through option arguments
while (argIndex < args.Length && args[argIndex][0] == '-')
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-train"))
{
train = true;
saveToSerializedFile = true;
int numSubArgs = NumSubArgs(args, argIndex);
argIndex++;
if (numSubArgs > 1)
{
treebankPath = args[argIndex];
argIndex++;
}
else
{
throw new Exception("Error: -train option must have treebankPath as first argument.");
}
if (numSubArgs == 2)
{
trainFilter = new NumberRangesFileFilter(args[argIndex++], true);
}
else
{
if (numSubArgs >= 3)
{
try
{
int low = System.Convert.ToInt32(args[argIndex]);
int high = System.Convert.ToInt32(args[argIndex + 1]);
trainFilter = new NumberRangeFileFilter(low, high, true);
argIndex += 2;
}
catch (NumberFormatException)
{
// maybe it's a ranges expression?
trainFilter = new NumberRangesFileFilter(args[argIndex], true);
argIndex++;
}
}
}
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-encoding"))
{
// sets encoding for TreebankLangParserParams
encoding = args[argIndex + 1];
op.tlpParams.SetInputEncoding(encoding);
op.tlpParams.SetOutputEncoding(encoding);
argIndex += 2;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-loadFromSerializedFile"))
{
// load the parser from a binary serialized file
// the next argument must be the path to the parser file
serializedInputFileOrUrl = args[argIndex + 1];
argIndex += 2;
}
else
{
// doesn't make sense to load from TextFile -pichuan
// } else if (args[argIndex].equalsIgnoreCase("-loadFromTextFile")) {
// // load the parser from declarative text file
// // the next argument must be the path to the parser file
// textInputFileOrUrl = args[argIndex + 1];
// argIndex += 2;
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-saveToSerializedFile"))
{
saveToSerializedFile = true;
serializedOutputFileOrUrl = args[argIndex + 1];
argIndex += 2;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-saveToTextFile"))
{
// save the parser to declarative text file
saveToTextFile = true;
textOutputFileOrUrl = args[argIndex + 1];
argIndex += 2;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-treebank"))
{
// the next argument is the treebank path and range for testing
int numSubArgs = NumSubArgs(args, argIndex);
argIndex++;
if (numSubArgs == 1)
{
testFilter = new NumberRangesFileFilter(args[argIndex++], true);
}
else
{
if (numSubArgs > 1)
{
testPath = args[argIndex++];
if (numSubArgs == 2)
{
testFilter = new NumberRangesFileFilter(args[argIndex++], true);
}
else
{
if (numSubArgs >= 3)
{
try
{
int low = System.Convert.ToInt32(args[argIndex]);
int high = System.Convert.ToInt32(args[argIndex + 1]);
testFilter = new NumberRangeFileFilter(low, high, true);
argIndex += 2;
}
catch (NumberFormatException)
{
// maybe it's a ranges expression?
testFilter = new NumberRangesFileFilter(args[argIndex++], true);
}
}
}
}
}
}
else
{
int j = op.tlpParams.SetOptionFlag(args, argIndex);
if (j == argIndex)
{
log.Info("Unknown option ignored: " + args[argIndex]);
j++;
}
argIndex = j;
}
}
}
}
}
}
}
// end while loop through arguments
ITreebankLangParserParams tlpParams = op.tlpParams;
// all other arguments are order dependent and
// are processed in order below
Edu.Stanford.Nlp.Parser.Lexparser.ChineseLexiconAndWordSegmenter cs = null;
if (!train && op.testOptions.verbose)
{
System.Console.Out.WriteLine("Currently " + new DateTime());
PrintArgs(args, System.Console.Out);
}
if (train)
{
PrintArgs(args, System.Console.Out);
// so we train a parser using the treebank
if (treebankPath == null)
{
// the next arg must be the treebank path, since it wasn't give earlier
treebankPath = args[argIndex];
argIndex++;
if (args.Length > argIndex + 1)
{
try
{
// the next two args might be the range
int low = System.Convert.ToInt32(args[argIndex]);
int high = System.Convert.ToInt32(args[argIndex + 1]);
trainFilter = new NumberRangeFileFilter(low, high, true);
argIndex += 2;
}
catch (NumberFormatException)
{
// maybe it's a ranges expression?
trainFilter = new NumberRangesFileFilter(args[argIndex], true);
argIndex++;
}
}
}
Treebank trainTreebank = MakeTreebank(treebankPath, op, trainFilter);
IIndex <string> wordIndex = new HashIndex <string>();
IIndex <string> tagIndex = new HashIndex <string>();
cs = new Edu.Stanford.Nlp.Parser.Lexparser.ChineseLexiconAndWordSegmenter(trainTreebank, op, wordIndex, tagIndex);
}
else
{
if (textInputFileOrUrl != null)
{
}
else
{
// so we load the segmenter from a text grammar file
// XXXXX fix later -pichuan
//cs = new LexicalizedParser(textInputFileOrUrl, true, op);
// so we load a serialized segmenter
if (serializedInputFileOrUrl == null)
{
// the next argument must be the path to the serialized parser
serializedInputFileOrUrl = args[argIndex];
argIndex++;
}
try
{
cs = new Edu.Stanford.Nlp.Parser.Lexparser.ChineseLexiconAndWordSegmenter(serializedInputFileOrUrl, op);
}
catch (ArgumentException)
{
log.Info("Error loading segmenter, exiting...");
System.Environment.Exit(0);
}
}
}
// the following has to go after reading parser to make sure
// op and tlpParams are the same for train and test
TreePrint treePrint = op.testOptions.TreePrint(tlpParams);
if (testFilter != null)
{
if (testPath == null)
{
if (treebankPath == null)
{
throw new Exception("No test treebank path specified...");
}
else
{
log.Info("No test treebank path specified. Using train path: \"" + treebankPath + "\"");
testPath = treebankPath;
}
}
testTreebank = tlpParams.TestMemoryTreebank();
testTreebank.LoadPath(testPath, testFilter);
}
op.trainOptions.sisterSplitters = Generics.NewHashSet(Arrays.AsList(tlpParams.SisterSplitters()));
// at this point we should be sure that op.tlpParams is
// set appropriately (from command line, or from grammar file),
// and will never change again. We also set the tlpParams of the
// LexicalizedParser instance to be the same object. This is
// redundancy that we probably should take out eventually.
//
// -- Roger
if (op.testOptions.verbose)
{
log.Info("Lexicon is " + cs.GetType().FullName);
}
PrintWriter pwOut = tlpParams.Pw();
PrintWriter pwErr = tlpParams.Pw(System.Console.Error);
// Now what do we do with the parser we've made
if (saveToTextFile)
{
// save the parser to textGrammar format
if (textOutputFileOrUrl != null)
{
SaveSegmenterDataToText(cs, textOutputFileOrUrl);
}
else
{
log.Info("Usage: must specify a text segmenter data output path");
}
}
if (saveToSerializedFile)
{
if (serializedOutputFileOrUrl == null && argIndex < args.Length)
{
// the next argument must be the path to serialize to
serializedOutputFileOrUrl = args[argIndex];
argIndex++;
}
if (serializedOutputFileOrUrl != null)
{
SaveSegmenterDataToSerialized(cs, serializedOutputFileOrUrl);
}
else
{
if (textOutputFileOrUrl == null && testTreebank == null)
{
// no saving/parsing request has been specified
log.Info("usage: " + "java edu.stanford.nlp.parser.lexparser.ChineseLexiconAndWordSegmenter" + "-train trainFilesPath [start stop] serializedParserFilename");
}
}
}
/* --------------------- Testing part!!!! ----------------------- */
if (op.testOptions.verbose)
{
}
// printOptions(false, op);
if (testTreebank != null || (argIndex < args.Length && Sharpen.Runtime.EqualsIgnoreCase(args[argIndex], "-treebank")))
{
// test parser on treebank
if (testTreebank == null)
{
// the next argument is the treebank path and range for testing
testTreebank = tlpParams.TestMemoryTreebank();
if (args.Length < argIndex + 4)
{
testTreebank.LoadPath(args[argIndex + 1]);
}
else
{
int testlow = System.Convert.ToInt32(args[argIndex + 2]);
int testhigh = System.Convert.ToInt32(args[argIndex + 3]);
testTreebank.LoadPath(args[argIndex + 1], new NumberRangeFileFilter(testlow, testhigh, true));
}
}
}
}
/// <summary>Lets you test out the TreeAnnotatorAndBinarizer on the command line.</summary>
/// <param name="args">
/// Command line arguments: All flags accepted by FactoredParser.setOptionFlag
/// and -train treebankPath [fileRanges]
/// </param>
public static void Main(string[] args)
{
Options op = new Options();
string treebankPath = null;
IFileFilter trainFilter = null;
int i = 0;
while (i < args.Length && args[i].StartsWith("-"))
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-train"))
{
int numSubArgs = NumSubArgs(args, i);
i++;
if (numSubArgs >= 1)
{
treebankPath = args[i];
i++;
}
else
{
throw new Exception("Error: -train option must have treebankPath as first argument.");
}
if (numSubArgs == 2)
{
trainFilter = new NumberRangesFileFilter(args[i++], true);
}
else
{
if (numSubArgs >= 3)
{
int low = System.Convert.ToInt32(args[i]);
int high = System.Convert.ToInt32(args[i + 1]);
trainFilter = new NumberRangeFileFilter(low, high, true);
i += 2;
}
}
}
else
{
i = op.SetOption(args, i);
}
}
if (i < args.Length)
{
log.Info("usage: java TreeAnnotatorAndBinarizer options*");
log.Info(" Options are like for lexicalized parser including -train treebankPath fileRange]");
return;
}
log.Info("Annotating from treebank dir: " + treebankPath);
Treebank trainTreebank = op.tlpParams.DiskTreebank();
if (trainFilter == null)
{
trainTreebank.LoadPath(treebankPath);
}
else
{
trainTreebank.LoadPath(treebankPath, trainFilter);
}
Treebank binaryTrainTreebank = GetAnnotatedBinaryTreebankFromTreebank(trainTreebank, null, null, op).First();
IEnumerator <Tree> it = trainTreebank.GetEnumerator();
foreach (Tree t in binaryTrainTreebank)
{
System.Console.Out.WriteLine("Original tree:");
it.Current.PennPrint();
System.Console.Out.WriteLine("Binarized tree:");
t.PennPrint();
System.Console.Out.WriteLine();
}
}