public virtual void TestAllocationFileParsing()
{
Configuration conf = new Configuration();
conf.Set(FairSchedulerConfiguration.AllocationFile, AllocFile);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
PrintWriter @out = new PrintWriter(new FileWriter(AllocFile));
@out.WriteLine("<?xml version=\"1.0\"?>");
@out.WriteLine("<allocations>");
// Give queue A a minimum of 1024 M
@out.WriteLine("<queue name=\"queueA\">");
@out.WriteLine("<minResources>1024mb,0vcores</minResources>");
@out.WriteLine("</queue>");
// Give queue B a minimum of 2048 M
@out.WriteLine("<queue name=\"queueB\">");
@out.WriteLine("<minResources>2048mb,0vcores</minResources>");
@out.WriteLine("<aclAdministerApps>alice,bob admins</aclAdministerApps>");
@out.WriteLine("<schedulingPolicy>fair</schedulingPolicy>");
@out.WriteLine("</queue>");
// Give queue C no minimum
@out.WriteLine("<queue name=\"queueC\">");
@out.WriteLine("<aclSubmitApps>alice,bob admins</aclSubmitApps>");
@out.WriteLine("</queue>");
// Give queue D a limit of 3 running apps and 0.4f maxAMShare
@out.WriteLine("<queue name=\"queueD\">");
@out.WriteLine("<maxRunningApps>3</maxRunningApps>");
@out.WriteLine("<maxAMShare>0.4</maxAMShare>");
@out.WriteLine("</queue>");
// Give queue E a preemption timeout of one minute
@out.WriteLine("<queue name=\"queueE\">");
@out.WriteLine("<minSharePreemptionTimeout>60</minSharePreemptionTimeout>");
@out.WriteLine("</queue>");
//Make queue F a parent queue without configured leaf queues using the 'type' attribute
@out.WriteLine("<queue name=\"queueF\" type=\"parent\" >");
@out.WriteLine("</queue>");
// Create hierarchical queues G,H, with different min/fair share preemption
// timeouts and preemption thresholds
@out.WriteLine("<queue name=\"queueG\">");
@out.WriteLine("<fairSharePreemptionTimeout>120</fairSharePreemptionTimeout>");
@out.WriteLine("<minSharePreemptionTimeout>50</minSharePreemptionTimeout>");
@out.WriteLine("<fairSharePreemptionThreshold>0.6</fairSharePreemptionThreshold>"
);
@out.WriteLine(" <queue name=\"queueH\">");
@out.WriteLine(" <fairSharePreemptionTimeout>180</fairSharePreemptionTimeout>");
@out.WriteLine(" <minSharePreemptionTimeout>40</minSharePreemptionTimeout>");
@out.WriteLine(" <fairSharePreemptionThreshold>0.7</fairSharePreemptionThreshold>"
);
@out.WriteLine(" </queue>");
@out.WriteLine("</queue>");
// Set default limit of apps per queue to 15
@out.WriteLine("<queueMaxAppsDefault>15</queueMaxAppsDefault>");
// Set default limit of apps per user to 5
@out.WriteLine("<userMaxAppsDefault>5</userMaxAppsDefault>");
// Set default limit of AMResourceShare to 0.5f
@out.WriteLine("<queueMaxAMShareDefault>0.5f</queueMaxAMShareDefault>");
// Give user1 a limit of 10 jobs
@out.WriteLine("<user name=\"user1\">");
@out.WriteLine("<maxRunningApps>10</maxRunningApps>");
@out.WriteLine("</user>");
// Set default min share preemption timeout to 2 minutes
@out.WriteLine("<defaultMinSharePreemptionTimeout>120" + "</defaultMinSharePreemptionTimeout>"
);
// Set default fair share preemption timeout to 5 minutes
@out.WriteLine("<defaultFairSharePreemptionTimeout>300</defaultFairSharePreemptionTimeout>"
);
// Set default fair share preemption threshold to 0.4
@out.WriteLine("<defaultFairSharePreemptionThreshold>0.4</defaultFairSharePreemptionThreshold>"
);
// Set default scheduling policy to DRF
@out.WriteLine("<defaultQueueSchedulingPolicy>drf</defaultQueueSchedulingPolicy>"
);
@out.WriteLine("</allocations>");
@out.Close();
allocLoader.Init(conf);
TestAllocationFileLoaderService.ReloadListener confHolder = new TestAllocationFileLoaderService.ReloadListener
(this);
allocLoader.SetReloadListener(confHolder);
allocLoader.ReloadAllocations();
AllocationConfiguration queueConf = confHolder.allocConf;
NUnit.Framework.Assert.AreEqual(6, queueConf.GetConfiguredQueues()[FSQueueType.Leaf
].Count);
NUnit.Framework.Assert.AreEqual(Resources.CreateResource(0), queueConf.GetMinResources
("root." + YarnConfiguration.DefaultQueueName));
NUnit.Framework.Assert.AreEqual(Resources.CreateResource(0), queueConf.GetMinResources
("root." + YarnConfiguration.DefaultQueueName));
NUnit.Framework.Assert.AreEqual(Resources.CreateResource(1024, 0), queueConf.GetMinResources
("root.queueA"));
NUnit.Framework.Assert.AreEqual(Resources.CreateResource(2048, 0), queueConf.GetMinResources
("root.queueB"));
NUnit.Framework.Assert.AreEqual(Resources.CreateResource(0), queueConf.GetMinResources
("root.queueC"));
NUnit.Framework.Assert.AreEqual(Resources.CreateResource(0), queueConf.GetMinResources
("root.queueD"));
NUnit.Framework.Assert.AreEqual(Resources.CreateResource(0), queueConf.GetMinResources
("root.queueE"));
NUnit.Framework.Assert.AreEqual(15, queueConf.GetQueueMaxApps("root." + YarnConfiguration
.DefaultQueueName));
NUnit.Framework.Assert.AreEqual(15, queueConf.GetQueueMaxApps("root.queueA"));
NUnit.Framework.Assert.AreEqual(15, queueConf.GetQueueMaxApps("root.queueB"));
NUnit.Framework.Assert.AreEqual(15, queueConf.GetQueueMaxApps("root.queueC"));
NUnit.Framework.Assert.AreEqual(3, queueConf.GetQueueMaxApps("root.queueD"));
NUnit.Framework.Assert.AreEqual(15, queueConf.GetQueueMaxApps("root.queueE"));
NUnit.Framework.Assert.AreEqual(10, queueConf.GetUserMaxApps("user1"));
NUnit.Framework.Assert.AreEqual(5, queueConf.GetUserMaxApps("user2"));
NUnit.Framework.Assert.AreEqual(.5f, queueConf.GetQueueMaxAMShare("root." + YarnConfiguration
.DefaultQueueName), 0.01);
NUnit.Framework.Assert.AreEqual(.5f, queueConf.GetQueueMaxAMShare("root.queueA"),
0.01);
NUnit.Framework.Assert.AreEqual(.5f, queueConf.GetQueueMaxAMShare("root.queueB"),
0.01);
NUnit.Framework.Assert.AreEqual(.5f, queueConf.GetQueueMaxAMShare("root.queueC"),
0.01);
NUnit.Framework.Assert.AreEqual(.4f, queueConf.GetQueueMaxAMShare("root.queueD"),
0.01);
NUnit.Framework.Assert.AreEqual(.5f, queueConf.GetQueueMaxAMShare("root.queueE"),
0.01);
// Root should get * ACL
NUnit.Framework.Assert.AreEqual("*", queueConf.GetQueueAcl("root", QueueACL.AdministerQueue
).GetAclString());
NUnit.Framework.Assert.AreEqual("*", queueConf.GetQueueAcl("root", QueueACL.SubmitApplications
).GetAclString());
// Unspecified queues should get default ACL
NUnit.Framework.Assert.AreEqual(" ", queueConf.GetQueueAcl("root.queueA", QueueACL
.AdministerQueue).GetAclString());
NUnit.Framework.Assert.AreEqual(" ", queueConf.GetQueueAcl("root.queueA", QueueACL
.SubmitApplications).GetAclString());
// Queue B ACL
NUnit.Framework.Assert.AreEqual("alice,bob admins", queueConf.GetQueueAcl("root.queueB"
, QueueACL.AdministerQueue).GetAclString());
// Queue C ACL
NUnit.Framework.Assert.AreEqual("alice,bob admins", queueConf.GetQueueAcl("root.queueC"
, QueueACL.SubmitApplications).GetAclString());
NUnit.Framework.Assert.AreEqual(120000, queueConf.GetMinSharePreemptionTimeout("root"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetMinSharePreemptionTimeout("root."
+ YarnConfiguration.DefaultQueueName));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetMinSharePreemptionTimeout("root.queueA"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetMinSharePreemptionTimeout("root.queueB"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetMinSharePreemptionTimeout("root.queueC"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetMinSharePreemptionTimeout("root.queueD"
));
NUnit.Framework.Assert.AreEqual(60000, queueConf.GetMinSharePreemptionTimeout("root.queueE"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetMinSharePreemptionTimeout("root.queueF"
));
NUnit.Framework.Assert.AreEqual(50000, queueConf.GetMinSharePreemptionTimeout("root.queueG"
));
NUnit.Framework.Assert.AreEqual(40000, queueConf.GetMinSharePreemptionTimeout("root.queueG.queueH"
));
NUnit.Framework.Assert.AreEqual(300000, queueConf.GetFairSharePreemptionTimeout("root"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root."
+ YarnConfiguration.DefaultQueueName));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root.queueA"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root.queueB"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root.queueC"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root.queueD"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root.queueE"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root.queueF"
));
NUnit.Framework.Assert.AreEqual(120000, queueConf.GetFairSharePreemptionTimeout("root.queueG"
));
NUnit.Framework.Assert.AreEqual(180000, queueConf.GetFairSharePreemptionTimeout("root.queueG.queueH"
));
NUnit.Framework.Assert.AreEqual(.4f, queueConf.GetFairSharePreemptionThreshold("root"
), 0.01);
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionThreshold("root."
+ YarnConfiguration.DefaultQueueName), 0.01);
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionThreshold("root.queueA"
), 0.01);
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionThreshold("root.queueB"
), 0.01);
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionThreshold("root.queueC"
), 0.01);
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionThreshold("root.queueD"
), 0.01);
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionThreshold("root.queueE"
), 0.01);
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionThreshold("root.queueF"
), 0.01);
NUnit.Framework.Assert.AreEqual(.6f, queueConf.GetFairSharePreemptionThreshold("root.queueG"
), 0.01);
NUnit.Framework.Assert.AreEqual(.7f, queueConf.GetFairSharePreemptionThreshold("root.queueG.queueH"
), 0.01);
NUnit.Framework.Assert.IsTrue(queueConf.GetConfiguredQueues()[FSQueueType.Parent]
.Contains("root.queueF"));
NUnit.Framework.Assert.IsTrue(queueConf.GetConfiguredQueues()[FSQueueType.Parent]
.Contains("root.queueG"));
NUnit.Framework.Assert.IsTrue(queueConf.GetConfiguredQueues()[FSQueueType.Leaf].Contains
("root.queueG.queueH"));
// Verify existing queues have default scheduling policy
NUnit.Framework.Assert.AreEqual(DominantResourceFairnessPolicy.Name, queueConf.GetSchedulingPolicy
("root").GetName());
NUnit.Framework.Assert.AreEqual(DominantResourceFairnessPolicy.Name, queueConf.GetSchedulingPolicy
("root.queueA").GetName());
// Verify default is overriden if specified explicitly
NUnit.Framework.Assert.AreEqual(FairSharePolicy.Name, queueConf.GetSchedulingPolicy
("root.queueB").GetName());
// Verify new queue gets default scheduling policy
NUnit.Framework.Assert.AreEqual(DominantResourceFairnessPolicy.Name, queueConf.GetSchedulingPolicy
("root.newqueue").GetName());
}
public override void Build()
{
foreach (File path in pathsToData)
{
treebank.LoadPath(path, treeFileExtension, false);
}
PrintWriter outfile = null;
PrintWriter flatFile = null;
try
{
outfile = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outFileName), "UTF-8")));
flatFile = (makeFlatFile) ? new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(flatFileName), "UTF-8"))) : null;
outputFileList.Add(outFileName);
if (makeFlatFile)
{
outputFileList.Add(flatFileName);
toStringBuffer.Append(" Made flat files\n");
}
PreprocessMWEs();
IList <TregexPattern> badTrees = new List <TregexPattern>();
//These trees appear in the Candito training set
//They are mangled by the TreeCorrector, so discard them ahead of time.
badTrees.Add(TregexPattern.Compile("@SENT <: @PUNC"));
badTrees.Add(TregexPattern.Compile("@SENT <1 @PUNC <2 @PUNC !<3 __"));
//wsg2011: This filters out tree #552 in the Candito test set. We saved this tree for the
//EMNLP2011 paper, but since it consists entirely of punctuation, it won't be evaluated anyway.
//Since we aren't doing the split in this data set, just remove the tree.
badTrees.Add(TregexPattern.Compile("@SENT <1 @PUNC <2 @PUNC <3 @PUNC <4 @PUNC !<5 __"));
foreach (Tree t in treebank)
{
//Filter out bad trees
bool skipTree = false;
foreach (TregexPattern p in badTrees)
{
skipTree = p.Matcher(t).Find();
if (skipTree)
{
break;
}
}
if (skipTree)
{
log.Info("Discarding tree: " + t.ToString());
continue;
}
// Filter out trees that aren't in this part of the split
if (splitSet != null)
{
string canditoTreeID = GetCanditoTreeID(t);
if (!splitSet.Contains(canditoTreeID))
{
continue;
}
}
if (customTreeVisitor != null)
{
customTreeVisitor.VisitTree(t);
}
// outfile.printf("%s\t%s%n",treeName,t.toString());
outfile.Println(t.ToString());
if (makeFlatFile)
{
string flatString = (removeEscapeTokens) ? ATBTreeUtils.UnEscape(ATBTreeUtils.FlattenTree(t)) : ATBTreeUtils.FlattenTree(t);
flatFile.Println(flatString);
}
}
}
catch (UnsupportedEncodingException e)
{
System.Console.Error.Printf("%s: Filesystem does not support UTF-8 output%n", this.GetType().FullName);
Sharpen.Runtime.PrintStackTrace(e);
}
catch (FileNotFoundException)
{
System.Console.Error.Printf("%s: Could not open %s for writing%n", this.GetType().FullName, outFileName);
}
catch (TregexParseException e)
{
System.Console.Error.Printf("%s: Could not compile Tregex expressions%n", this.GetType().FullName);
Sharpen.Runtime.PrintStackTrace(e);
}
finally
{
if (outfile != null)
{
outfile.Close();
}
if (flatFile != null)
{
flatFile.Close();
}
}
}
public virtual void TestContainerSetup()
{
containerManager.Start();
// ////// Create the resources for the container
FilePath dir = new FilePath(tmpDir, "dir");
dir.Mkdirs();
FilePath file = new FilePath(dir, "file");
PrintWriter fileWriter = new PrintWriter(file);
fileWriter.Write("Hello World!");
fileWriter.Close();
// ////// Construct the Container-id
ContainerId cId = CreateContainerId(0);
// ////// Construct the container-spec.
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(file.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(file.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete);
// Now ascertain that the resources are localised correctly.
ApplicationId appId = cId.GetApplicationAttemptId().GetApplicationId();
string appIDStr = ConverterUtils.ToString(appId);
string containerIDStr = ConverterUtils.ToString(cId);
FilePath userCacheDir = new FilePath(localDir, ContainerLocalizer.Usercache);
FilePath userDir = new FilePath(userCacheDir, user);
FilePath appCache = new FilePath(userDir, ContainerLocalizer.Appcache);
FilePath appDir = new FilePath(appCache, appIDStr);
FilePath containerDir = new FilePath(appDir, containerIDStr);
FilePath targetFile = new FilePath(containerDir, destinationFile);
FilePath sysDir = new FilePath(localDir, ResourceLocalizationService.NmPrivateDir
);
FilePath appSysDir = new FilePath(sysDir, appIDStr);
FilePath containerSysDir = new FilePath(appSysDir, containerIDStr);
foreach (FilePath f in new FilePath[] { localDir, sysDir, userCacheDir, appDir, appSysDir
, containerDir, containerSysDir })
{
NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath() + " doesn't exist!!", f.Exists(
));
NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath() + " is not a directory!!", f.IsDirectory
());
}
NUnit.Framework.Assert.IsTrue(targetFile.GetAbsolutePath() + " doesn't exist!!",
targetFile.Exists());
// Now verify the contents of the file
BufferedReader reader = new BufferedReader(new FileReader(targetFile));
NUnit.Framework.Assert.AreEqual("Hello World!", reader.ReadLine());
NUnit.Framework.Assert.AreEqual(null, reader.ReadLine());
}
/// <summary>
/// usage: java ChineseDocumentToSentenceProcessor [-segmentIBM]
/// -file filename [-encoding encoding]
/// <p>
/// The -segmentIBM option is for IBM GALE-specific splitting of an
/// XML element into sentences.
/// </summary>
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
//String encoding = "GB18030";
Properties props = StringUtils.ArgsToProperties(args);
// log.info("Here are the properties:");
// props.list(System.err);
bool alwaysAddS = props.Contains("alwaysAddS");
Edu.Stanford.Nlp.Process.ChineseDocumentToSentenceProcessor cp;
if (!props.Contains("file"))
{
log.Info("usage: java ChineseDocumentToSentenceProcessor [-segmentIBM] -file filename [-encoding encoding]");
return;
}
cp = new Edu.Stanford.Nlp.Process.ChineseDocumentToSentenceProcessor();
if (props.Contains("encoding"))
{
log.Info("WARNING: for now the default encoding is " + cp.encoding + ". It's not changeable for now");
}
string input = IOUtils.SlurpFileNoExceptions(props.GetProperty("file"), cp.encoding);
// String input = StringUtils.slurpGBURLNoExceptions(new URL(props.getProperty("file")));
if (props.Contains("segmentIBM"))
{
ITokenizer <Word> tok = WhitespaceTokenizer.NewWordWhitespaceTokenizer(new StringReader(input), true);
string parseInside = props.GetProperty("parseInside");
if (parseInside == null)
{
parseInside = string.Empty;
}
Pattern p1;
Pattern p2;
Pattern p3;
Pattern p4;
PrintWriter pw = new PrintWriter(new OutputStreamWriter(System.Console.Out, cp.encoding), true);
StringBuilder buff = new StringBuilder();
StringBuilder sgmlbuff = new StringBuilder();
string lastSgml = string.Empty;
p1 = Pattern.Compile("<.*>");
p2 = Pattern.Compile("\uFEFF?<[\\p{Alpha}]+");
p3 = Pattern.Compile("[A-Za-z0-9=\"]+>");
p4 = Pattern.Compile("<(?:" + parseInside + ")[ >]");
bool inSGML = false;
int splitItems = 0;
int numAdded = 0;
while (tok.MoveNext())
{
string s = tok.Current.Word();
// pw.println("The token is |" + s + "|");
if (p2.Matcher(s).Matches())
{
inSGML = true;
sgmlbuff.Append(s).Append(" ");
}
else
{
if (p1.Matcher(s).Matches() || inSGML && p3.Matcher(s).Matches() || "\n".Equals(s))
{
inSGML = false;
if (buff.ToString().Trim().Length > 0)
{
// pw.println("Dumping sentences");
// pw.println("Buff is " + buff);
bool processIt = false;
if (parseInside.Equals(string.Empty))
{
processIt = true;
}
else
{
if (p4.Matcher(lastSgml).Find())
{
processIt = true;
}
}
if (processIt)
{
IList <string> sents = Edu.Stanford.Nlp.Process.ChineseDocumentToSentenceProcessor.FromPlainText(buff.ToString(), true);
// pw.println("Sents is " + sents);
// pw.println();
if (alwaysAddS || sents.Count > 1)
{
int i = 1;
foreach (string str in sents)
{
pw.Print("<s id=\"" + i + "\">");
pw.Print(str);
pw.Println("</s>");
i++;
}
if (sents.Count > 1)
{
splitItems++;
numAdded += sents.Count - 1;
}
}
else
{
if (sents.Count == 1)
{
pw.Print(sents[0]);
}
}
}
else
{
pw.Print(buff);
}
buff = new StringBuilder();
}
sgmlbuff.Append(s);
// pw.println("sgmlbuff is " + sgmlbuff);
pw.Print(sgmlbuff);
lastSgml = sgmlbuff.ToString();
sgmlbuff = new StringBuilder();
}
else
{
if (inSGML)
{
sgmlbuff.Append(s).Append(" ");
}
else
{
buff.Append(s).Append(" ");
}
}
}
}
// pw.println("Buff is now |" + buff + "|");
// end while (tok.hasNext()) {
// empty remaining buffers
pw.Flush();
pw.Close();
log.Info("Split " + splitItems + " segments, adding " + numAdded + " sentences.");
}
else
{
IList <string> sent = Edu.Stanford.Nlp.Process.ChineseDocumentToSentenceProcessor.FromHTML(input);
PrintWriter pw = new PrintWriter(new OutputStreamWriter(System.Console.Error, cp.encoding), true);
foreach (string a in sent)
{
pw.Println(a);
}
}
}
public static void Main(string[] args)
{
if (args.Length < minArgs)
{
System.Console.Out.WriteLine(usage.ToString());
System.Environment.Exit(-1);
}
ITreebankLangParserParams tlpp = new EnglishTreebankParserParams();
DiskTreebank tb = null;
string encoding = "UTF-8";
string puncTag = null;
for (int i = 0; i < args.Length; i++)
{
if (args[i].StartsWith("-"))
{
switch (args[i])
{
case "-l":
{
Language lang = Language.ValueOf(args[++i].Trim());
tlpp = lang.@params;
break;
}
case "-e":
{
encoding = args[++i];
break;
}
default:
{
System.Console.Out.WriteLine(usage.ToString());
System.Environment.Exit(-1);
break;
}
}
}
else
{
puncTag = args[i++];
if (tb == null)
{
if (tlpp == null)
{
System.Console.Out.WriteLine(usage.ToString());
System.Environment.Exit(-1);
}
else
{
tlpp.SetInputEncoding(encoding);
tlpp.SetOutputEncoding(encoding);
tb = tlpp.DiskTreebank();
}
}
tb.LoadPath(args[i]);
}
}
ICounter <string> puncTypes = new ClassicCounter <string>();
foreach (Tree t in tb)
{
IList <CoreLabel> yield = t.TaggedLabeledYield();
foreach (CoreLabel word in yield)
{
if (word.Tag().Equals(puncTag))
{
puncTypes.IncrementCount(word.Word());
}
}
}
IList <string> biggestKeys = new List <string>(puncTypes.KeySet());
biggestKeys.Sort(Counters.ToComparatorDescending(puncTypes));
PrintWriter pw = tlpp.Pw();
foreach (string wordType in biggestKeys)
{
pw.Printf("%s\t%d%n", wordType, (int)puncTypes.GetCount(wordType));
}
pw.Close();
}
/// <exception cref="System.IO.IOException"/>
public virtual void Annotate(ICoreMap document)
{
// write input file in GUTime format
IElement inputXML = ToInputXML(document);
File inputFile = File.CreateTempFile("gutime", ".input");
//Document doc = new Document(inputXML);
PrintWriter inputWriter = new PrintWriter(inputFile);
inputWriter.Println(XMLUtils.NodeToString(inputXML, false));
// new XMLOutputter().output(inputXML, inputWriter);
inputWriter.Close();
bool useFirstDate = (!document.ContainsKey(typeof(CoreAnnotations.CalendarAnnotation)) && !document.ContainsKey(typeof(CoreAnnotations.DocDateAnnotation)));
List <string> args = new List <string>();
args.Add("perl");
args.Add("-I" + this.gutimePath.GetPath());
args.Add(new File(this.gutimePath, "TimeTag.pl").GetPath());
if (useFirstDate)
{
args.Add("-FDNW");
}
args.Add(inputFile.GetPath());
// run GUTime on the input file
ProcessBuilder process = new ProcessBuilder(args);
StringWriter outputWriter = new StringWriter();
SystemUtils.Run(process, outputWriter, null);
string output = outputWriter.GetBuffer().ToString();
Pattern docClose = Pattern.Compile("</DOC>.*", Pattern.Dotall);
output = docClose.Matcher(output).ReplaceAll("</DOC>");
//The TimeTag.pl result file contains next tags which must be removed
output = output.ReplaceAll("<lex.*?>", string.Empty);
output = output.Replace("</lex>", string.Empty);
output = output.Replace("<NG>", string.Empty);
output = output.Replace("</NG>", string.Empty);
output = output.Replace("<VG>", string.Empty);
output = output.Replace("</VG>", string.Empty);
output = output.Replace("<s>", string.Empty);
output = output.Replace("</s>", string.Empty);
// parse the GUTime output
IElement outputXML;
try
{
outputXML = XMLUtils.ParseElement(output);
}
catch (Exception ex)
{
throw new Exception(string.Format("error:\n%s\ninput:\n%s\noutput:\n%s", ex, IOUtils.SlurpFile(inputFile), output), ex);
}
/*
* try {
* outputXML = new SAXBuilder().build(new StringReader(output)).getRootElement();
* } catch (JDOMException e) {
* throw new RuntimeException(String.format("error:\n%s\ninput:\n%s\noutput:\n%s",
* e, IOUtils.slurpFile(inputFile), output));
* } */
inputFile.Delete();
// get Timex annotations
IList <ICoreMap> timexAnns = ToTimexCoreMaps(outputXML, document);
document.Set(typeof(TimeAnnotations.TimexAnnotations), timexAnns);
if (outputResults)
{
System.Console.Out.WriteLine(timexAnns);
}
// align Timex annotations to sentences
int timexIndex = 0;
foreach (ICoreMap sentence in document.Get(typeof(CoreAnnotations.SentencesAnnotation)))
{
int sentBegin = BeginOffset(sentence);
int sentEnd = EndOffset(sentence);
// skip times before the sentence
while (timexIndex < timexAnns.Count && BeginOffset(timexAnns[timexIndex]) < sentBegin)
{
++timexIndex;
}
// determine times within the sentence
int sublistBegin = timexIndex;
int sublistEnd = timexIndex;
while (timexIndex < timexAnns.Count && sentBegin <= BeginOffset(timexAnns[timexIndex]) && EndOffset(timexAnns[timexIndex]) <= sentEnd)
{
++sublistEnd;
++timexIndex;
}
// set the sentence timexes
sentence.Set(typeof(TimeAnnotations.TimexAnnotations), timexAnns.SubList(sublistBegin, sublistEnd));
}
}
/// <summary>Print some statistics about this lexicon.</summary>
public virtual void PrintLexStats()
{
System.Console.Out.WriteLine("BaseLexicon statistics");
System.Console.Out.WriteLine("unknownLevel is " + GetUnknownWordModel().GetUnknownLevel());
// System.out.println("Rules size: " + rules.size());
System.Console.Out.WriteLine("Sum of rulesWithWord: " + NumRules());
System.Console.Out.WriteLine("Tags size: " + tags.Count);
int wsize = words.Count;
System.Console.Out.WriteLine("Words size: " + wsize);
// System.out.println("Unseen Sigs size: " + sigs.size() +
// " [number of unknown equivalence classes]");
System.Console.Out.WriteLine("rulesWithWord length: " + rulesWithWord.Length + " [should be sum of words + unknown sigs]");
int[] lengths = new int[StatsBins];
List <string>[] wArr = new ArrayList[StatsBins];
for (int j = 0; j < StatsBins; j++)
{
wArr[j] = new List <string>();
}
for (int i = 0; i < rulesWithWord.Length; i++)
{
int num = rulesWithWord[i].Count;
if (num > StatsBins - 1)
{
num = StatsBins - 1;
}
lengths[num]++;
if (wsize <= 20 || num >= StatsBins / 2)
{
wArr[num].Add(wordIndex.Get(i));
}
}
System.Console.Out.WriteLine("Stats on how many taggings for how many words");
for (int j_1 = 0; j_1 < StatsBins; j_1++)
{
System.Console.Out.Write(j_1 + " taggings: " + lengths[j_1] + " words ");
if (wsize <= 20 || j_1 >= StatsBins / 2)
{
System.Console.Out.Write(wArr[j_1]);
}
System.Console.Out.WriteLine();
}
NumberFormat nf = NumberFormat.GetNumberInstance();
nf.SetMaximumFractionDigits(0);
System.Console.Out.WriteLine("Unseen counter: " + Counters.ToString(uwModel.UnSeenCounter(), nf));
if (wsize < 50 && tags.Count < 10)
{
nf.SetMaximumFractionDigits(3);
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
pw.Println("Tagging probabilities log P(word|tag)");
for (int t = 0; t < tags.Count; t++)
{
pw.Print('\t');
pw.Print(tagIndex.Get(t));
}
pw.Println();
for (int w = 0; w < wsize; w++)
{
pw.Print(wordIndex.Get(w));
pw.Print('\t');
for (int t_1 = 0; t_1 < tags.Count; t_1++)
{
IntTaggedWord iTW = new IntTaggedWord(w, t_1);
pw.Print(nf.Format(Score(iTW, 1, wordIndex.Get(w), null)));
if (t_1 == tags.Count - 1)
{
pw.Println();
}
else
{
pw.Print('\t');
}
}
}
pw.Close();
System.Console.Out.WriteLine(sw.ToString());
}
}
public virtual void TestContainerKillOnMemoryOverflow()
{
if (!ProcfsBasedProcessTree.IsAvailable())
{
return;
}
containerManager.Start();
FilePath scriptFile = new FilePath(tmpDir, "scriptFile.sh");
PrintWriter fileWriter = new PrintWriter(scriptFile);
FilePath processStartFile = new FilePath(tmpDir, "start_file.txt").GetAbsoluteFile
();
fileWriter.Write("\numask 0");
// So that start file is readable by the
// test.
fileWriter.Write("\necho Hello World! > " + processStartFile);
fileWriter.Write("\necho $$ >> " + processStartFile);
fileWriter.Write("\nsleep 15");
fileWriter.Close();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
// ////// Construct the Container-id
ApplicationId appId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 1);
ContainerId cId = ContainerId.NewContainerId(appAttemptId, 0);
int port = 12345;
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = new AList <string>();
commands.AddItem("/bin/bash");
commands.AddItem(scriptFile.GetAbsolutePath());
containerLaunchContext.SetCommands(commands);
Resource r = BuilderUtils.NewResource(8 * 1024 * 1024, 1);
ContainerTokenIdentifier containerIdentifier = new ContainerTokenIdentifier(cId,
context.GetNodeId().ToString(), user, r, Runtime.CurrentTimeMillis() + 120000, 123
, DummyRmIdentifier, Priority.NewInstance(0), 0);
Token containerToken = BuilderUtils.NewContainerToken(context.GetNodeId(), containerManager
.GetContext().GetContainerTokenSecretManager().CreatePassword(containerIdentifier
), containerIdentifier);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
int timeoutSecs = 0;
while (!processStartFile.Exists() && timeoutSecs++ < 20)
{
Sharpen.Thread.Sleep(1000);
Log.Info("Waiting for process start-file to be created");
}
NUnit.Framework.Assert.IsTrue("ProcessStartFile doesn't exist!", processStartFile
.Exists());
// Now verify the contents of the file
BufferedReader reader = new BufferedReader(new FileReader(processStartFile));
NUnit.Framework.Assert.AreEqual("Hello World!", reader.ReadLine());
// Get the pid of the process
string pid = reader.ReadLine().Trim();
// No more lines
NUnit.Framework.Assert.AreEqual(null, reader.ReadLine());
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete, 60);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
containerIds);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(gcsRequest
).GetContainerStatuses()[0];
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.KilledExceededVmem, containerStatus
.GetExitStatus());
string expectedMsgPattern = "Container \\[pid=" + pid + ",containerID=" + cId + "\\] is running beyond virtual memory limits. Current usage: "
+ "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B physical memory used; " + "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B virtual memory used. "
+ "Killing container.\nDump of the process-tree for " + cId + " :\n";
Sharpen.Pattern pat = Sharpen.Pattern.Compile(expectedMsgPattern);
NUnit.Framework.Assert.AreEqual("Expected message pattern is: " + expectedMsgPattern
+ "\n\nObserved message is: " + containerStatus.GetDiagnostics(), true, pat.Matcher
(containerStatus.GetDiagnostics()).Find());
// Assert that the process is not alive anymore
NUnit.Framework.Assert.IsFalse("Process is still alive!", exec.SignalContainer(user
, pid, ContainerExecutor.Signal.Null));
}
/// <summary>Process a GET request for the specified resource.</summary>
/// <param name="request">The servlet request we are processing</param>
/// <param name="response">The servlet response we are creating</param>
protected override void DoGet(HttpServletRequest request, HttpServletResponse response
)
{
string jsonpcb = null;
PrintWriter writer = null;
try
{
if (!IsInstrumentationAccessAllowed(request, response))
{
return;
}
JsonGenerator jg = null;
try
{
writer = response.GetWriter();
response.SetContentType("application/json; charset=utf8");
response.SetHeader(AccessControlAllowMethods, "GET");
response.SetHeader(AccessControlAllowOrigin, "*");
JsonFactory jsonFactory = new JsonFactory();
jg = jsonFactory.CreateJsonGenerator(writer);
jg.Disable(JsonGenerator.Feature.AutoCloseTarget);
jg.UseDefaultPrettyPrinter();
jg.WriteStartObject();
if (mBeanServer == null)
{
jg.WriteStringField("result", "ERROR");
jg.WriteStringField("message", "No MBeanServer could be found");
jg.Close();
Log.Error("No MBeanServer could be found.");
response.SetStatus(HttpServletResponse.ScNotFound);
return;
}
// query per mbean attribute
string getmethod = request.GetParameter("get");
if (getmethod != null)
{
string[] splitStrings = getmethod.Split("\\:\\:");
if (splitStrings.Length != 2)
{
jg.WriteStringField("result", "ERROR");
jg.WriteStringField("message", "query format is not as expected.");
jg.Close();
response.SetStatus(HttpServletResponse.ScBadRequest);
return;
}
ListBeans(jg, new ObjectName(splitStrings[0]), splitStrings[1], response);
jg.Close();
return;
}
// query per mbean
string qry = request.GetParameter("qry");
if (qry == null)
{
qry = "*:*";
}
ListBeans(jg, new ObjectName(qry), null, response);
}
finally
{
if (jg != null)
{
jg.Close();
}
if (writer != null)
{
writer.Close();
}
}
}
catch (IOException e)
{
Log.Error("Caught an exception while processing JMX request", e);
response.SetStatus(HttpServletResponse.ScInternalServerError);
}
catch (MalformedObjectNameException e)
{
Log.Error("Caught an exception while processing JMX request", e);
response.SetStatus(HttpServletResponse.ScBadRequest);
}
finally
{
if (writer != null)
{
writer.Close();
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestIsStarvedForFairShare()
{
conf.Set(FairSchedulerConfiguration.AllocationFile, AllocFile);
PrintWriter @out = new PrintWriter(new FileWriter(AllocFile));
@out.WriteLine("<?xml version=\"1.0\"?>");
@out.WriteLine("<allocations>");
@out.WriteLine("<queue name=\"queueA\">");
@out.WriteLine("<weight>.2</weight>");
@out.WriteLine("</queue>");
@out.WriteLine("<queue name=\"queueB\">");
@out.WriteLine("<weight>.8</weight>");
@out.WriteLine("<fairSharePreemptionThreshold>.4</fairSharePreemptionThreshold>");
@out.WriteLine("<queue name=\"queueB1\">");
@out.WriteLine("</queue>");
@out.WriteLine("<queue name=\"queueB2\">");
@out.WriteLine("<fairSharePreemptionThreshold>.6</fairSharePreemptionThreshold>");
@out.WriteLine("</queue>");
@out.WriteLine("</queue>");
@out.WriteLine("<defaultFairSharePreemptionThreshold>.5</defaultFairSharePreemptionThreshold>"
);
@out.WriteLine("</allocations>");
@out.Close();
resourceManager = new MockRM(conf);
resourceManager.Start();
scheduler = (FairScheduler)resourceManager.GetResourceScheduler();
// Add one big node (only care about aggregate capacity)
RMNode node1 = MockNodes.NewNodeInfo(1, Resources.CreateResource(10 * 1024, 10),
1, "127.0.0.1");
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.Handle(nodeEvent1);
scheduler.Update();
// Queue A wants 4 * 1024. Node update gives this all to A
CreateSchedulingRequest(1 * 1024, "queueA", "user1", 4);
scheduler.Update();
NodeUpdateSchedulerEvent nodeEvent2 = new NodeUpdateSchedulerEvent(node1);
for (int i = 0; i < 4; i++)
{
scheduler.Handle(nodeEvent2);
}
QueueManager queueMgr = scheduler.GetQueueManager();
FSLeafQueue queueA = queueMgr.GetLeafQueue("queueA", false);
NUnit.Framework.Assert.AreEqual(4 * 1024, queueA.GetResourceUsage().GetMemory());
// Both queue B1 and queue B2 want 3 * 1024
CreateSchedulingRequest(1 * 1024, "queueB.queueB1", "user1", 3);
CreateSchedulingRequest(1 * 1024, "queueB.queueB2", "user1", 3);
scheduler.Update();
for (int i_1 = 0; i_1 < 4; i_1++)
{
scheduler.Handle(nodeEvent2);
}
FSLeafQueue queueB1 = queueMgr.GetLeafQueue("queueB.queueB1", false);
FSLeafQueue queueB2 = queueMgr.GetLeafQueue("queueB.queueB2", false);
NUnit.Framework.Assert.AreEqual(2 * 1024, queueB1.GetResourceUsage().GetMemory());
NUnit.Framework.Assert.AreEqual(2 * 1024, queueB2.GetResourceUsage().GetMemory());
// For queue B1, the fairSharePreemptionThreshold is 0.4, and the fair share
// threshold is 1.6 * 1024
NUnit.Framework.Assert.IsFalse(queueB1.IsStarvedForFairShare());
// For queue B2, the fairSharePreemptionThreshold is 0.6, and the fair share
// threshold is 2.4 * 1024
NUnit.Framework.Assert.IsTrue(queueB2.IsStarvedForFairShare());
// Node checks in again
scheduler.Handle(nodeEvent2);
scheduler.Handle(nodeEvent2);
NUnit.Framework.Assert.AreEqual(3 * 1024, queueB1.GetResourceUsage().GetMemory());
NUnit.Framework.Assert.AreEqual(3 * 1024, queueB2.GetResourceUsage().GetMemory());
// Both queue B1 and queue B2 usages go to 3 * 1024
NUnit.Framework.Assert.IsFalse(queueB1.IsStarvedForFairShare());
NUnit.Framework.Assert.IsFalse(queueB2.IsStarvedForFairShare());
}
public virtual void TestPreemptionWithFreeResources()
{
PrintWriter @out = new PrintWriter(new FileWriter(AllocFile));
@out.WriteLine("<?xml version=\"1.0\"?>");
@out.WriteLine("<allocations>");
@out.WriteLine("<queue name=\"default\">");
@out.WriteLine("<maxResources>0mb,0vcores</maxResources>");
@out.WriteLine("</queue>");
@out.WriteLine("<queue name=\"queueA\">");
@out.WriteLine("<weight>1</weight>");
@out.WriteLine("<minResources>1024mb,0vcores</minResources>");
@out.WriteLine("</queue>");
@out.WriteLine("<queue name=\"queueB\">");
@out.WriteLine("<weight>1</weight>");
@out.WriteLine("<minResources>1024mb,0vcores</minResources>");
@out.WriteLine("</queue>");
@out.Write("<defaultMinSharePreemptionTimeout>5</defaultMinSharePreemptionTimeout>"
);
@out.Write("<fairSharePreemptionTimeout>10</fairSharePreemptionTimeout>");
@out.WriteLine("</allocations>");
@out.Close();
StartResourceManager(0f);
// Create node with 4GB memory and 4 vcores
RegisterNodeAndSubmitApp(4 * 1024, 4, 2, 1024);
// Verify submitting another request triggers preemption
CreateSchedulingRequest(1024, "queueB", "user1", 1, 1);
scheduler.Update();
clock.Tick(6);
((TestFairSchedulerPreemption.StubbedFairScheduler)scheduler).ResetLastPreemptResources
();
scheduler.PreemptTasksIfNecessary();
NUnit.Framework.Assert.AreEqual("preemptResources() should have been called", 1024
, ((TestFairSchedulerPreemption.StubbedFairScheduler)scheduler).lastPreemptMemory
);
resourceManager.Stop();
StartResourceManager(0.8f);
// Create node with 4GB memory and 4 vcores
RegisterNodeAndSubmitApp(4 * 1024, 4, 3, 1024);
// Verify submitting another request doesn't trigger preemption
CreateSchedulingRequest(1024, "queueB", "user1", 1, 1);
scheduler.Update();
clock.Tick(6);
((TestFairSchedulerPreemption.StubbedFairScheduler)scheduler).ResetLastPreemptResources
();
scheduler.PreemptTasksIfNecessary();
NUnit.Framework.Assert.AreEqual("preemptResources() should not have been called",
-1, ((TestFairSchedulerPreemption.StubbedFairScheduler)scheduler).lastPreemptMemory
);
resourceManager.Stop();
StartResourceManager(0.7f);
// Create node with 4GB memory and 4 vcores
RegisterNodeAndSubmitApp(4 * 1024, 4, 3, 1024);
// Verify submitting another request triggers preemption
CreateSchedulingRequest(1024, "queueB", "user1", 1, 1);
scheduler.Update();
clock.Tick(6);
((TestFairSchedulerPreemption.StubbedFairScheduler)scheduler).ResetLastPreemptResources
();
scheduler.PreemptTasksIfNecessary();
NUnit.Framework.Assert.AreEqual("preemptResources() should have been called", 1024
, ((TestFairSchedulerPreemption.StubbedFairScheduler)scheduler).lastPreemptMemory
);
}
/// <exception cref="System.Exception"/>
public virtual void TestReload()
{
PrintWriter @out = new PrintWriter(new FileWriter(AllocFile));
@out.WriteLine("<?xml version=\"1.0\"?>");
@out.WriteLine("<allocations>");
@out.WriteLine(" <queue name=\"queueA\">");
@out.WriteLine(" <maxRunningApps>1</maxRunningApps>");
@out.WriteLine(" </queue>");
@out.WriteLine(" <queue name=\"queueB\" />");
@out.WriteLine(" <queuePlacementPolicy>");
@out.WriteLine(" <rule name='default' />");
@out.WriteLine(" </queuePlacementPolicy>");
@out.WriteLine("</allocations>");
@out.Close();
TestAllocationFileLoaderService.MockClock clock = new TestAllocationFileLoaderService.MockClock
(this);
Configuration conf = new Configuration();
conf.Set(FairSchedulerConfiguration.AllocationFile, AllocFile);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(clock);
allocLoader.reloadIntervalMs = 5;
allocLoader.Init(conf);
TestAllocationFileLoaderService.ReloadListener confHolder = new TestAllocationFileLoaderService.ReloadListener
(this);
allocLoader.SetReloadListener(confHolder);
allocLoader.ReloadAllocations();
AllocationConfiguration allocConf = confHolder.allocConf;
// Verify conf
QueuePlacementPolicy policy = allocConf.GetPlacementPolicy();
IList <QueuePlacementRule> rules = policy.GetRules();
NUnit.Framework.Assert.AreEqual(1, rules.Count);
NUnit.Framework.Assert.AreEqual(typeof(QueuePlacementRule.Default), rules[0].GetType
());
NUnit.Framework.Assert.AreEqual(1, allocConf.GetQueueMaxApps("root.queueA"));
NUnit.Framework.Assert.AreEqual(2, allocConf.GetConfiguredQueues()[FSQueueType.Leaf
].Count);
NUnit.Framework.Assert.IsTrue(allocConf.GetConfiguredQueues()[FSQueueType.Leaf].Contains
("root.queueA"));
NUnit.Framework.Assert.IsTrue(allocConf.GetConfiguredQueues()[FSQueueType.Leaf].Contains
("root.queueB"));
confHolder.allocConf = null;
// Modify file and advance the clock
@out = new PrintWriter(new FileWriter(AllocFile));
@out.WriteLine("<?xml version=\"1.0\"?>");
@out.WriteLine("<allocations>");
@out.WriteLine(" <queue name=\"queueB\">");
@out.WriteLine(" <maxRunningApps>3</maxRunningApps>");
@out.WriteLine(" </queue>");
@out.WriteLine(" <queuePlacementPolicy>");
@out.WriteLine(" <rule name='specified' />");
@out.WriteLine(" <rule name='nestedUserQueue' >");
@out.WriteLine(" <rule name='primaryGroup' />");
@out.WriteLine(" </rule>");
@out.WriteLine(" <rule name='default' />");
@out.WriteLine(" </queuePlacementPolicy>");
@out.WriteLine("</allocations>");
@out.Close();
clock.Tick(Runtime.CurrentTimeMillis() + AllocationFileLoaderService.AllocReloadWaitMs
+ 10000);
allocLoader.Start();
while (confHolder.allocConf == null)
{
Sharpen.Thread.Sleep(20);
}
// Verify conf
allocConf = confHolder.allocConf;
policy = allocConf.GetPlacementPolicy();
rules = policy.GetRules();
NUnit.Framework.Assert.AreEqual(3, rules.Count);
NUnit.Framework.Assert.AreEqual(typeof(QueuePlacementRule.Specified), rules[0].GetType
());
NUnit.Framework.Assert.AreEqual(typeof(QueuePlacementRule.NestedUserQueue), rules
[1].GetType());
NUnit.Framework.Assert.AreEqual(typeof(QueuePlacementRule.PrimaryGroup), ((QueuePlacementRule.NestedUserQueue
)(rules[1])).nestedRule.GetType());
NUnit.Framework.Assert.AreEqual(typeof(QueuePlacementRule.Default), rules[2].GetType
());
NUnit.Framework.Assert.AreEqual(3, allocConf.GetQueueMaxApps("root.queueB"));
NUnit.Framework.Assert.AreEqual(1, allocConf.GetConfiguredQueues()[FSQueueType.Leaf
].Count);
NUnit.Framework.Assert.IsTrue(allocConf.GetConfiguredQueues()[FSQueueType.Leaf].Contains
("root.queueB"));
}
public virtual void TestBackwardsCompatibleAllocationFileParsing()
{
Configuration conf = new Configuration();
conf.Set(FairSchedulerConfiguration.AllocationFile, AllocFile);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
PrintWriter @out = new PrintWriter(new FileWriter(AllocFile));
@out.WriteLine("<?xml version=\"1.0\"?>");
@out.WriteLine("<allocations>");
// Give queue A a minimum of 1024 M
@out.WriteLine("<pool name=\"queueA\">");
@out.WriteLine("<minResources>1024mb,0vcores</minResources>");
@out.WriteLine("</pool>");
// Give queue B a minimum of 2048 M
@out.WriteLine("<pool name=\"queueB\">");
@out.WriteLine("<minResources>2048mb,0vcores</minResources>");
@out.WriteLine("<aclAdministerApps>alice,bob admins</aclAdministerApps>");
@out.WriteLine("</pool>");
// Give queue C no minimum
@out.WriteLine("<pool name=\"queueC\">");
@out.WriteLine("<aclSubmitApps>alice,bob admins</aclSubmitApps>");
@out.WriteLine("</pool>");
// Give queue D a limit of 3 running apps
@out.WriteLine("<pool name=\"queueD\">");
@out.WriteLine("<maxRunningApps>3</maxRunningApps>");
@out.WriteLine("</pool>");
// Give queue E a preemption timeout of one minute and 0.3f threshold
@out.WriteLine("<pool name=\"queueE\">");
@out.WriteLine("<minSharePreemptionTimeout>60</minSharePreemptionTimeout>");
@out.WriteLine("<fairSharePreemptionThreshold>0.3</fairSharePreemptionThreshold>"
);
@out.WriteLine("</pool>");
// Set default limit of apps per queue to 15
@out.WriteLine("<queueMaxAppsDefault>15</queueMaxAppsDefault>");
// Set default limit of apps per user to 5
@out.WriteLine("<userMaxAppsDefault>5</userMaxAppsDefault>");
// Give user1 a limit of 10 jobs
@out.WriteLine("<user name=\"user1\">");
@out.WriteLine("<maxRunningApps>10</maxRunningApps>");
@out.WriteLine("</user>");
// Set default min share preemption timeout to 2 minutes
@out.WriteLine("<defaultMinSharePreemptionTimeout>120" + "</defaultMinSharePreemptionTimeout>"
);
// Set fair share preemption timeout to 5 minutes
@out.WriteLine("<fairSharePreemptionTimeout>300</fairSharePreemptionTimeout>");
// Set default fair share preemption threshold to 0.6f
@out.WriteLine("<defaultFairSharePreemptionThreshold>0.6</defaultFairSharePreemptionThreshold>"
);
@out.WriteLine("</allocations>");
@out.Close();
allocLoader.Init(conf);
TestAllocationFileLoaderService.ReloadListener confHolder = new TestAllocationFileLoaderService.ReloadListener
(this);
allocLoader.SetReloadListener(confHolder);
allocLoader.ReloadAllocations();
AllocationConfiguration queueConf = confHolder.allocConf;
NUnit.Framework.Assert.AreEqual(5, queueConf.GetConfiguredQueues()[FSQueueType.Leaf
].Count);
NUnit.Framework.Assert.AreEqual(Resources.CreateResource(0), queueConf.GetMinResources
("root." + YarnConfiguration.DefaultQueueName));
NUnit.Framework.Assert.AreEqual(Resources.CreateResource(0), queueConf.GetMinResources
("root." + YarnConfiguration.DefaultQueueName));
NUnit.Framework.Assert.AreEqual(Resources.CreateResource(1024, 0), queueConf.GetMinResources
("root.queueA"));
NUnit.Framework.Assert.AreEqual(Resources.CreateResource(2048, 0), queueConf.GetMinResources
("root.queueB"));
NUnit.Framework.Assert.AreEqual(Resources.CreateResource(0), queueConf.GetMinResources
("root.queueC"));
NUnit.Framework.Assert.AreEqual(Resources.CreateResource(0), queueConf.GetMinResources
("root.queueD"));
NUnit.Framework.Assert.AreEqual(Resources.CreateResource(0), queueConf.GetMinResources
("root.queueE"));
NUnit.Framework.Assert.AreEqual(15, queueConf.GetQueueMaxApps("root." + YarnConfiguration
.DefaultQueueName));
NUnit.Framework.Assert.AreEqual(15, queueConf.GetQueueMaxApps("root.queueA"));
NUnit.Framework.Assert.AreEqual(15, queueConf.GetQueueMaxApps("root.queueB"));
NUnit.Framework.Assert.AreEqual(15, queueConf.GetQueueMaxApps("root.queueC"));
NUnit.Framework.Assert.AreEqual(3, queueConf.GetQueueMaxApps("root.queueD"));
NUnit.Framework.Assert.AreEqual(15, queueConf.GetQueueMaxApps("root.queueE"));
NUnit.Framework.Assert.AreEqual(10, queueConf.GetUserMaxApps("user1"));
NUnit.Framework.Assert.AreEqual(5, queueConf.GetUserMaxApps("user2"));
// Unspecified queues should get default ACL
NUnit.Framework.Assert.AreEqual(" ", queueConf.GetQueueAcl("root.queueA", QueueACL
.AdministerQueue).GetAclString());
NUnit.Framework.Assert.AreEqual(" ", queueConf.GetQueueAcl("root.queueA", QueueACL
.SubmitApplications).GetAclString());
// Queue B ACL
NUnit.Framework.Assert.AreEqual("alice,bob admins", queueConf.GetQueueAcl("root.queueB"
, QueueACL.AdministerQueue).GetAclString());
// Queue C ACL
NUnit.Framework.Assert.AreEqual("alice,bob admins", queueConf.GetQueueAcl("root.queueC"
, QueueACL.SubmitApplications).GetAclString());
NUnit.Framework.Assert.AreEqual(120000, queueConf.GetMinSharePreemptionTimeout("root"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetMinSharePreemptionTimeout("root."
+ YarnConfiguration.DefaultQueueName));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetMinSharePreemptionTimeout("root.queueA"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetMinSharePreemptionTimeout("root.queueB"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetMinSharePreemptionTimeout("root.queueC"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetMinSharePreemptionTimeout("root.queueD"
));
NUnit.Framework.Assert.AreEqual(60000, queueConf.GetMinSharePreemptionTimeout("root.queueE"
));
NUnit.Framework.Assert.AreEqual(300000, queueConf.GetFairSharePreemptionTimeout("root"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root."
+ YarnConfiguration.DefaultQueueName));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root.queueA"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root.queueB"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root.queueC"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root.queueD"
));
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root.queueE"
));
NUnit.Framework.Assert.AreEqual(.6f, queueConf.GetFairSharePreemptionThreshold("root"
), 0.01);
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionThreshold("root."
+ YarnConfiguration.DefaultQueueName), 0.01);
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionThreshold("root.queueA"
), 0.01);
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionThreshold("root.queueB"
), 0.01);
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionThreshold("root.queueC"
), 0.01);
NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionThreshold("root.queueD"
), 0.01);
NUnit.Framework.Assert.AreEqual(.3f, queueConf.GetFairSharePreemptionThreshold("root.queueE"
), 0.01);
}
/// <summary>Test the parser on a treebank.</summary>
/// <remarks>
/// Test the parser on a treebank. Parses will be written to stdout, and
/// various other information will be written to stderr and stdout,
/// particularly if <code>op.testOptions.verbose</code> is true.
/// </remarks>
/// <param name="testTreebank">The treebank to parse</param>
/// <returns>
/// The labeled precision/recall F<sub>1</sub> (EVALB measure)
/// of the parser on the treebank.
/// </returns>
public virtual double TestOnTreebank(Treebank testTreebank)
{
log.Info("Testing on treebank");
Timing treebankTotalTimer = new Timing();
TreePrint treePrint = op.testOptions.TreePrint(op.tlpParams);
ITreebankLangParserParams tlpParams = op.tlpParams;
ITreebankLanguagePack tlp = op.Langpack();
PrintWriter pwOut;
PrintWriter pwErr;
if (op.testOptions.quietEvaluation)
{
NullOutputStream quiet = new NullOutputStream();
pwOut = tlpParams.Pw(quiet);
pwErr = tlpParams.Pw(quiet);
}
else
{
pwOut = tlpParams.Pw();
pwErr = tlpParams.Pw(System.Console.Error);
}
if (op.testOptions.verbose)
{
pwErr.Print("Testing ");
pwErr.Println(testTreebank.TextualSummary(tlp));
}
if (op.testOptions.evalb)
{
EvalbFormatWriter.InitEVALBfiles(tlpParams);
}
PrintWriter pwFileOut = null;
if (op.testOptions.writeOutputFiles)
{
string fname = op.testOptions.outputFilesPrefix + "." + op.testOptions.outputFilesExtension;
try
{
pwFileOut = op.tlpParams.Pw(new FileOutputStream(fname));
}
catch (IOException ioe)
{
Sharpen.Runtime.PrintStackTrace(ioe);
}
}
PrintWriter pwStats = null;
if (op.testOptions.outputkBestEquivocation != null)
{
try
{
pwStats = op.tlpParams.Pw(new FileOutputStream(op.testOptions.outputkBestEquivocation));
}
catch (IOException ioe)
{
Sharpen.Runtime.PrintStackTrace(ioe);
}
}
if (op.testOptions.testingThreads != 1)
{
MulticoreWrapper <IList <IHasWord>, IParserQuery> wrapper = new MulticoreWrapper <IList <IHasWord>, IParserQuery>(op.testOptions.testingThreads, new ParsingThreadsafeProcessor(pqFactory, pwErr));
LinkedList <Tree> goldTrees = new LinkedList <Tree>();
foreach (Tree goldTree in testTreebank)
{
IList <IHasWord> sentence = GetInputSentence(goldTree);
goldTrees.Add(goldTree);
pwErr.Println("Parsing [len. " + sentence.Count + "]: " + SentenceUtils.ListToString(sentence));
wrapper.Put(sentence);
while (wrapper.Peek())
{
IParserQuery pq = wrapper.Poll();
goldTree = goldTrees.Poll();
ProcessResults(pq, goldTree, pwErr, pwOut, pwFileOut, pwStats, treePrint);
}
}
// for tree iterator
wrapper.Join();
while (wrapper.Peek())
{
IParserQuery pq = wrapper.Poll();
Tree goldTree_1 = goldTrees.Poll();
ProcessResults(pq, goldTree_1, pwErr, pwOut, pwFileOut, pwStats, treePrint);
}
}
else
{
IParserQuery pq = pqFactory.ParserQuery();
foreach (Tree goldTree in testTreebank)
{
IList <CoreLabel> sentence = GetInputSentence(goldTree);
pwErr.Println("Parsing [len. " + sentence.Count + "]: " + SentenceUtils.ListToString(sentence));
pq.ParseAndReport(sentence, pwErr);
ProcessResults(pq, goldTree, pwErr, pwOut, pwFileOut, pwStats, treePrint);
}
}
// for tree iterator
//Done parsing...print the results of the evaluations
treebankTotalTimer.Done("Testing on treebank");
if (op.testOptions.quietEvaluation)
{
pwErr = tlpParams.Pw(System.Console.Error);
}
if (saidMemMessage)
{
ParserUtils.PrintOutOfMemory(pwErr);
}
if (op.testOptions.evalb)
{
EvalbFormatWriter.CloseEVALBfiles();
}
if (numSkippedEvals != 0)
{
pwErr.Printf("Unable to evaluate %d parser hypotheses due to yield mismatch\n", numSkippedEvals);
}
// only created here so we know what parser types are supported...
IParserQuery pq_1 = pqFactory.ParserQuery();
if (summary)
{
if (pcfgLB != null)
{
pcfgLB.Display(false, pwErr);
}
if (pcfgChildSpecific != null)
{
pcfgChildSpecific.Display(false, pwErr);
}
if (pcfgLA != null)
{
pcfgLA.Display(false, pwErr);
}
if (pcfgCB != null)
{
pcfgCB.Display(false, pwErr);
}
if (pcfgDA != null)
{
pcfgDA.Display(false, pwErr);
}
if (pcfgTA != null)
{
pcfgTA.Display(false, pwErr);
}
if (pcfgLL != null && pq_1.GetPCFGParser() != null)
{
pcfgLL.Display(false, pwErr);
}
if (depDA != null)
{
depDA.Display(false, pwErr);
}
if (depTA != null)
{
depTA.Display(false, pwErr);
}
if (depLL != null && pq_1.GetDependencyParser() != null)
{
depLL.Display(false, pwErr);
}
if (factLB != null)
{
factLB.Display(false, pwErr);
}
if (factChildSpecific != null)
{
factChildSpecific.Display(false, pwErr);
}
if (factLA != null)
{
factLA.Display(false, pwErr);
}
if (factCB != null)
{
factCB.Display(false, pwErr);
}
if (factDA != null)
{
factDA.Display(false, pwErr);
}
if (factTA != null)
{
factTA.Display(false, pwErr);
}
if (factLL != null && pq_1.GetFactoredParser() != null)
{
factLL.Display(false, pwErr);
}
if (pcfgCatE != null)
{
pcfgCatE.Display(false, pwErr);
}
foreach (IEval eval in evals)
{
eval.Display(false, pwErr);
}
foreach (BestOfTopKEval eval_1 in topKEvals)
{
eval_1.Display(false, pwErr);
}
}
// these ones only have a display mode, so display if turned on!!
if (pcfgRUO != null)
{
pcfgRUO.Display(true, pwErr);
}
if (pcfgCUO != null)
{
pcfgCUO.Display(true, pwErr);
}
if (tsv)
{
NumberFormat nf = new DecimalFormat("0.00");
pwErr.Println("factF1\tfactDA\tfactEx\tpcfgF1\tdepDA\tfactTA\tnum");
if (factLB != null)
{
pwErr.Print(nf.Format(factLB.GetEvalbF1Percent()));
}
pwErr.Print("\t");
if (pq_1.GetDependencyParser() != null && factDA != null)
{
pwErr.Print(nf.Format(factDA.GetEvalbF1Percent()));
}
pwErr.Print("\t");
if (factLB != null)
{
pwErr.Print(nf.Format(factLB.GetExactPercent()));
}
pwErr.Print("\t");
if (pcfgLB != null)
{
pwErr.Print(nf.Format(pcfgLB.GetEvalbF1Percent()));
}
pwErr.Print("\t");
if (pq_1.GetDependencyParser() != null && depDA != null)
{
pwErr.Print(nf.Format(depDA.GetEvalbF1Percent()));
}
pwErr.Print("\t");
if (pq_1.GetPCFGParser() != null && factTA != null)
{
pwErr.Print(nf.Format(factTA.GetEvalbF1Percent()));
}
pwErr.Print("\t");
if (factLB != null)
{
pwErr.Print(factLB.GetNum());
}
pwErr.Println();
}
double f1 = 0.0;
if (factLB != null)
{
f1 = factLB.GetEvalbF1();
}
//Close files (if necessary)
if (pwFileOut != null)
{
pwFileOut.Close();
}
if (pwStats != null)
{
pwStats.Close();
}
if (parserQueryEvals != null)
{
foreach (IParserQueryEval parserQueryEval in parserQueryEvals)
{
parserQueryEval.Display(false, pwErr);
}
}
return(f1);
}
/// <exception cref="System.IO.IOException"/>
public static void Main(string[] args)
{
if (args.Length != 4)
{
logger.Info("Usage: MnistConverter dataFile labelFile outFile propsFile");
return;
}
DataInputStream xStream = IOUtils.GetDataInputStream(args[0]);
DataInputStream yStream = IOUtils.GetDataInputStream(args[1]);
PrintWriter oStream = new PrintWriter(new FileWriter(args[2]));
PrintWriter pStream = new PrintWriter(new FileWriter(args[3]));
int xMagic = xStream.ReadInt();
if (xMagic != 2051)
{
throw new Exception("Bad format of xStream");
}
int yMagic = yStream.ReadInt();
if (yMagic != 2049)
{
throw new Exception("Bad format of yStream");
}
int xNumImages = xStream.ReadInt();
int yNumLabels = yStream.ReadInt();
if (xNumImages != yNumLabels)
{
throw new Exception("x and y sizes don't match");
}
logger.Info("Images and label file both contain " + xNumImages + " entries.");
int xRows = xStream.ReadInt();
int xColumns = xStream.ReadInt();
for (int i = 0; i < xNumImages; i++)
{
int label = yStream.ReadUnsignedByte();
int[] matrix = new int[xRows * xColumns];
for (int j = 0; j < xRows * xColumns; j++)
{
matrix[j] = xStream.ReadUnsignedByte();
}
oStream.Print(label);
foreach (int k in matrix)
{
oStream.Print('\t');
oStream.Print(k);
}
oStream.Println();
}
logger.Info("Converted.");
xStream.Close();
yStream.Close();
oStream.Close();
// number from 1; column 0 is the class
pStream.Println("goldAnswerColumn = 0");
pStream.Println("useClassFeature = true");
pStream.Println("sigma = 10");
// not optimized, but weak regularization seems appropriate when much data, few features
for (int j_1 = 0; j_1 < xRows * xColumns; j_1++)
{
pStream.Println((j_1 + 1) + ".realValued = true");
}
pStream.Close();
}
public static void Main(string[] args)
{
if (args.Length < minArgs)
{
System.Console.Out.WriteLine(usage.ToString());
System.Environment.Exit(-1);
}
ITreebankLangParserParams tlpp = new EnglishTreebankParserParams();
DiskTreebank tb = null;
string encoding = "UTF-8";
TregexPattern rootMatch = null;
for (int i = 0; i < args.Length; i++)
{
if (args[i].StartsWith("-"))
{
switch (args[i])
{
case "-l":
{
Language lang = Language.ValueOf(args[++i].Trim());
tlpp = lang.@params;
break;
}
case "-e":
{
encoding = args[++i];
break;
}
default:
{
System.Console.Out.WriteLine(usage.ToString());
System.Environment.Exit(-1);
break;
}
}
}
else
{
rootMatch = TregexPattern.Compile("@" + args[i++]);
if (tb == null)
{
if (tlpp == null)
{
System.Console.Out.WriteLine(usage.ToString());
System.Environment.Exit(-1);
}
else
{
tlpp.SetInputEncoding(encoding);
tlpp.SetOutputEncoding(encoding);
tb = tlpp.DiskTreebank();
}
}
tb.LoadPath(args[i++]);
}
}
ICounter <string> rhsCounter = new ClassicCounter <string>();
foreach (Tree t in tb)
{
TregexMatcher m = rootMatch.Matcher(t);
while (m.FindNextMatchingNode())
{
Tree match = m.GetMatch();
StringBuilder sb = new StringBuilder();
foreach (Tree kid in match.Children())
{
sb.Append(kid.Value()).Append(" ");
}
rhsCounter.IncrementCount(sb.ToString().Trim());
}
}
IList <string> biggestKeys = new List <string>(rhsCounter.KeySet());
biggestKeys.Sort(Counters.ToComparatorDescending(rhsCounter));
PrintWriter pw = tlpp.Pw();
foreach (string rhs in biggestKeys)
{
pw.Printf("%s\t%d%n", rhs, (int)rhsCounter.GetCount(rhs));
}
pw.Close();
}
public virtual void TestCombiner()
{
if (!new FilePath(TestRootDir).Mkdirs())
{
throw new RuntimeException("Could not create test dir: " + TestRootDir);
}
FilePath @in = new FilePath(TestRootDir, "input");
if ([email protected]())
{
throw new RuntimeException("Could not create test dir: " + @in);
}
FilePath @out = new FilePath(TestRootDir, "output");
PrintWriter pw = new PrintWriter(new FileWriter(new FilePath(@in, "data.txt")));
pw.WriteLine("A|a,1");
pw.WriteLine("A|b,2");
pw.WriteLine("B|a,3");
pw.WriteLine("B|b,4");
pw.WriteLine("B|c,5");
pw.Close();
JobConf conf = new JobConf();
conf.Set("mapreduce.framework.name", "local");
Job job = new Job(conf);
TextInputFormat.SetInputPaths(job, new Path(@in.GetPath()));
TextOutputFormat.SetOutputPath(job, new Path(@out.GetPath()));
job.SetMapperClass(typeof(TestNewCombinerGrouping.Map));
job.SetReducerClass(typeof(TestNewCombinerGrouping.Reduce));
job.SetInputFormatClass(typeof(TextInputFormat));
job.SetMapOutputKeyClass(typeof(Text));
job.SetMapOutputValueClass(typeof(LongWritable));
job.SetOutputFormatClass(typeof(TextOutputFormat));
job.SetGroupingComparatorClass(typeof(TestNewCombinerGrouping.GroupComparator));
job.SetCombinerKeyGroupingComparatorClass(typeof(TestNewCombinerGrouping.GroupComparator
));
job.SetCombinerClass(typeof(TestNewCombinerGrouping.Combiner));
job.GetConfiguration().SetInt("min.num.spills.for.combine", 0);
job.Submit();
job.WaitForCompletion(false);
if (job.IsSuccessful())
{
Counters counters = job.GetCounters();
long combinerInputRecords = counters.FindCounter("org.apache.hadoop.mapreduce.TaskCounter"
, "COMBINE_INPUT_RECORDS").GetValue();
long combinerOutputRecords = counters.FindCounter("org.apache.hadoop.mapreduce.TaskCounter"
, "COMBINE_OUTPUT_RECORDS").GetValue();
NUnit.Framework.Assert.IsTrue(combinerInputRecords > 0);
NUnit.Framework.Assert.IsTrue(combinerInputRecords > combinerOutputRecords);
BufferedReader br = new BufferedReader(new FileReader(new FilePath(@out, "part-r-00000"
)));
ICollection <string> output = new HashSet <string>();
string line = br.ReadLine();
NUnit.Framework.Assert.IsNotNull(line);
output.AddItem(Sharpen.Runtime.Substring(line, 0, 1) + Sharpen.Runtime.Substring(
line, 4, 5));
line = br.ReadLine();
NUnit.Framework.Assert.IsNotNull(line);
output.AddItem(Sharpen.Runtime.Substring(line, 0, 1) + Sharpen.Runtime.Substring(
line, 4, 5));
line = br.ReadLine();
NUnit.Framework.Assert.IsNull(line);
br.Close();
ICollection <string> expected = new HashSet <string>();
expected.AddItem("A2");
expected.AddItem("B5");
NUnit.Framework.Assert.AreEqual(expected, output);
}
else
{
NUnit.Framework.Assert.Fail("Job failed");
}
}
public virtual void ParseFiles <_T0>(string[] args, int argIndex, bool tokenized, ITokenizerFactory <_T0> tokenizerFactory, string elementDelimiter, string sentenceDelimiter, IFunction <IList <IHasWord>, IList <IHasWord> > escaper, string tagDelimiter
)
where _T0 : IHasWord
{
DocumentPreprocessor.DocType docType = (elementDelimiter == null) ? DocumentPreprocessor.DocType.Plain : DocumentPreprocessor.DocType.Xml;
if (op.testOptions.verbose)
{
if (tokenizerFactory != null)
{
pwErr.Println("parseFiles: Tokenizer factory is: " + tokenizerFactory);
}
}
Timing timer = new Timing();
// timer.start(); // constructor already starts it.
//Loop over the files
for (int i = argIndex; i < args.Length; i++)
{
string filename = args[i];
DocumentPreprocessor documentPreprocessor;
if (filename.Equals("-"))
{
try
{
documentPreprocessor = new DocumentPreprocessor(IOUtils.ReaderFromStdin(op.tlpParams.GetInputEncoding()), docType);
}
catch (IOException e)
{
throw new RuntimeIOException(e);
}
}
else
{
documentPreprocessor = new DocumentPreprocessor(filename, docType, op.tlpParams.GetInputEncoding());
}
//Unused values are null per the main() method invocation below
//null is the default for these properties
documentPreprocessor.SetSentenceFinalPuncWords(tlp.SentenceFinalPunctuationWords());
documentPreprocessor.SetEscaper(escaper);
documentPreprocessor.SetSentenceDelimiter(sentenceDelimiter);
documentPreprocessor.SetTagDelimiter(tagDelimiter);
documentPreprocessor.SetElementDelimiter(elementDelimiter);
if (tokenizerFactory == null)
{
documentPreprocessor.SetTokenizerFactory((tokenized) ? null : tlp.GetTokenizerFactory());
}
else
{
documentPreprocessor.SetTokenizerFactory(tokenizerFactory);
}
//Setup the output
PrintWriter pwo = pwOut;
if (op.testOptions.writeOutputFiles)
{
string normalizedName = filename;
try
{
new URL(normalizedName);
// this will exception if not a URL
normalizedName = normalizedName.ReplaceAll("/", "_");
}
catch (MalformedURLException)
{
}
//It isn't a URL, so silently ignore
string ext = (op.testOptions.outputFilesExtension == null) ? "stp" : op.testOptions.outputFilesExtension;
string fname = normalizedName + '.' + ext;
if (op.testOptions.outputFilesDirectory != null && !op.testOptions.outputFilesDirectory.IsEmpty())
{
string fseparator = Runtime.GetProperty("file.separator");
if (fseparator == null || fseparator.IsEmpty())
{
fseparator = "/";
}
File fnameFile = new File(fname);
fname = op.testOptions.outputFilesDirectory + fseparator + fnameFile.GetName();
}
try
{
pwo = op.tlpParams.Pw(new FileOutputStream(fname));
}
catch (IOException ioe)
{
throw new RuntimeIOException(ioe);
}
}
treePrint.PrintHeader(pwo, op.tlpParams.GetOutputEncoding());
pwErr.Println("Parsing file: " + filename);
int num = 0;
int numProcessed = 0;
if (op.testOptions.testingThreads != 1)
{
MulticoreWrapper <IList <IHasWord>, IParserQuery> wrapper = new MulticoreWrapper <IList <IHasWord>, IParserQuery>(op.testOptions.testingThreads, new ParsingThreadsafeProcessor(pqFactory, pwErr));
foreach (IList <IHasWord> sentence in documentPreprocessor)
{
num++;
numSents++;
int len = sentence.Count;
numWords += len;
pwErr.Println("Parsing [sent. " + num + " len. " + len + "]: " + SentenceUtils.ListToString(sentence, true));
wrapper.Put(sentence);
while (wrapper.Peek())
{
IParserQuery pq = wrapper.Poll();
ProcessResults(pq, numProcessed++, pwo);
}
}
wrapper.Join();
while (wrapper.Peek())
{
IParserQuery pq = wrapper.Poll();
ProcessResults(pq, numProcessed++, pwo);
}
}
else
{
IParserQuery pq = pqFactory.ParserQuery();
foreach (IList <IHasWord> sentence in documentPreprocessor)
{
num++;
numSents++;
int len = sentence.Count;
numWords += len;
pwErr.Println("Parsing [sent. " + num + " len. " + len + "]: " + SentenceUtils.ListToString(sentence, true));
pq.ParseAndReport(sentence, pwErr);
ProcessResults(pq, numProcessed++, pwo);
}
}
treePrint.PrintFooter(pwo);
if (op.testOptions.writeOutputFiles)
{
pwo.Close();
}
pwErr.Println("Parsed file: " + filename + " [" + num + " sentences].");
}
long millis = timer.Stop();
if (summary)
{
if (pcfgLL != null)
{
pcfgLL.Display(false, pwErr);
}
if (depLL != null)
{
depLL.Display(false, pwErr);
}
if (factLL != null)
{
factLL.Display(false, pwErr);
}
}
if (saidMemMessage)
{
ParserUtils.PrintOutOfMemory(pwErr);
}
double wordspersec = numWords / (((double)millis) / 1000);
double sentspersec = numSents / (((double)millis) / 1000);
NumberFormat nf = new DecimalFormat("0.00");
// easier way!
pwErr.Println("Parsed " + numWords + " words in " + numSents + " sentences (" + nf.Format(wordspersec) + " wds/sec; " + nf.Format(sentspersec) + " sents/sec).");
if (numFallback > 0)
{
pwErr.Println(" " + numFallback + " sentences were parsed by fallback to PCFG.");
}
if (numUnparsable > 0 || numNoMemory > 0 || numSkipped > 0)
{
pwErr.Println(" " + (numUnparsable + numNoMemory + numSkipped) + " sentences were not parsed:");
if (numUnparsable > 0)
{
pwErr.Println(" " + numUnparsable + " were not parsable with non-zero probability.");
}
if (numNoMemory > 0)
{
pwErr.Println(" " + numNoMemory + " were skipped because of insufficient memory.");
}
if (numSkipped > 0)
{
pwErr.Println(" " + numSkipped + " were skipped as length 0 or greater than " + op.testOptions.maxLength);
}
}
}
/// <summary>Execute with no arguments for usage.</summary>
public static void Main(string[] args)
{
if (!ValidateCommandLine(args))
{
log.Info(Usage);
System.Environment.Exit(-1);
}
ITreebankLangParserParams tlpp = Language.@params;
PrintWriter pwOut = tlpp.Pw();
Treebank guessTreebank = tlpp.DiskTreebank();
guessTreebank.LoadPath(guessFile);
pwOut.Println("GUESS TREEBANK:");
pwOut.Println(guessTreebank.TextualSummary());
Treebank goldTreebank = tlpp.DiskTreebank();
goldTreebank.LoadPath(goldFile);
pwOut.Println("GOLD TREEBANK:");
pwOut.Println(goldTreebank.TextualSummary());
Edu.Stanford.Nlp.Parser.Metrics.LeafAncestorEval metric = new Edu.Stanford.Nlp.Parser.Metrics.LeafAncestorEval("LeafAncestor");
ITreeTransformer tc = tlpp.Collinizer();
//The evalb ref implementation assigns status for each tree pair as follows:
//
// 0 - Ok (yields match)
// 1 - length mismatch
// 2 - null parse e.g. (()).
//
//In the cases of 1,2, evalb does not include the tree pair in the LP/LR computation.
IEnumerator <Tree> goldItr = goldTreebank.GetEnumerator();
IEnumerator <Tree> guessItr = guessTreebank.GetEnumerator();
int goldLineId = 0;
int guessLineId = 0;
int skippedGuessTrees = 0;
while (guessItr.MoveNext() && goldItr.MoveNext())
{
Tree guessTree = guessItr.Current;
IList <ILabel> guessYield = guessTree.Yield();
guessLineId++;
Tree goldTree = goldItr.Current;
IList <ILabel> goldYield = goldTree.Yield();
goldLineId++;
// Check that we should evaluate this tree
if (goldYield.Count > MaxGoldYield)
{
skippedGuessTrees++;
continue;
}
// Only trees with equal yields can be evaluated
if (goldYield.Count != guessYield.Count)
{
pwOut.Printf("Yield mismatch gold: %d tokens vs. guess: %d tokens (lines: gold %d guess %d)%n", goldYield.Count, guessYield.Count, goldLineId, guessLineId);
skippedGuessTrees++;
continue;
}
Tree evalGuess = tc.TransformTree(guessTree);
Tree evalGold = tc.TransformTree(goldTree);
metric.Evaluate(evalGuess, evalGold, ((Verbose) ? pwOut : null));
}
if (guessItr.MoveNext() || goldItr.MoveNext())
{
System.Console.Error.Printf("Guess/gold files do not have equal lengths (guess: %d gold: %d)%n.", guessLineId, goldLineId);
}
pwOut.Println("================================================================================");
if (skippedGuessTrees != 0)
{
pwOut.Printf("%s %d guess trees%n", "Unable to evaluate", skippedGuessTrees);
}
metric.Display(true, pwOut);
pwOut.Close();
}
/// <summary>Prints the container report for an container id.</summary>
/// <param name="containerId"/>
/// <returns>exitCode</returns>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private int PrintContainerReport(string containerId)
{
ContainerReport containerReport = null;
try
{
containerReport = client.GetContainerReport((ConverterUtils.ToContainerId(containerId
)));
}
catch (ApplicationNotFoundException)
{
sysout.WriteLine("Application for Container with id '" + containerId + "' doesn't exist in RM or Timeline Server."
);
return(-1);
}
catch (ApplicationAttemptNotFoundException)
{
sysout.WriteLine("Application Attempt for Container with id '" + containerId + "' doesn't exist in RM or Timeline Server."
);
return(-1);
}
catch (ContainerNotFoundException)
{
sysout.WriteLine("Container with id '" + containerId + "' doesn't exist in RM or Timeline Server."
);
return(-1);
}
// Use PrintWriter.println, which uses correct platform line ending.
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter containerReportStr = new PrintWriter(new OutputStreamWriter(baos, Sharpen.Extensions.GetEncoding
("UTF-8")));
if (containerReport != null)
{
containerReportStr.WriteLine("Container Report : ");
containerReportStr.Write("\tContainer-Id : ");
containerReportStr.WriteLine(containerReport.GetContainerId());
containerReportStr.Write("\tStart-Time : ");
containerReportStr.WriteLine(containerReport.GetCreationTime());
containerReportStr.Write("\tFinish-Time : ");
containerReportStr.WriteLine(containerReport.GetFinishTime());
containerReportStr.Write("\tState : ");
containerReportStr.WriteLine(containerReport.GetContainerState());
containerReportStr.Write("\tLOG-URL : ");
containerReportStr.WriteLine(containerReport.GetLogUrl());
containerReportStr.Write("\tHost : ");
containerReportStr.WriteLine(containerReport.GetAssignedNode());
containerReportStr.Write("\tNodeHttpAddress : ");
containerReportStr.WriteLine(containerReport.GetNodeHttpAddress() == null ? "N/A"
: containerReport.GetNodeHttpAddress());
containerReportStr.Write("\tDiagnostics : ");
containerReportStr.Write(containerReport.GetDiagnosticsInfo());
}
else
{
containerReportStr.Write("Container with id '" + containerId + "' doesn't exist in Timeline Server."
);
containerReportStr.Close();
sysout.WriteLine(baos.ToString("UTF-8"));
return(-1);
}
containerReportStr.Close();
sysout.WriteLine(baos.ToString("UTF-8"));
return(0);
}
/// <summary>Run the scoring metric on guess/gold input.</summary>
/// <remarks>
/// Run the scoring metric on guess/gold input. This method performs "Collinization."
/// The default language is English.
/// </remarks>
/// <param name="args"/>
public static void Main(string[] args)
{
if (args.Length < minArgs)
{
System.Console.Out.WriteLine(usage.ToString());
System.Environment.Exit(-1);
}
ITreebankLangParserParams tlpp = new EnglishTreebankParserParams();
int maxGoldYield = int.MaxValue;
int maxGuessYield = int.MaxValue;
bool Verbose = false;
bool skipGuess = false;
bool tagMode = false;
string guessFile = null;
string goldFile = null;
for (int i = 0; i < args.Length; i++)
{
if (args[i].StartsWith("-"))
{
switch (args[i])
{
case "-l":
{
Language lang = Language.ValueOf(args[++i].Trim());
tlpp = lang.@params;
break;
}
case "-y":
{
maxGoldYield = System.Convert.ToInt32(args[++i].Trim());
break;
}
case "-t":
{
tagMode = true;
break;
}
case "-v":
{
Verbose = true;
break;
}
case "-g":
{
maxGuessYield = System.Convert.ToInt32(args[++i].Trim());
skipGuess = true;
break;
}
default:
{
System.Console.Out.WriteLine(usage.ToString());
System.Environment.Exit(-1);
break;
}
}
}
else
{
//Required parameters
goldFile = args[i++];
guessFile = args[i];
break;
}
}
PrintWriter pwOut = tlpp.Pw();
Treebank guessTreebank = tlpp.DiskTreebank();
guessTreebank.LoadPath(guessFile);
pwOut.Println("GUESS TREEBANK:");
pwOut.Println(guessTreebank.TextualSummary());
Treebank goldTreebank = tlpp.DiskTreebank();
goldTreebank.LoadPath(goldFile);
pwOut.Println("GOLD TREEBANK:");
pwOut.Println(goldTreebank.TextualSummary());
string evalName = (tagMode) ? "TsarfatyTAG" : "TsarfatySEG";
Edu.Stanford.Nlp.Parser.Metrics.TsarfatyEval eval = new Edu.Stanford.Nlp.Parser.Metrics.TsarfatyEval(evalName, tagMode);
ITreeTransformer tc = tlpp.Collinizer();
//PennTreeReader skips over null/malformed parses. So when the yields of the gold/guess trees
//don't match, we need to keep looking for the next gold tree that matches.
//The evalb ref implementation differs slightly as it expects one tree per line. It assigns
//status as follows:
//
// 0 - Ok (yields match)
// 1 - length mismatch
// 2 - null parse e.g. (()).
//
//In the cases of 1,2, evalb does not include the tree pair in the LP/LR computation.
IEnumerator <Tree> goldItr = goldTreebank.GetEnumerator();
int goldLineId = 0;
int skippedGuessTrees = 0;
foreach (Tree guess in guessTreebank)
{
Tree evalGuess = tc.TransformTree(guess);
List <ILabel> guessSent = guess.Yield();
string guessChars = SentenceUtils.ListToString(guessSent).ReplaceAll("\\s+", string.Empty);
if (guessSent.Count > maxGuessYield)
{
skippedGuessTrees++;
continue;
}
bool doneEval = false;
while (goldItr.MoveNext() && !doneEval)
{
Tree gold = goldItr.Current;
Tree evalGold = tc.TransformTree(gold);
goldLineId++;
List <ILabel> goldSent = gold.Yield();
string goldChars = SentenceUtils.ListToString(goldSent).ReplaceAll("\\s+", string.Empty);
if (goldSent.Count > maxGoldYield)
{
continue;
}
else
{
if (goldChars.Length != guessChars.Length)
{
pwOut.Printf("Char level yield mismatch at line %d (guess: %d gold: %d)\n", goldLineId, guessChars.Length, goldChars.Length);
skippedGuessTrees++;
break;
}
}
//Default evalb behavior -- skip this guess tree
eval.Evaluate(evalGuess, evalGold, ((Verbose) ? pwOut : null));
doneEval = true;
}
}
//Move to the next guess parse
pwOut.Println("================================================================================");
if (skippedGuessTrees != 0)
{
pwOut.Printf("%s %d guess trees\n", ((skipGuess) ? "Skipped" : "Unable to evaluate"), skippedGuessTrees);
}
eval.Display(true, pwOut);
pwOut.Println();
pwOut.Close();
}
/// <summary>Prints the application report for an application id.</summary>
/// <param name="applicationId"/>
/// <returns>exitCode</returns>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private int PrintApplicationReport(string applicationId)
{
ApplicationReport appReport = null;
try
{
appReport = client.GetApplicationReport(ConverterUtils.ToApplicationId(applicationId
));
}
catch (ApplicationNotFoundException)
{
sysout.WriteLine("Application with id '" + applicationId + "' doesn't exist in RM or Timeline Server."
);
return(-1);
}
// Use PrintWriter.println, which uses correct platform line ending.
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter appReportStr = new PrintWriter(new OutputStreamWriter(baos, Sharpen.Extensions.GetEncoding
("UTF-8")));
if (appReport != null)
{
appReportStr.WriteLine("Application Report : ");
appReportStr.Write("\tApplication-Id : ");
appReportStr.WriteLine(appReport.GetApplicationId());
appReportStr.Write("\tApplication-Name : ");
appReportStr.WriteLine(appReport.GetName());
appReportStr.Write("\tApplication-Type : ");
appReportStr.WriteLine(appReport.GetApplicationType());
appReportStr.Write("\tUser : "******"\tQueue : ");
appReportStr.WriteLine(appReport.GetQueue());
appReportStr.Write("\tStart-Time : ");
appReportStr.WriteLine(appReport.GetStartTime());
appReportStr.Write("\tFinish-Time : ");
appReportStr.WriteLine(appReport.GetFinishTime());
appReportStr.Write("\tProgress : ");
DecimalFormat formatter = new DecimalFormat("###.##%");
string progress = formatter.Format(appReport.GetProgress());
appReportStr.WriteLine(progress);
appReportStr.Write("\tState : ");
appReportStr.WriteLine(appReport.GetYarnApplicationState());
appReportStr.Write("\tFinal-State : ");
appReportStr.WriteLine(appReport.GetFinalApplicationStatus());
appReportStr.Write("\tTracking-URL : ");
appReportStr.WriteLine(appReport.GetOriginalTrackingUrl());
appReportStr.Write("\tRPC Port : ");
appReportStr.WriteLine(appReport.GetRpcPort());
appReportStr.Write("\tAM Host : ");
appReportStr.WriteLine(appReport.GetHost());
appReportStr.Write("\tAggregate Resource Allocation : ");
ApplicationResourceUsageReport usageReport = appReport.GetApplicationResourceUsageReport
();
if (usageReport != null)
{
//completed app report in the timeline server doesn't have usage report
appReportStr.Write(usageReport.GetMemorySeconds() + " MB-seconds, ");
appReportStr.WriteLine(usageReport.GetVcoreSeconds() + " vcore-seconds");
}
else
{
appReportStr.WriteLine("N/A");
}
appReportStr.Write("\tDiagnostics : ");
appReportStr.Write(appReport.GetDiagnostics());
}
else
{
appReportStr.Write("Application with id '" + applicationId + "' doesn't exist in RM."
);
appReportStr.Close();
sysout.WriteLine(baos.ToString("UTF-8"));
return(-1);
}
appReportStr.Close();
sysout.WriteLine(baos.ToString("UTF-8"));
return(0);
}
/// <summary>A simple, deterministic sentence-splitter.</summary>
/// <remarks>
/// A simple, deterministic sentence-splitter. This method only supports the English
/// tokenizer, so for other languages you should run the tokenizer first and then
/// run this sentence splitter with the "-whitespaceTokenization" option.
/// </remarks>
/// <param name="args">Command-line arguments</param>
/// <exception cref="System.IO.IOException"/>
public static void Main(string[] args)
{
Properties options = StringUtils.ArgsToProperties(args, ArgOptionDefs());
if (options.Contains("help"))
{
log.Info(Usage());
return;
}
// Command-line flags
string encoding = options.GetProperty("encoding", "utf-8");
bool printSentenceLengths = PropertiesUtils.GetBool(options, "printSentenceLengths", false);
string xmlElementDelimiter = options.GetProperty("xml", null);
DocumentPreprocessor.DocType docType = xmlElementDelimiter == null ? DocumentPreprocessor.DocType.Plain : DocumentPreprocessor.DocType.Xml;
string sentenceDelimiter = options.Contains("noTokenization") ? Runtime.GetProperty("line.separator") : null;
string sDelim = options.GetProperty("sentenceDelimiter");
if (sDelim != null)
{
if (Sharpen.Runtime.EqualsIgnoreCase(sDelim, "newline"))
{
sentenceDelimiter = "\n";
}
else
{
sentenceDelimiter = sDelim;
}
}
string tagDelimiter = options.GetProperty("tag", null);
string[] sentenceDelims = null;
// Setup the TokenizerFactory
int numFactoryFlags = 0;
bool suppressEscaping = options.Contains("suppressEscaping");
if (suppressEscaping)
{
numFactoryFlags += 1;
}
bool customTokenizer = options.Contains("tokenizerOptions");
if (customTokenizer)
{
numFactoryFlags += 1;
}
bool printOriginalText = options.Contains("printOriginalText");
if (printOriginalText)
{
numFactoryFlags += 1;
}
bool whitespaceTokenization = options.Contains("whitespaceTokenization");
if (whitespaceTokenization)
{
numFactoryFlags += 1;
}
if (numFactoryFlags > 1)
{
log.Info("Only one tokenizer flag allowed at a time: ");
log.Info(" -suppressEscaping, -tokenizerOptions, -printOriginalText, -whitespaceTokenization");
return;
}
ITokenizerFactory <IHasWord> tf = null;
if (suppressEscaping)
{
tf = PTBTokenizer.Factory(new CoreLabelTokenFactory(), "ptb3Escaping=false");
}
else
{
if (customTokenizer)
{
tf = PTBTokenizer.Factory(new CoreLabelTokenFactory(), options.GetProperty("tokenizerOptions"));
}
else
{
if (printOriginalText)
{
tf = PTBTokenizer.Factory(new CoreLabelTokenFactory(), "invertible=true");
}
else
{
if (whitespaceTokenization)
{
IList <string> whitespaceDelims = new List <string>(Arrays.AsList(DocumentPreprocessor.DefaultSentenceDelims));
whitespaceDelims.Add(WhitespaceLexer.Newline);
sentenceDelims = Sharpen.Collections.ToArray(whitespaceDelims, new string[whitespaceDelims.Count]);
}
else
{
tf = PTBTokenizer.Factory(new CoreLabelTokenFactory(), string.Empty);
}
}
}
}
string fileList = options.GetProperty(string.Empty, null);
string[] files = fileList == null ? new string[1] : fileList.Split("\\s+");
int numSents = 0;
PrintWriter pw = new PrintWriter(new OutputStreamWriter(System.Console.Out, encoding), true);
foreach (string file in files)
{
DocumentPreprocessor docPreprocessor;
if (file == null || file.IsEmpty())
{
docPreprocessor = new DocumentPreprocessor(new InputStreamReader(Runtime.@in, encoding));
}
else
{
docPreprocessor = new DocumentPreprocessor(file, docType, encoding);
}
if (docType == DocumentPreprocessor.DocType.Xml)
{
docPreprocessor.SetElementDelimiter(xmlElementDelimiter);
}
docPreprocessor.SetTokenizerFactory(tf);
if (sentenceDelimiter != null)
{
docPreprocessor.SetSentenceDelimiter(sentenceDelimiter);
}
if (tagDelimiter != null)
{
docPreprocessor.SetTagDelimiter(tagDelimiter);
}
if (sentenceDelims != null)
{
docPreprocessor.SetSentenceFinalPuncWords(sentenceDelims);
}
foreach (IList <IHasWord> sentence in docPreprocessor)
{
numSents++;
if (printSentenceLengths)
{
System.Console.Error.Printf("Length: %d%n", sentence.Count);
}
bool printSpace = false;
foreach (IHasWord word in sentence)
{
if (printOriginalText)
{
CoreLabel cl = (CoreLabel)word;
if (!printSpace)
{
pw.Print(cl.Get(typeof(CoreAnnotations.BeforeAnnotation)));
printSpace = true;
}
pw.Print(cl.Get(typeof(CoreAnnotations.OriginalTextAnnotation)));
pw.Print(cl.Get(typeof(CoreAnnotations.AfterAnnotation)));
}
else
{
if (printSpace)
{
pw.Print(" ");
}
printSpace = true;
pw.Print(word.Word());
}
}
pw.Println();
}
}
pw.Close();
System.Console.Error.Printf("Read in %d sentences.%n", numSents);
}
/// <param name="args"/>
public static void Main(string[] args)
{
if (args.Length < MinArgs)
{
log.Info(Usage());
System.Environment.Exit(-1);
}
Properties options = StringUtils.ArgsToProperties(args, OptionArgDefs());
bool Verbose = PropertiesUtils.GetBool(options, "v", false);
Language Language = PropertiesUtils.Get(options, "l", Language.English, typeof(Language));
int MaxGoldYield = PropertiesUtils.GetInt(options, "g", int.MaxValue);
int MaxGuessYield = PropertiesUtils.GetInt(options, "y", int.MaxValue);
string[] parsedArgs = options.GetProperty(string.Empty, string.Empty).Split("\\s+");
if (parsedArgs.Length != MinArgs)
{
log.Info(Usage());
System.Environment.Exit(-1);
}
File goldFile = new File(parsedArgs[0]);
File guessFile = new File(parsedArgs[1]);
ITreebankLangParserParams tlpp = Language.@params;
PrintWriter pwOut = tlpp.Pw();
Treebank guessTreebank = tlpp.DiskTreebank();
guessTreebank.LoadPath(guessFile);
pwOut.Println("GUESS TREEBANK:");
pwOut.Println(guessTreebank.TextualSummary());
Treebank goldTreebank = tlpp.DiskTreebank();
goldTreebank.LoadPath(goldFile);
pwOut.Println("GOLD TREEBANK:");
pwOut.Println(goldTreebank.TextualSummary());
Edu.Stanford.Nlp.Parser.Metrics.CollinsDepEval depEval = new Edu.Stanford.Nlp.Parser.Metrics.CollinsDepEval("CollinsDep", true, tlpp.HeadFinder(), tlpp.TreebankLanguagePack().StartSymbol());
ITreeTransformer tc = tlpp.Collinizer();
//PennTreeReader skips over null/malformed parses. So when the yields of the gold/guess trees
//don't match, we need to keep looking for the next gold tree that matches.
//The evalb ref implementation differs slightly as it expects one tree per line. It assigns
//status as follows:
//
// 0 - Ok (yields match)
// 1 - length mismatch
// 2 - null parse e.g. (()).
//
//In the cases of 1,2, evalb does not include the tree pair in the LP/LR computation.
IEnumerator <Tree> goldItr = goldTreebank.GetEnumerator();
int goldLineId = 0;
int skippedGuessTrees = 0;
foreach (Tree guess in guessTreebank)
{
Tree evalGuess = tc.TransformTree(guess);
if (guess.Yield().Count > MaxGuessYield)
{
skippedGuessTrees++;
continue;
}
bool doneEval = false;
while (goldItr.MoveNext() && !doneEval)
{
Tree gold = goldItr.Current;
Tree evalGold = tc.TransformTree(gold);
goldLineId++;
if (gold.Yield().Count > MaxGoldYield)
{
continue;
}
else
{
if (evalGold.Yield().Count != evalGuess.Yield().Count)
{
pwOut.Println("Yield mismatch at gold line " + goldLineId);
skippedGuessTrees++;
break;
}
}
//Default evalb behavior -- skip this guess tree
depEval.Evaluate(evalGuess, evalGold, ((Verbose) ? pwOut : null));
doneEval = true;
}
}
//Move to the next guess parse
pwOut.Println("================================================================================");
if (skippedGuessTrees != 0)
{
pwOut.Printf("%s %d guess trees\n", ((MaxGuessYield < int.MaxValue) ? "Skipped" : "Unable to evaluate"), skippedGuessTrees);
}
depEval.Display(true, pwOut);
pwOut.Close();
}
public virtual double[] Minimize(Func function, double functionTolerance, double[] initial, int maxIterations)
{
// check for stochastic derivatives
if (!(function is AbstractStochasticCachingDiffFunction))
{
throw new NotSupportedException();
}
AbstractStochasticCachingDiffFunction dfunction = (AbstractStochasticCachingDiffFunction)function;
dfunction.method = StochasticCalculateMethods.GradientOnly;
/* ---
* StochasticDiffFunctionTester sdft = new StochasticDiffFunctionTester(dfunction);
* ArrayMath.add(initial, gen.nextDouble() ); // to make sure that priors are working.
* sdft.testSumOfBatches(initial, 1e-4);
* System.exit(1);
* --- */
x = initial;
grad = new double[x.Length];
newX = new double[x.Length];
gradList = new List <double[]>();
numBatches = dfunction.DataDimension() / bSize;
outputFrequency = (int)System.Math.Ceil(((double)numBatches) / ((double)outputFrequency));
Init(dfunction);
InitFiles();
bool have_max = (maxIterations > 0 || numPasses > 0);
if (!have_max)
{
throw new NotSupportedException("No maximum number of iterations has been specified.");
}
else
{
maxIterations = System.Math.Max(maxIterations, numPasses) * numBatches;
}
Sayln(" Batchsize of: " + bSize);
Sayln(" Data dimension of: " + dfunction.DataDimension());
Sayln(" Batches per pass through data: " + numBatches);
Sayln(" Max iterations is = " + maxIterations);
if (outputIterationsToFile)
{
infoFile.Println(function.DomainDimension() + "; DomainDimension ");
infoFile.Println(bSize + "; batchSize ");
infoFile.Println(maxIterations + "; maxIterations");
infoFile.Println(numBatches + "; numBatches ");
infoFile.Println(outputFrequency + "; outputFrequency");
}
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
// Loop
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Timing total = new Timing();
Timing current = new Timing();
total.Start();
current.Start();
for (k = 0; k < maxIterations; k++)
{
try
{
bool doEval = (k > 0 && evaluateIters > 0 && k % evaluateIters == 0);
if (doEval)
{
DoEvaluation(x);
}
int pass = k / numBatches;
int batch = k % numBatches;
Say("Iter: " + k + " pass " + pass + " batch " + batch);
// restrict number of saved gradients
// (recycle memory of first gradient in list for new gradient)
if (k > 0 && gradList.Count >= memory)
{
newGrad = gradList.Remove(0);
}
else
{
newGrad = new double[grad.Length];
}
dfunction.hasNewVals = true;
System.Array.Copy(dfunction.DerivativeAt(x, v, bSize), 0, newGrad, 0, newGrad.Length);
ArrayMath.AssertFinite(newGrad, "newGrad");
gradList.Add(newGrad);
grad = Smooth(gradList);
//Get the next X
TakeStep(dfunction);
ArrayMath.AssertFinite(newX, "newX");
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
// THIS IS FOR DEBUG ONLY
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
if (outputIterationsToFile && (k % outputFrequency == 0) && k != 0)
{
double curVal = dfunction.ValueAt(x);
Say(" TrueValue{ " + curVal + " } ");
file.Println(k + " , " + curVal + " , " + total.Report());
}
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
// END OF DEBUG STUFF
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
if (k >= maxIterations)
{
Sayln("Stochastic Optimization complete. Stopped after max iterations");
x = newX;
break;
}
if (total.Report() >= maxTime)
{
Sayln("Stochastic Optimization complete. Stopped after max time");
x = newX;
break;
}
System.Array.Copy(newX, 0, x, 0, x.Length);
Say("[" + (total.Report()) / 1000.0 + " s ");
Say("{" + (current.Restart() / 1000.0) + " s}] ");
Say(" " + dfunction.LastValue());
if (quiet)
{
log.Info(".");
}
else
{
Sayln(string.Empty);
}
}
catch (ArrayMath.InvalidElementException e)
{
log.Info(e.ToString());
for (int i = 0; i < x.Length; i++)
{
x[i] = double.NaN;
}
break;
}
}
if (evaluateIters > 0)
{
// do final evaluation
DoEvaluation(x);
}
if (outputIterationsToFile)
{
infoFile.Println(k + "; Iterations");
infoFile.Println((total.Report()) / 1000.0 + "; Completion Time");
infoFile.Println(dfunction.ValueAt(x) + "; Finalvalue");
infoFile.Close();
file.Close();
log.Info("Output Files Closed");
}
//System.exit(1);
Say("Completed in: " + (total.Report()) / 1000.0 + " s");
return(x);
}
/// <exception cref="System.Exception"/>
public virtual void Finish()
{
dataWriter.Close();
goldClusterWriter.Close();
}
public virtual void TestContainerLaunchAndStop()
{
containerManager.Start();
FilePath scriptFile = Shell.AppendScriptExtension(tmpDir, "scriptFile");
PrintWriter fileWriter = new PrintWriter(scriptFile);
FilePath processStartFile = new FilePath(tmpDir, "start_file.txt").GetAbsoluteFile
();
// ////// Construct the Container-id
ContainerId cId = CreateContainerId(0);
if (Shell.Windows)
{
fileWriter.WriteLine("@echo Hello World!> " + processStartFile);
fileWriter.WriteLine("@echo " + cId + ">> " + processStartFile);
fileWriter.WriteLine("@ping -n 100 127.0.0.1 >nul");
}
else
{
fileWriter.Write("\numask 0");
// So that start file is readable by the test
fileWriter.Write("\necho Hello World! > " + processStartFile);
fileWriter.Write("\necho $$ >> " + processStartFile);
fileWriter.Write("\nexec sleep 100");
}
fileWriter.Close();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));
containerLaunchContext.SetCommands(commands);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
int timeoutSecs = 0;
while (!processStartFile.Exists() && timeoutSecs++ < 20)
{
Sharpen.Thread.Sleep(1000);
Log.Info("Waiting for process start-file to be created");
}
NUnit.Framework.Assert.IsTrue("ProcessStartFile doesn't exist!", processStartFile
.Exists());
// Now verify the contents of the file
BufferedReader reader = new BufferedReader(new FileReader(processStartFile));
NUnit.Framework.Assert.AreEqual("Hello World!", reader.ReadLine());
// Get the pid of the process
string pid = reader.ReadLine().Trim();
// No more lines
NUnit.Framework.Assert.AreEqual(null, reader.ReadLine());
// Now test the stop functionality.
// Assert that the process is alive
NUnit.Framework.Assert.IsTrue("Process is not alive!", DefaultContainerExecutor.ContainerIsAlive
(pid));
// Once more
NUnit.Framework.Assert.IsTrue("Process is not alive!", DefaultContainerExecutor.ContainerIsAlive
(pid));
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds
);
containerManager.StopContainers(stopRequest);
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete);
GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
containerIds);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(gcsRequest
).GetContainerStatuses()[0];
int expectedExitCode = ContainerExitStatus.KilledByAppmaster;
NUnit.Framework.Assert.AreEqual(expectedExitCode, containerStatus.GetExitStatus()
);
// Assert that the process is not alive anymore
NUnit.Framework.Assert.IsFalse("Process is still alive!", DefaultContainerExecutor
.ContainerIsAlive(pid));
}
public virtual void TestRMInitialsWithFileSystemBasedConfigurationProvider()
{
configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"
);
// upload configurations
FilePath excludeHostsFile = new FilePath(tmpDir.ToString(), "excludeHosts");
if (excludeHostsFile.Exists())
{
excludeHostsFile.Delete();
}
if (!excludeHostsFile.CreateNewFile())
{
NUnit.Framework.Assert.Fail("Can not create " + "excludeHosts");
}
PrintWriter fileWriter = new PrintWriter(excludeHostsFile);
fileWriter.Write("0.0.0.0:123");
fileWriter.Close();
UploadToRemoteFileSystem(new Path(excludeHostsFile.GetAbsolutePath()));
YarnConfiguration yarnConf = new YarnConfiguration();
yarnConf.Set(YarnConfiguration.YarnAdminAcl, "world:anyone:rwcda");
yarnConf.Set(YarnConfiguration.RmNodesExcludeFilePath, this.workingPath + "/excludeHosts"
);
UploadConfiguration(yarnConf, "yarn-site.xml");
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
csConf.Set("yarn.scheduler.capacity.maximum-applications", "5000");
UploadConfiguration(csConf, "capacity-scheduler.xml");
string aclsString = "alice,bob users,wheel";
Configuration newConf = new Configuration();
newConf.Set("security.applicationclient.protocol.acl", aclsString);
UploadConfiguration(newConf, "hadoop-policy.xml");
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeysPublic.HadoopSecurityAuthorization, true);
conf.Set("hadoop.proxyuser.test.groups", "test_groups");
conf.Set("hadoop.proxyuser.test.hosts", "test_hosts");
conf.SetClass(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(TestRMAdminService.MockUnixGroupsMapping
), typeof(GroupMappingServiceProvider));
UploadConfiguration(conf, "core-site.xml");
// update the groups
TestRMAdminService.MockUnixGroupsMapping.UpdateGroups();
ResourceManager resourceManager = null;
try
{
try
{
resourceManager = new ResourceManager();
resourceManager.Init(configuration);
resourceManager.Start();
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("Should not get any exceptions");
}
// validate values for excludeHosts
ICollection <string> excludeHosts = resourceManager.GetRMContext().GetNodesListManager
().GetHostsReader().GetExcludedHosts();
NUnit.Framework.Assert.IsTrue(excludeHosts.Count == 1);
NUnit.Framework.Assert.IsTrue(excludeHosts.Contains("0.0.0.0:123"));
// validate values for admin-acls
string aclStringAfter = resourceManager.adminService.GetAccessControlList().GetAclString
().Trim();
NUnit.Framework.Assert.AreEqual(aclStringAfter, "world:anyone:rwcda," + UserGroupInformation
.GetCurrentUser().GetShortUserName());
// validate values for queue configuration
CapacityScheduler cs = (CapacityScheduler)resourceManager.GetRMContext().GetScheduler
();
int maxAppsAfter = cs.GetConfiguration().GetMaximumSystemApplications();
NUnit.Framework.Assert.AreEqual(maxAppsAfter, 5000);
// verify service Acls for AdminService
ServiceAuthorizationManager adminServiceServiceManager = resourceManager.adminService
.GetServer().GetServiceAuthorizationManager();
VerifyServiceACLsRefresh(adminServiceServiceManager, typeof(ApplicationClientProtocolPB
), aclsString);
// verify service ACLs for ClientRMService
ServiceAuthorizationManager clientRMServiceServiceManager = resourceManager.GetRMContext
().GetClientRMService().GetServer().GetServiceAuthorizationManager();
VerifyServiceACLsRefresh(clientRMServiceServiceManager, typeof(ApplicationClientProtocolPB
), aclsString);
// verify service ACLs for ApplicationMasterService
ServiceAuthorizationManager appMasterService = resourceManager.GetRMContext().GetApplicationMasterService
().GetServer().GetServiceAuthorizationManager();
VerifyServiceACLsRefresh(appMasterService, typeof(ApplicationClientProtocolPB), aclsString
);
// verify service ACLs for ResourceTrackerService
ServiceAuthorizationManager RTService = resourceManager.GetRMContext().GetResourceTrackerService
().GetServer().GetServiceAuthorizationManager();
VerifyServiceACLsRefresh(RTService, typeof(ApplicationClientProtocolPB), aclsString
);
// verify ProxyUsers and ProxyHosts
ProxyUsers.RefreshSuperUserGroupsConfiguration(configuration);
NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups
()["hadoop.proxyuser.test.groups"].Count == 1);
NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups
()["hadoop.proxyuser.test.groups"].Contains("test_groups"));
NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts
()["hadoop.proxyuser.test.hosts"].Count == 1);
NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts
()["hadoop.proxyuser.test.hosts"].Contains("test_hosts"));
// verify UserToGroupsMappings
IList <string> groupAfter = Groups.GetUserToGroupsMappingService(configuration).GetGroups
(UserGroupInformation.GetCurrentUser().GetUserName());
NUnit.Framework.Assert.IsTrue(groupAfter.Contains("test_group_D") && groupAfter.Contains
("test_group_E") && groupAfter.Contains("test_group_F") && groupAfter.Count == 3
);
}
finally
{
if (resourceManager != null)
{
resourceManager.Stop();
}
}
}