public virtual void AverageScoredModels(ICollection <ScoredObject <Edu.Stanford.Nlp.Parser.Shiftreduce.PerceptronModel> > scoredModels)
{
if (scoredModels.IsEmpty())
{
throw new ArgumentException("Cannot average empty models");
}
log.Info("Averaging " + scoredModels.Count + " models with scores");
foreach (ScoredObject <Edu.Stanford.Nlp.Parser.Shiftreduce.PerceptronModel> model in scoredModels)
{
log.Info(" " + Nf.Format(model.Score()));
}
log.Info();
IList <Edu.Stanford.Nlp.Parser.Shiftreduce.PerceptronModel> models = CollectionUtils.TransformAsList(scoredModels, null);
AverageModels(models);
}
public override string ToString()
{
StringBuilder compliantBlocksSB = new StringBuilder();
compliantBlocksSB.Append("\nBlocks satisfying the specified storage policy:");
compliantBlocksSB.Append("\nStorage Policy # of blocks % of blocks\n"
);
StringBuilder nonCompliantBlocksSB = new StringBuilder();
Formatter compliantFormatter = new Formatter(compliantBlocksSB);
Formatter nonCompliantFormatter = new Formatter(nonCompliantBlocksSB);
NumberFormat percentFormat = NumberFormat.GetPercentInstance();
percentFormat.SetMinimumFractionDigits(4);
percentFormat.SetMaximumFractionDigits(4);
foreach (KeyValuePair <StoragePolicySummary.StorageTypeAllocation, long> storageComboCount
in SortByComparator(storageComboCounts))
{
double percent = (double)storageComboCount.Value / (double)totalBlocks;
StoragePolicySummary.StorageTypeAllocation sta = storageComboCount.Key;
if (sta.PolicyMatches())
{
compliantFormatter.Format("%-25s %10d %20s%n", sta.GetStoragePolicyDescriptor(),
storageComboCount.Value, percentFormat.Format(percent));
}
else
{
if (nonCompliantBlocksSB.Length == 0)
{
nonCompliantBlocksSB.Append("\nBlocks NOT satisfying the specified storage policy:"
);
nonCompliantBlocksSB.Append("\nStorage Policy ");
nonCompliantBlocksSB.Append("Specified Storage Policy # of blocks % of blocks\n"
);
}
nonCompliantFormatter.Format("%-35s %-20s %10d %20s%n", sta.GetStoragePolicyDescriptor
(), sta.GetSpecifiedStoragePolicy().GetName(), storageComboCount.Value, percentFormat
.Format(percent));
}
}
if (nonCompliantBlocksSB.Length == 0)
{
nonCompliantBlocksSB.Append("\nAll blocks satisfy specified storage policy.\n");
}
compliantFormatter.Close();
nonCompliantFormatter.Close();
return(compliantBlocksSB.ToString() + nonCompliantBlocksSB);
}
private static string GetFormattedValue(IExcelDataReader reader, int columnIndex, CultureInfo culture)
{
string result;
var value = reader.GetValue(columnIndex);
var formatString = reader.GetNumberFormatString(columnIndex);
if (formatString != null)
{
var format = new NumberFormat(formatString);
result = format.Format(value, culture);
}
else
{
result = Convert.ToString(value, culture);
}
return(result.Replace('\r', ' ').Replace('\n', ' '));
}
/// <summary>Generate a unique filename, based on the task id, name, and extension</summary>
/// <param name="context">the task that is calling this</param>
/// <param name="name">the base filename</param>
/// <param name="extension">the filename extension</param>
/// <returns>a string like $name-[mrsct]-$id$extension</returns>
public static string GetUniqueFile(TaskAttemptContext context, string name, string
extension)
{
lock (typeof(FileOutputFormat))
{
TaskID taskId = context.GetTaskAttemptID().GetTaskID();
int partition = taskId.GetId();
StringBuilder result = new StringBuilder();
result.Append(name);
result.Append('-');
result.Append(TaskID.GetRepresentingCharacter(taskId.GetTaskType()));
result.Append('-');
result.Append(NumberFormat.Format(partition));
result.Append(extension);
return(result.ToString());
}
}
/// <summary>
/// Works out whether the model expectations match the empirical
/// expectations.
/// </summary>
/// <returns>Whether the model is correct</returns>
public override bool CheckCorrectness()
{
log.Info("Checking model correctness; x size " + p.data.xSize + ' ' + ", ysize " + p.data.ySize);
NumberFormat nf = NumberFormat.GetNumberInstance();
nf.SetMaximumFractionDigits(4);
bool flag = true;
for (int f = 0; f < lambda.Length; f++)
{
if (Math.Abs(lambda[f]) > 100)
{
log.Info(" Lambda too big " + lambda[f]);
log.Info(" empirical " + ftildeArr[f] + " expected " + FExpected(p.functions.Get(f)));
}
}
for (int i = 0; i < ftildeArr.Length; i++)
{
double exp = Math.Abs(ftildeArr[i] - FExpected(p.functions.Get(i)));
if (exp > 0.001)
{
flag = false;
log.Info("Constraint " + i + " not satisfied emp " + nf.Format(ftildeArr[i]) + " exp " + nf.Format(FExpected(p.functions.Get(i))) + " diff " + nf.Format(exp) + " lambda " + nf.Format(lambda[i]));
}
}
for (int x = 0; x < p.data.xSize; x++)
{
double s = 0.0;
for (int y = 0; y < p.data.ySize; y++)
{
s = s + probConds[x][y];
}
if (Math.Abs(s - 1) > 0.0001)
{
for (int y_1 = 0; y_1 < p.data.ySize; y_1++)
{
log.Info(y_1 + " : " + probConds[x][y_1]);
}
log.Info("probabilities do not sum to one " + x + ' ' + (float)s);
}
}
return(flag);
}
/// <summary>
/// updateTableReportFacilities
/// </summary>
private void updateTable(TimeSeriesClasses.ComparisonPollutant data)
{
this.grdCompareDetailsAll.Visible = true;
this.grdCompareDetailsBothYears.Visible = true;
//all facilites
List <CompareDetailElement> elementsAll = new List <CompareDetailElement>();
elementsAll.Add(new CompareDetailElement(Resources.GetGlobal("Common", "Total"),
data != null ? QuantityFormat.Format(data.QuantityFrom, CODE_KG) : String.Empty,
data != null ? QuantityFormat.Format(data.QuantityTo, CODE_KG) : String.Empty));
elementsAll.Add(new CompareDetailElement(Resources.GetGlobal("Pollutant", "AccidentalQuantity"),
data != null ? QuantityFormat.Format(data.AccidentalFrom, CODE_KG) : String.Empty,
data != null ? QuantityFormat.Format(data.AccidentalTo, CODE_KG) : String.Empty));
elementsAll.Add(new CompareDetailElement(Resources.GetGlobal("Common", "Facilities"),
data != null ? NumberFormat.Format(data.FacilitiesFrom) : String.Empty,
data != null ? NumberFormat.Format(data.FacilitiesTo) : String.Empty));
//facilites in both years
List <CompareDetailElement> elementsBothYears = new List <CompareDetailElement>();
elementsBothYears.Add(new CompareDetailElement(Resources.GetGlobal("Common", "Total"),
data != null ? QuantityFormat.Format(data.BothQuantityFrom, CODE_KG) : String.Empty,
data != null ? QuantityFormat.Format(data.BothQuantityTo, CODE_KG) : String.Empty));
elementsBothYears.Add(new CompareDetailElement(Resources.GetGlobal("Pollutant", "AccidentalQuantity"),
data != null ? QuantityFormat.Format(data.BothAccidentalFrom, CODE_KG) : String.Empty,
data != null ? QuantityFormat.Format(data.BothAccidentalTo, CODE_KG) : String.Empty));
elementsBothYears.Add(new CompareDetailElement(Resources.GetGlobal("Common", "Facilities"),
data != null ? NumberFormat.Format(data.BothFacilities) : String.Empty,
data != null ? NumberFormat.Format(data.BothFacilities) : String.Empty));
// data binding
this.grdCompareDetailsAll.DataSource = elementsAll;
grdCompareDetailsAll.DataBind();
this.grdCompareDetailsBothYears.DataSource = elementsBothYears;
grdCompareDetailsBothYears.DataBind();
}
public void VerifyFormatOfQuantityKind()
{
var format = string.Empty;
var simpleQuantityKind = new SimpleQuantityKind();
format = NumberFormat.Format(simpleQuantityKind);
Assert.AreEqual("@", format);
var ratioscale = new RatioScale();
ratioscale.NumberSet = NumberSetKind.INTEGER_NUMBER_SET;
format = NumberFormat.Format(simpleQuantityKind, ratioscale);
Assert.AreEqual("0", format);
ratioscale.NumberSet = NumberSetKind.NATURAL_NUMBER_SET;
format = NumberFormat.Format(simpleQuantityKind, ratioscale);
Assert.AreEqual("general", format);
}
public string GetFormattedValue(IExcelDataReader reader, int columnIndex, CultureInfo culture)
{
var value = reader.GetValue(columnIndex);
var formatString = reader.GetNumberFormatString(columnIndex);
if (formatString != null)
{
var format = new NumberFormat(formatString);
var forstr = format.Format(value, culture);
if (value != null && value.GetType() == typeof(System.Double) && getDoublePointCnt(forstr) > mDoublePointKeepCnt)
{
forstr = Convert.ToDouble(forstr).ToString("F" + mDoublePointKeepCnt.ToString());
forstr = Convert.ToDouble(forstr).ToString("G");
}
return(forstr);
}
return(Convert.ToString(value, culture));
}
/// <summary>
/// Initializes a new instance of the <see cref="ComponentExcelRow"/> class.
/// </summary>
/// <param name="parameterTypeComponent">
/// The <see cref="ParameterTypeComponent"/> that represents the current row
/// </param>
/// <param name="parameterValueSet">
/// The <see cref="ParameterValueSet"/> from which the value is to be represented
/// </param>
/// <param name="level">
/// the grouping level of the current row.
/// </param>
public ComponentExcelRow(ParameterTypeComponent parameterTypeComponent, ParameterValueSet parameterValueSet, int level)
: base(parameterTypeComponent)
{
this.Level = level;
this.Name = string.Format("{0}{1}", new string(' ', 3 * level), parameterTypeComponent.ShortName);
this.ShortName = parameterTypeComponent.ShortName;
this.Type = ParameterSheetConstants.PVSCT;
this.Switch = parameterValueSet.ValueSwitch.ToString();
this.Id = string.Format("{0}:{1}", parameterValueSet.Iid, parameterTypeComponent.Index);
this.Owner = parameterValueSet.Owner.ShortName;
this.ParameterType = parameterTypeComponent.ParameterType;
this.Format = NumberFormat.Format(parameterTypeComponent.ParameterType, parameterTypeComponent.Scale);
this.ModelCode = parameterValueSet.ModelCode(parameterTypeComponent.Index);
if (parameterTypeComponent.Scale == null)
{
this.ParameterTypeShortName = parameterTypeComponent.ParameterType.ShortName;
}
else
{
this.ParameterTypeShortName = string.Format("{0} [{1}]", parameterTypeComponent.ParameterType.ShortName, parameterTypeComponent.Scale.ShortName);
}
try
{
this.ActualValue = ParameterValueConverter.ConvertToObject(this.Thing.ParameterType, parameterValueSet.ActualValue[parameterTypeComponent.Index]);
this.ComputedValue = ParameterValueConverter.ConvertToObject(this.Thing.ParameterType, parameterValueSet.Computed[parameterTypeComponent.Index]);
this.ManualValue = ParameterValueConverter.ConvertToObject(this.Thing.ParameterType, parameterValueSet.Manual[parameterTypeComponent.Index]);
this.ReferenceValue = ParameterValueConverter.ConvertToObject(this.Thing.ParameterType, parameterValueSet.Reference[parameterTypeComponent.Index]);
this.Formula = parameterValueSet.Formula[parameterTypeComponent.Index];
}
catch (Exception ex)
{
this.SetDefaultValues();
Logger.Debug(ex);
}
}
public static string FormatDistanceBetween(LatLng point1, LatLng point2)
{
if (point1 == null || point2 == null)
{
return(null);
}
NumberFormat numberFormat = NumberFormat.NumberInstance;
double distance = Math.Round(ComputeDistanceBetween(point1, point2));
distance *= 0.621371192;
numberFormat.MaximumFractionDigits = 1;
// Adjust to KM if M goes over 1000 (see javadoc of method for note
// on only supporting metric)
// if (distance >= 1000)
// {
// numberFormat.MaximumFractionDigits = 1;
// return numberFormat.Format(distance / 1000) + DISTANCE_KM_POSTFIX;
// }
return(numberFormat.Format(distance) + DISTANCE_MI_POSTFIX);
}
/// <summary>Print some statistics about this lexicon.</summary>
public virtual void PrintLexStats()
{
System.Console.Out.WriteLine("BaseLexicon statistics");
System.Console.Out.WriteLine("unknownLevel is " + GetUnknownWordModel().GetUnknownLevel());
// System.out.println("Rules size: " + rules.size());
System.Console.Out.WriteLine("Sum of rulesWithWord: " + NumRules());
System.Console.Out.WriteLine("Tags size: " + tags.Count);
int wsize = words.Count;
System.Console.Out.WriteLine("Words size: " + wsize);
// System.out.println("Unseen Sigs size: " + sigs.size() +
// " [number of unknown equivalence classes]");
System.Console.Out.WriteLine("rulesWithWord length: " + rulesWithWord.Length + " [should be sum of words + unknown sigs]");
int[] lengths = new int[StatsBins];
List <string>[] wArr = new ArrayList[StatsBins];
for (int j = 0; j < StatsBins; j++)
{
wArr[j] = new List <string>();
}
for (int i = 0; i < rulesWithWord.Length; i++)
{
int num = rulesWithWord[i].Count;
if (num > StatsBins - 1)
{
num = StatsBins - 1;
}
lengths[num]++;
if (wsize <= 20 || num >= StatsBins / 2)
{
wArr[num].Add(wordIndex.Get(i));
}
}
System.Console.Out.WriteLine("Stats on how many taggings for how many words");
for (int j_1 = 0; j_1 < StatsBins; j_1++)
{
System.Console.Out.Write(j_1 + " taggings: " + lengths[j_1] + " words ");
if (wsize <= 20 || j_1 >= StatsBins / 2)
{
System.Console.Out.Write(wArr[j_1]);
}
System.Console.Out.WriteLine();
}
NumberFormat nf = NumberFormat.GetNumberInstance();
nf.SetMaximumFractionDigits(0);
System.Console.Out.WriteLine("Unseen counter: " + Counters.ToString(uwModel.UnSeenCounter(), nf));
if (wsize < 50 && tags.Count < 10)
{
nf.SetMaximumFractionDigits(3);
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
pw.Println("Tagging probabilities log P(word|tag)");
for (int t = 0; t < tags.Count; t++)
{
pw.Print('\t');
pw.Print(tagIndex.Get(t));
}
pw.Println();
for (int w = 0; w < wsize; w++)
{
pw.Print(wordIndex.Get(w));
pw.Print('\t');
for (int t_1 = 0; t_1 < tags.Count; t_1++)
{
IntTaggedWord iTW = new IntTaggedWord(w, t_1);
pw.Print(nf.Format(Score(iTW, 1, wordIndex.Get(w), null)));
if (t_1 == tags.Count - 1)
{
pw.Println();
}
else
{
pw.Print('\t');
}
}
}
pw.Close();
System.Console.Out.WriteLine(sw.ToString());
}
}
/// <summary>
/// Provides some testing and opportunities for exploration of the
/// probabilities of a BaseLexicon.
/// </summary>
/// <remarks>
/// Provides some testing and opportunities for exploration of the
/// probabilities of a BaseLexicon. What's here currently probably
/// only works for the English Penn Treeebank, as it uses default
/// constructors. Of the words given to test on,
/// the first is treated as sentence initial, and the rest as not
/// sentence initial.
/// </remarks>
/// <param name="args">
/// The command line arguments:
/// java BaseLexicon treebankPath fileRange unknownWordModel words
/// </param>
public static void Main(string[] args)
{
if (args.Length < 3)
{
log.Info("java BaseLexicon treebankPath fileRange unknownWordModel words*");
return;
}
System.Console.Out.Write("Training BaseLexicon from " + args[0] + ' ' + args[1] + " ... ");
Treebank tb = new DiskTreebank();
tb.LoadPath(args[0], new NumberRangesFileFilter(args[1], true));
// TODO: change this interface so the lexicon creates its own indices?
IIndex <string> wordIndex = new HashIndex <string>();
IIndex <string> tagIndex = new HashIndex <string>();
Options op = new Options();
op.lexOptions.useUnknownWordSignatures = System.Convert.ToInt32(args[2]);
Edu.Stanford.Nlp.Parser.Lexparser.BaseLexicon lex = new Edu.Stanford.Nlp.Parser.Lexparser.BaseLexicon(op, wordIndex, tagIndex);
lex.InitializeTraining(tb.Count);
lex.Train(tb);
lex.FinishTraining();
System.Console.Out.WriteLine("done.");
System.Console.Out.WriteLine();
NumberFormat nf = NumberFormat.GetNumberInstance();
nf.SetMaximumFractionDigits(4);
IList <string> impos = new List <string>();
for (int i = 3; i < args.Length; i++)
{
if (lex.IsKnown(args[i]))
{
System.Console.Out.WriteLine(args[i] + " is a known word. Log probabilities [log P(w|t)] for its taggings are:");
for (IEnumerator <IntTaggedWord> it = lex.RuleIteratorByWord(wordIndex.AddToIndex(args[i]), i - 3, null); it.MoveNext();)
{
IntTaggedWord iTW = it.Current;
System.Console.Out.WriteLine(StringUtils.Pad(iTW, 24) + nf.Format(lex.Score(iTW, i - 3, wordIndex.Get(iTW.word), null)));
}
}
else
{
string sig = lex.GetUnknownWordModel().GetSignature(args[i], i - 3);
System.Console.Out.WriteLine(args[i] + " is an unknown word. Signature with uwm " + lex.GetUnknownWordModel().GetUnknownLevel() + ((i == 3) ? " init" : "non-init") + " is: " + sig);
impos.Clear();
IList <string> lis = new List <string>(tagIndex.ObjectsList());
lis.Sort();
foreach (string tStr in lis)
{
IntTaggedWord iTW = new IntTaggedWord(args[i], tStr, wordIndex, tagIndex);
double score = lex.Score(iTW, 1, args[i], null);
if (score == float.NegativeInfinity)
{
impos.Add(tStr);
}
else
{
System.Console.Out.WriteLine(StringUtils.Pad(iTW, 24) + nf.Format(score));
}
}
if (impos.Count > 0)
{
System.Console.Out.WriteLine(args[i] + " impossible tags: " + impos);
}
}
System.Console.Out.WriteLine();
}
}
/// <summary>
/// Initializes a new instance of the <see cref="ParameterOverrideValueSetExcelRow"/> class.
/// </summary>
/// <param name="parameterOverrideValueSet">
/// The <see cref="ParameterOverrideValueSet"/> that is represented by the current row.
/// </param>
/// <param name="level">
/// the grouping level of the current row.
/// </param>
/// <param name="clones">
/// The <see cref="Thing"/>s for which the values need to be restored to what the user provided.
/// </param>
public ParameterOverrideValueSetExcelRow(ParameterOverrideValueSet parameterOverrideValueSet, int level, IReadOnlyDictionary <Guid, ProcessedValueSet> processedValueSets)
: base(parameterOverrideValueSet)
{
this.Level = level;
this.Id = this.Thing.Iid.ToString();
ProcessedValueSet processedValueSet;
if (processedValueSets.TryGetValue(parameterOverrideValueSet.Iid, out processedValueSet))
{
parameterOverrideValueSet = (ParameterOverrideValueSet)processedValueSet.ClonedThing;
}
var parameterOverride = parameterOverrideValueSet.Container as ParameterOverride;
if (parameterOverride != null)
{
this.ParameterType = parameterOverride.ParameterType;
if (this.ParameterType is CompoundParameterType)
{
this.Type = ParameterSheetConstants.POVSCD;
}
else if (this.ParameterType is SampledFunctionParameterType)
{
this.Type = ParameterSheetConstants.SFPOVS;
}
else
{
this.Type = ParameterSheetConstants.POVS;
}
var spaces = new string(' ', 3 * Math.Abs(level - 1));
this.ParameterTypeShortName = parameterOverride.Scale == null ? parameterOverride.ParameterType.ShortName : string.Format("{0} [{1}]", parameterOverride.ParameterType.ShortName, parameterOverride.Scale.ShortName);
var optionPart = parameterOverrideValueSet.ActualOption != null ? parameterOverrideValueSet.ActualOption.ShortName : string.Empty;
var statePart = parameterOverrideValueSet.ActualState != null ? parameterOverrideValueSet.ActualState.ShortName : string.Empty;
this.Name = string.Format("{0}{1}\\{2}\\{3}", spaces, parameterOverride.ParameterType.Name, optionPart, statePart);
this.ShortName = string.Format("{0}\\{1}\\{2}", parameterOverride.ParameterType.ShortName, optionPart, statePart);
this.Format = NumberFormat.Format(parameterOverride.ParameterType, parameterOverride.Scale);
this.Switch = parameterOverrideValueSet.ValueSwitch.ToString();
this.ModelCode = parameterOverrideValueSet.ModelCode();
var sampledFunctionParameterType = this.ParameterType as SampledFunctionParameterType;
if (sampledFunctionParameterType != null)
{
this.Id = parameterOverrideValueSet.Iid.ToString();
this.Switch = parameterOverrideValueSet.ValueSwitch.ToString();
this.ActualValue = "N/A";
this.ComputedValue = "N/A";
this.ManualValue = "N/A";
this.ReferenceValue = "N/A";
this.Formula = string.Empty;
this.Format = NumberFormat.Format(this.ParameterType, null);
this.ModelCode = parameterOverrideValueSet.ModelCode();
// do not create sub rows, the parameter row represents the only valueset that exists
return;
}
var compoundParameterType = parameterOverride.ParameterType as CompoundParameterType;
if (compoundParameterType == null)
{
this.ActualValue = ParameterValueConverter.ConvertToObject(this.ParameterType, parameterOverrideValueSet.ActualValue[0]);
this.ComputedValue = ParameterValueConverter.ConvertToObject(this.ParameterType, parameterOverrideValueSet.Computed[0]);
this.ManualValue = ParameterValueConverter.ConvertToObject(this.ParameterType, parameterOverrideValueSet.Manual[0]);
this.ReferenceValue = ParameterValueConverter.ConvertToObject(this.ParameterType, parameterOverrideValueSet.Reference[0]);
this.Formula = parameterOverrideValueSet.Formula[0];
}
else
{
// create rows for each component
foreach (ParameterTypeComponent component in compoundParameterType.Component)
{
var componentExcelRow = new ComponentExcelRow(component, parameterOverrideValueSet, this.Level + 1);
this.ContainedRows.Add(componentExcelRow);
componentExcelRow.Container = this;
}
}
}
}
public virtual string AsDOTString()
{
NumberFormat nf = NumberFormat.GetNumberInstance();
nf.SetMaximumFractionDigits(3);
nf.SetMinimumFractionDigits(1);
StringBuilder result = new StringBuilder();
ISet nodes = GetNodes();
result.Append("digraph G {\n");
// result.append("page = \"8.5,11\";\n");
// result.append("margin = \"0.25\";\n");
// Heuristic number of pages
int sz = arcs.Count;
int ht = 105;
int mag = 250;
while (sz > mag)
{
ht += 105;
mag *= 2;
}
int wd = 8;
mag = 500;
while (sz > mag)
{
wd += 8;
mag *= 4;
}
double htd = ht / 10.0;
result.Append("size = \"" + wd + "," + htd + "\";\n");
result.Append("graph [rankdir = \"LR\"];\n");
result.Append("graph [ranksep = \"0.2\"];\n");
foreach (object node in nodes)
{
string cleanString = StringUtils.FileNameClean(node.ToString());
result.Append(cleanString);
result.Append(" [ ");
// if (getEndNodes().contains(node)) {
// result.append("label=\"" + node.toString() + "\", style=filled, ");
// } else
result.Append("label=\"" + node.ToString() + "\"");
result.Append("height=\"0.3\", width=\"0.3\"");
result.Append(" ];\n");
foreach (TransducerGraph.Arc arc in GetArcsBySource(node))
{
result.Append(StringUtils.FileNameClean(arc.GetSourceNode().ToString()));
result.Append(" -> ");
result.Append(StringUtils.FileNameClean(arc.GetTargetNode().ToString()));
result.Append(" [ ");
result.Append("label=\"");
result.Append(arc.GetInput());
result.Append(" : ");
// result.append(arc.getOutput());
object output = arc.GetOutput();
string wt = string.Empty;
if (output is Number)
{
double dd = ((Number)output);
if (dd == -0.0d)
{
result.Append(nf.Format(0.0d));
}
else
{
result.Append(nf.Format(output));
}
int weight;
if (dotWeightInverted)
{
weight = (int)(20.0 - dd);
}
else
{
weight = (int)dd;
}
if (weight > 0)
{
wt = ", weight = \"" + weight + "\"";
}
if (dotWeightInverted && dd <= 2.0 || (!dotWeightInverted) && dd >= 20.0)
{
wt += ", style=bold";
}
}
else
{
result.Append(output);
}
result.Append("\"");
result.Append(wt);
// result.append("fontsize = 14 ");
if (arc.GetInput().ToString().Equals("EPSILON"))
{
result.Append(", style = \"dashed\" ");
}
else
{
result.Append(", style = \"solid\" ");
}
// result.append(", weight = \"" + arc.getOutput() + "\" ");
result.Append("];\n");
}
}
result.Append("}\n");
return(result.ToString());
}
public static void ConstructInputOutputData(StringBuilder inputData, StringBuilder
expectedOutput, int numOfInputLines)
{
for (int i = 0; i < numOfInputLines; i++)
{
inputData.Append(idFormat.Format(i));
inputData.Append("-").Append(idFormat.Format(i + 1));
inputData.Append("-").Append(idFormat.Format(i + 2));
inputData.Append("-").Append(idFormat.Format(i + 3));
inputData.Append("-").Append(idFormat.Format(i + 4));
inputData.Append("-").Append(idFormat.Format(i + 5));
inputData.Append("-").Append(idFormat.Format(i + 6));
inputData.Append("\n");
expectedOutput.Append(idFormat.Format(i + 3));
expectedOutput.Append("-").Append(idFormat.Format(i + 2));
expectedOutput.Append("-").Append(idFormat.Format(i + 1));
expectedOutput.Append("-").Append(idFormat.Format(i + 5));
expectedOutput.Append("-").Append(idFormat.Format(i + 6));
expectedOutput.Append("-").Append(idFormat.Format(i + 6));
expectedOutput.Append("-").Append(idFormat.Format(i + 5));
expectedOutput.Append("-").Append(idFormat.Format(i + 1));
expectedOutput.Append("-").Append(idFormat.Format(i + 2));
expectedOutput.Append("-").Append(idFormat.Format(i + 3));
expectedOutput.Append("-").Append(idFormat.Format(i + 0));
expectedOutput.Append("-").Append(idFormat.Format(i + 1));
expectedOutput.Append("-").Append(idFormat.Format(i + 2));
expectedOutput.Append("-").Append(idFormat.Format(i + 3));
expectedOutput.Append("-").Append(idFormat.Format(i + 4));
expectedOutput.Append("-").Append(idFormat.Format(i + 5));
expectedOutput.Append("-").Append(idFormat.Format(i + 6));
expectedOutput.Append("\n");
}
System.Console.Out.WriteLine("inputData:");
System.Console.Out.WriteLine(inputData.ToString());
System.Console.Out.WriteLine("ExpectedData:");
System.Console.Out.WriteLine(expectedOutput.ToString());
}
public static string ToString(int[][] counts, Object[] rowLabels, Object[] colLabels, int labelSize, int cellSize, NumberFormat nf, bool printTotals)
{
if (counts.Length == 0 || counts[0].Length == 0)
return @"";
int[] rowTotals = new int[counts.Length];
int[] colTotals = new int[counts[0].Length];
int total = 0;
for (int i = 0; i < counts.Length; i++)
{
for (int j = 0; j < counts[i].Length; j++)
{
rowTotals[i] += counts[i][j];
colTotals[j] += counts[i][j];
total += counts[i][j];
}
}
StringBuilder result = new StringBuilder();
if (colLabels != null)
{
result.Append(StringUtils.PadLeft(@"", labelSize));
for (int j = 0; j < counts[0].Length; j++)
{
string s = (colLabels[j] == null ? @"null" : colLabels[j].ToString());
if (s.Length() > cellSize - 1)
{
s = s.Substring(0, cellSize - 1);
}
s = StringUtils.PadLeft(s, cellSize);
result.Append(s);
}
if (printTotals)
{
result.Append(StringUtils.PadLeftOrTrim(@"Total", cellSize));
}
result.Append('\n');
}
for (int i = 0; i < counts.Length; i++)
{
if (rowLabels != null)
{
string s = (rowLabels[i] == null ? @"null" : rowLabels[i].ToString());
s = StringUtils.PadOrTrim(s, labelSize);
result.Append(s);
}
for (int j = 0; j < counts[i].Length; j++)
{
result.Append(StringUtils.PadLeft(nf.Format(counts[i][j]), cellSize));
}
if (printTotals)
{
result.Append(StringUtils.PadLeft(nf.Format(rowTotals[i]), cellSize));
}
result.Append('\n');
}
if (printTotals)
{
result.Append(StringUtils.Pad(@"Total", labelSize));
foreach (int colTotal in colTotals)
{
result.Append(StringUtils.PadLeft(nf.Format(colTotal), cellSize));
}
result.Append(StringUtils.PadLeft(nf.Format(total), cellSize));
}
return result.ToString();
}
/// <summary>
/// This method is used to get the value converted to <see cref="string"/> and
/// escaped using the given <see cref="IEscapeQuerySyntax"/>.
/// </summary>
/// <param name="escaper">The <see cref="IEscapeQuerySyntax"/> used to escape the value <see cref="string"/></param>
/// <returns>The value converted to <see cref="string"/> and escaped</returns>
protected string GetTermEscaped(IEscapeQuerySyntax escaper)
{
return(escaper.Escape(numberFormat.Format(this.value),
CultureInfo.CurrentCulture, EscapeQuerySyntaxType.NORMAL));
}
protected string GetFacilities(object obj)
{
return(NumberFormat.Format(((WasteTransfers.ActivityTreeListRow)obj).Facilities));
}
/// <summary>
/// Manual insertion of all facility level details into List, which is in turn bound to GridView
/// </summary>
private void populateDetails(int facilityReportId)
{
FACILITYDETAIL_DETAIL fac = Facility.GetFacilityDetails(facilityReportId).First();
FACILITYDETAIL_COMPETENTAUTHORITYPARTY authority = Facility.GetFacilityCompetentAuthority(facilityReportId).First();
List <FacilityDetailElement> elements = new List <FacilityDetailElement>();
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "FacilityDetailsTitle"),
String.Empty,
true));
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "ParentCompanyName"),
ConfidentialFormat.Format(fac.ParentCompanyName, fac.ConfidentialIndicator)));
// Take "Valid" string from FACILITY_DETAIL_DETAIL
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "Coords"),
CoordinateFormat.Format(fac.Coordinates, "VALID")));
//NUTS is voluntary. Only add if reported.
if (!string.IsNullOrEmpty(fac.NUTSRegionSourceCode))
{
//Add both reported and geo-coded value - if they differ.
if (fac.NUTSRegionSourceCode != fac.NUTSRegionLevel2Code)
{
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "NUTSMap"),
LOVResources.NutsRegionName(fac.NUTSRegionLevel2Code)));
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "NUTSReported"),
LOVResources.NutsRegionName(fac.NUTSRegionSourceCode)));
}
else
{
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "NUTS"),
LOVResources.NutsRegionName(fac.NUTSRegionLevel2Code)));
}
}
//Add both reported and geo-coded value - if they differ.
if (fac.RiverBasinDistrictSourceCode != fac.RiverBasinDistrictCode)
{
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "RBDMap"),
LOVResources.RiverBasinDistrictName(fac.RiverBasinDistrictCode)));
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "RBDReported"),
LOVResources.RiverBasinDistrictName(fac.RiverBasinDistrictSourceCode)));
}
else
{
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "RBD"),
LOVResources.RiverBasinDistrictName(fac.RiverBasinDistrictCode)));
}
//NACE code reported on sub-activity level, except for EPER where some is reported on Activity level
//string naceCode = !String.IsNullOrEmpty(fac.NACESubActivityCode) ? fac.NACESubActivityCode : fac.NACEActivityCode;
string naceCode = !String.IsNullOrEmpty(fac.NACEActivityCode) ? fac.NACEActivityCode : fac.NACESectorCode;
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "NACE"),
LOVResources.NaceActivityName(naceCode)));
//Production volume is voluntary. Only add if reported.
if (fac.ProductionVolumeQuantity != null)
{
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "ProductionV"),
String.Format("{0} {1} {2}",
fac.ProductionVolumeProductName,
NumberFormat.Format(fac.ProductionVolumeQuantity),
LOVResources.UnitName(fac.ProductionVolumeUnitCode))));
}
//No. of IPPC installations is voluntary. Only add if reported.
if (fac.TotalIPPCInstallationQuantity != null)
{
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "IPPC"),
NumberFormat.Format(fac.TotalIPPCInstallationQuantity)));
}
//No. of emplyees is voluntary. Only add if reported.
if (fac.TotalEmployeeQuantity != null)
{
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "Employ"),
NumberFormat.Format(fac.TotalEmployeeQuantity)));
}
//Operating hours is voluntary. Only add if reported.
if (fac.OperatingHours != null)
{
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "OperationHours"),
String.Format("{0}", fac.OperatingHours)));
}
//Website is voluntary. Only add if reported.
if (!string.IsNullOrEmpty(fac.WebsiteCommunication))
{
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "WebSite"),
String.Format("{0}", fac.WebsiteCommunication)));
}
if (fac.ConfidentialIndicatorCode != null)
{
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Pollutant", "ConfidentialityReason"),
LOVResources.ConfidentialityReason(fac.ConfidentialIndicatorCode)));
}
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Common", "NationalID") + ":",
FacilityDetailsFormat.FormatNationalId(fac)));
// This is not the most elegant way to obtain a spacing spacing
elements.Add(new FacilityDetailElement(String.Empty, String.Empty));
string updatedDateString = authority.CALastUpdate == null
? Resources.GetGlobal("Facility", "LastUpdatedUnknown")
: authority.CALastUpdate.Format();
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "CompetentA"),
String.Format(Resources.GetGlobal("Facility", "LastUpdated"), updatedDateString),
true));
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "Name"),
String.Format(authority.CAName)));
if (authority.CAAddress != null)
{
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "Address"),
AddressFormat.Format(authority.CAAddress, authority.CACity, authority.CAPostalCode, false)));
}
if (authority.CATelephoneCommunication != null)
{
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "Phone"),
String.Format(authority.CATelephoneCommunication)));
}
if (authority.CAFaxCommunication != null)
{
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "Fax"),
String.Format(authority.CAFaxCommunication)));
}
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "Email"),
String.Format(authority.CAEmailCommunication)));
if (authority.CAContactPersonName != null)
{
elements.Add(new FacilityDetailElement(
Resources.GetGlobal("Facility", "CPerson"),
String.Format(authority.CAContactPersonName)));
}
// data binding
facilityreportDetails.DataSource = elements;
facilityreportDetails.DataBind();
}
protected string GetPTFacilitiesConf(object obj)
{
IndustrialActivity.ConfidentialTransfersRow row = (IndustrialActivity.ConfidentialTransfersRow)obj;
return(NumberFormat.Format(row.FacilitiesConfidential));
}
protected string GetPRFacilitiesSoilConf(object obj)
{
IndustrialActivity.ConfidentialReleasesRow row = (IndustrialActivity.ConfidentialReleasesRow)obj;
return(NumberFormat.Format(row.FacilitiesSoilConfidential));
}
protected internal virtual string GetName()
{
return("SGDWithAdaGradAndFOBOS" + bSize + "_lambda" + nf.Format(lambda) + "_alpha" + nf.Format(alpha));
}
/// <exception cref="System.IO.IOException"/>
public virtual void Train(IList <Tree> sentences, IdentityHashMap <Tree, byte[]> compressedParses, Treebank testTreebank, string modelPath, string resultsRecordPath)
{
// process:
// we come up with a cost and a derivative for the model
// we always use the gold tree as the example to train towards
// every time through, we will look at the top N trees from
// the LexicalizedParser and pick the best one according to
// our model (at the start, this is essentially random)
// we use QN to minimize the cost function for the model
// to do this minimization, we turn all of the matrices in the
// DVModel into one big Theta, which is the set of variables to
// be optimized by the QN.
Timing timing = new Timing();
long maxTrainTimeMillis = op.trainOptions.maxTrainTimeSeconds * 1000;
int batchCount = 0;
int debugCycle = 0;
double bestLabelF1 = 0.0;
if (op.trainOptions.useContextWords)
{
foreach (Tree tree in sentences)
{
Edu.Stanford.Nlp.Trees.Trees.ConvertToCoreLabels(tree);
tree.SetSpans();
}
}
// for AdaGrad
double[] sumGradSquare = new double[dvModel.TotalParamSize()];
Arrays.Fill(sumGradSquare, 1.0);
int numBatches = sentences.Count / op.trainOptions.batchSize + 1;
log.Info("Training on " + sentences.Count + " trees in " + numBatches + " batches");
log.Info("Times through each training batch: " + op.trainOptions.trainingIterations);
log.Info("QN iterations per batch: " + op.trainOptions.qnIterationsPerBatch);
for (int iter = 0; iter < op.trainOptions.trainingIterations; ++iter)
{
IList <Tree> shuffledSentences = new List <Tree>(sentences);
Java.Util.Collections.Shuffle(shuffledSentences, dvModel.rand);
for (int batch = 0; batch < numBatches; ++batch)
{
++batchCount;
// This did not help performance
//log.info("Setting AdaGrad's sum of squares to 1...");
//Arrays.fill(sumGradSquare, 1.0);
log.Info("======================================");
log.Info("Iteration " + iter + " batch " + batch);
// Each batch will be of the specified batch size, except the
// last batch will include any leftover trees at the end of
// the list
int startTree = batch * op.trainOptions.batchSize;
int endTree = (batch + 1) * op.trainOptions.batchSize;
if (endTree > shuffledSentences.Count)
{
endTree = shuffledSentences.Count;
}
ExecuteOneTrainingBatch(shuffledSentences.SubList(startTree, endTree), compressedParses, sumGradSquare);
long totalElapsed = timing.Report();
log.Info("Finished iteration " + iter + " batch " + batch + "; total training time " + totalElapsed + " ms");
if (maxTrainTimeMillis > 0 && totalElapsed > maxTrainTimeMillis)
{
// no need to debug output, we're done now
break;
}
if (op.trainOptions.debugOutputFrequency > 0 && batchCount % op.trainOptions.debugOutputFrequency == 0)
{
log.Info("Finished " + batchCount + " total batches, running evaluation cycle");
// Time for debugging output!
double tagF1 = 0.0;
double labelF1 = 0.0;
if (testTreebank != null)
{
EvaluateTreebank evaluator = new EvaluateTreebank(AttachModelToLexicalizedParser());
evaluator.TestOnTreebank(testTreebank);
labelF1 = evaluator.GetLBScore();
tagF1 = evaluator.GetTagScore();
if (labelF1 > bestLabelF1)
{
bestLabelF1 = labelF1;
}
log.Info("Best label f1 on dev set so far: " + Nf.Format(bestLabelF1));
}
string tempName = null;
if (modelPath != null)
{
tempName = modelPath;
if (modelPath.EndsWith(".ser.gz"))
{
tempName = Sharpen.Runtime.Substring(modelPath, 0, modelPath.Length - 7) + "-" + Filename.Format(debugCycle) + "-" + Nf.Format(labelF1) + ".ser.gz";
}
SaveModel(tempName);
}
string statusLine = ("CHECKPOINT:" + " iteration " + iter + " batch " + batch + " labelF1 " + Nf.Format(labelF1) + " tagF1 " + Nf.Format(tagF1) + " bestLabelF1 " + Nf.Format(bestLabelF1) + " model " + tempName + op.trainOptions + " word vectors: "
+ op.lexOptions.wordVectorFile + " numHid: " + op.lexOptions.numHid);
log.Info(statusLine);
if (resultsRecordPath != null)
{
FileWriter fout = new FileWriter(resultsRecordPath, true);
// append
fout.Write(statusLine);
fout.Write("\n");
fout.Close();
}
++debugCycle;
}
}
long totalElapsed_1 = timing.Report();
if (maxTrainTimeMillis > 0 && totalElapsed_1 > maxTrainTimeMillis)
{
// no need to debug output, we're done now
log.Info("Max training time exceeded, exiting");
break;
}
}
}
private static String NumberToString(/*Number*/ object number)
{
return(number == null ? "*" : ESCAPER.Escape(NUMBER_FORMAT.Format(number),
LOCALE, EscapeQuerySyntax.Type.STRING).toString());
}
/// <summary>Provides a more detailed report for a given operation.</summary>
/// <remarks>
/// Provides a more detailed report for a given operation. This will output the
/// keys and values for all input and then sort based on measurement type and
/// attempt to show rates for various metrics which have expected types to be
/// able to measure there rate. Currently this will show rates for bytes
/// written, success count, files created, directory entries, op count and
/// bytes read if the variable for time taken is available for each measurement
/// type.
/// </remarks>
/// <param name="operation">the operation that is being reported on.</param>
/// <param name="input">the set of data for that that operation.</param>
/// <param name="os">
/// any print writer for which output should be written to (along with
/// the logging library)
/// </param>
internal virtual void OpReport(string operation, IList <OperationOutput> input, PrintWriter
os)
{
WriteMessage("Basic report for operation type " + operation, os);
WriteMessage(GetSectionDelimiter(), os);
foreach (OperationOutput data in input)
{
WriteMessage("Measurement \"" + data.GetMeasurementType() + "\" = " + data.GetValue
(), os);
}
// split up into measurement types for rates...
IDictionary <string, OperationOutput> combined = new SortedDictionary <string, OperationOutput
>();
foreach (OperationOutput data_1 in input)
{
if (combined.Contains(data_1.GetMeasurementType()))
{
OperationOutput curr = combined[data_1.GetMeasurementType()];
combined[data_1.GetMeasurementType()] = OperationOutput.Merge(curr, data_1);
}
else
{
combined[data_1.GetMeasurementType()] = data_1;
}
}
// handle the known types
OperationOutput timeTaken = combined[OkTimeTaken];
if (timeTaken != null)
{
long mTaken = long.Parse(timeTaken.GetValue().ToString());
if (mTaken > 0)
{
NumberFormat formatter = Formatter.GetDecimalFormatter();
foreach (string measurementType in combined.Keys)
{
double rate = null;
string rateType = string.Empty;
if (measurementType.Equals(BytesWritten))
{
long mbWritten = long.Parse(combined[measurementType].GetValue().ToString()) / (Constants
.Megabytes);
rate = (double)mbWritten / (double)(mTaken / 1000.0d);
rateType = "MB/sec";
}
else
{
if (measurementType.Equals(Successes))
{
long succ = long.Parse(combined[measurementType].GetValue().ToString());
rate = (double)succ / (double)(mTaken / 1000.0d);
rateType = "successes/sec";
}
else
{
if (measurementType.Equals(FilesCreated))
{
long filesCreated = long.Parse(combined[measurementType].GetValue().ToString());
rate = (double)filesCreated / (double)(mTaken / 1000.0d);
rateType = "files created/sec";
}
else
{
if (measurementType.Equals(DirEntries))
{
long entries = long.Parse(combined[measurementType].GetValue().ToString());
rate = (double)entries / (double)(mTaken / 1000.0d);
rateType = "directory entries/sec";
}
else
{
if (measurementType.Equals(OpCount))
{
long opCount = long.Parse(combined[measurementType].GetValue().ToString());
rate = (double)opCount / (double)(mTaken / 1000.0d);
rateType = "operations/sec";
}
else
{
if (measurementType.Equals(BytesRead))
{
long mbRead = long.Parse(combined[measurementType].GetValue().ToString()) / (Constants
.Megabytes);
rate = (double)mbRead / (double)(mTaken / 1000.0d);
rateType = "MB/sec";
}
}
}
}
}
}
if (rate != null)
{
WriteMessage("Rate for measurement \"" + measurementType + "\" = " + formatter.Format
(rate) + " " + rateType, os);
}
}
}
}
WriteMessage(GetSectionDelimiter(), os);
}
/// <summary>
/// Process the current <see cref="ParameterSubscription"/>s to create any ContainedRows for value-sets
/// </summary>
/// <param name="processedValueSets">
/// The <see cref="Thing"/>s for which the values need to be restored to what the user provided.
/// </param>
private void ProcessParameter(IReadOnlyDictionary <Guid, ProcessedValueSet> processedValueSets)
{
if (!this.Thing.IsOptionDependent && this.Thing.StateDependence == null)
{
var scalarParameterType = this.Thing.ParameterType as ScalarParameterType;
if (scalarParameterType != null)
{
var parameterValueSet = this.Thing.ValueSet.FirstOrDefault();
if (parameterValueSet != null)
{
ProcessedValueSet processedValueSet;
if (processedValueSets.TryGetValue(parameterValueSet.Iid, out processedValueSet))
{
parameterValueSet = (ParameterSubscriptionValueSet)processedValueSet.ClonedThing;
}
this.Id = parameterValueSet.Iid.ToString();
this.Switch = parameterValueSet.ValueSwitch.ToString();
this.ActualValue = ParameterValueConverter.ConvertToObject(scalarParameterType, parameterValueSet.ActualValue[0]);
this.ComputedValue = ParameterValueConverter.ConvertToObject(scalarParameterType, parameterValueSet.Computed[0]);
this.ManualValue = ParameterValueConverter.ConvertToObject(scalarParameterType, parameterValueSet.Manual[0]);
this.ReferenceValue = ParameterValueConverter.ConvertToObject(scalarParameterType, parameterValueSet.Reference[0]);
this.Format = NumberFormat.Format(this.Thing.ParameterType, this.Thing.Scale);
this.ModelCode = parameterValueSet.ModelCode();
}
// do not create sub rows, the parameter row represents the only valueset that exists
return;
}
var compoundParameterType = this.Thing.ParameterType as CompoundParameterType;
if (compoundParameterType != null)
{
this.Format = NumberFormat.Format(this.Thing.ParameterType, this.Thing.Scale);
this.ModelCode = this.Thing.ModelCode();
var parameterValueSet = this.Thing.ValueSet.FirstOrDefault();
if (parameterValueSet != null)
{
ProcessedValueSet processedValueSet;
if (processedValueSets.TryGetValue(parameterValueSet.Iid, out processedValueSet))
{
parameterValueSet = (ParameterSubscriptionValueSet)processedValueSet.ClonedThing;
}
this.Id = parameterValueSet.Iid.ToString();
this.Switch = parameterValueSet.ValueSwitch.ToString();
}
// create rows for each component
foreach (ParameterTypeComponent component in compoundParameterType.Component)
{
var componentExcelRow = new ComponentExcelRow(component, parameterValueSet, this.Level + 1);
this.ContainedRows.Add(componentExcelRow);
componentExcelRow.Container = this;
}
}
return;
}
var sortedvaluesets = this.Thing.ValueSet.OrderBy(p => p, new ParameterSubscriptionValueSetComparer());
foreach (var valueset in sortedvaluesets)
{
if (valueset.ActualState != null && valueset.ActualState.Kind == ActualFiniteStateKind.FORBIDDEN)
{
continue;
}
var valueSetRow = new ParameterSubscriptionValuesetExcelRow(valueset, this.Level + 1, processedValueSets);
this.ContainedRows.Add(valueSetRow);
valueSetRow.Container = this;
}
}
/// <summary>Dumps out the given options for the given config extractor</summary>
/// <param name="cfg">the config to write to the log</param>
internal static void DumpOptions(Org.Apache.Hadoop.FS.Slive.ConfigExtractor cfg)
{
if (cfg == null)
{
return;
}
Log.Info("Base directory = " + cfg.GetBaseDirectory());
Log.Info("Data directory = " + cfg.GetDataPath());
Log.Info("Output directory = " + cfg.GetOutputPath());
Log.Info("Result file = " + cfg.GetResultFile());
Log.Info("Grid queue = " + cfg.GetQueueName());
Log.Info("Should exit on first error = " + cfg.ShouldExitOnFirstError());
{
string duration = "Duration = ";
if (cfg.GetDurationMilliseconds() == int.MaxValue)
{
duration += "unlimited";
}
else
{
duration += cfg.GetDurationMilliseconds() + " milliseconds";
}
Log.Info(duration);
}
Log.Info("Map amount = " + cfg.GetMapAmount());
Log.Info("Reducer amount = " + cfg.GetReducerAmount());
Log.Info("Operation amount = " + cfg.GetOpCount());
Log.Info("Total file limit = " + cfg.GetTotalFiles());
Log.Info("Total dir file limit = " + cfg.GetDirSize());
{
string read = "Read size = ";
if (cfg.ShouldReadFullFile())
{
read += "entire file";
}
else
{
read += cfg.GetReadSize() + " bytes";
}
Log.Info(read);
}
{
string write = "Write size = ";
if (cfg.ShouldWriteUseBlockSize())
{
write += "blocksize";
}
else
{
write += cfg.GetWriteSize() + " bytes";
}
Log.Info(write);
}
{
string append = "Append size = ";
if (cfg.ShouldAppendUseBlockSize())
{
append += "blocksize";
}
else
{
append += cfg.GetAppendSize() + " bytes";
}
Log.Info(append);
}
{
string bsize = "Block size = ";
bsize += cfg.GetBlockSize() + " bytes";
Log.Info(bsize);
}
if (cfg.GetRandomSeed() != null)
{
Log.Info("Random seed = " + cfg.GetRandomSeed());
}
if (cfg.GetSleepRange() != null)
{
Log.Info("Sleep range = " + cfg.GetSleepRange() + " milliseconds");
}
Log.Info("Replication amount = " + cfg.GetReplication());
Log.Info("Operations are:");
NumberFormat percFormatter = Formatter.GetPercentFormatter();
IDictionary <Constants.OperationType, OperationData> operations = cfg.GetOperations
();
foreach (Constants.OperationType type in operations.Keys)
{
string name = type.ToString();
Log.Info(name);
OperationData opInfo = operations[type];
Log.Info(" " + opInfo.GetDistribution().ToString());
if (opInfo.GetPercent() != null)
{
Log.Info(" " + percFormatter.Format(opInfo.GetPercent()));
}
else
{
Log.Info(" ???");
}
}
}
protected internal virtual string GetName()
{
return("SGD_InPlace_b" + bSize + "_lambda" + nf.Format(lambda));
}
public virtual void PrintStats()
{
NumberFormat nf = NumberFormat.GetNumberInstance();
nf.SetMaximumFractionDigits(2);
// System.out.println("Node rules");
// System.out.println(nodeRules);
// System.out.println("Parent rules");
// System.out.println(pRules);
// System.out.println("Grandparent rules");
// System.out.println(gPRules);
// Store java code for selSplit
StringBuilder[] javaSB = new StringBuilder[Cutoffs.Length];
for (int i = 0; i < Cutoffs.Length; i++)
{
javaSB[i] = new StringBuilder(" private static String[] splitters" + (i + 1) + " = new String[] {");
}
ClassicCounter <IList <string> > allScores = new ClassicCounter <IList <string> >();
// do value of parent
foreach (string node in nodeRules.Keys)
{
List <Pair <IList <string>, double> > answers = Generics.NewArrayList();
ClassicCounter <IList <string> > cntr = nodeRules[node];
double support = (cntr.TotalCount());
System.Console.Out.WriteLine("Node " + node + " support is " + support);
foreach (IList <string> key in pRules.Keys)
{
if (key[0].Equals(node))
{
// only do it if they match
ClassicCounter <IList <string> > cntr2 = pRules[key];
double support2 = (cntr2.TotalCount());
double kl = Counters.KlDivergence(cntr2, cntr);
System.Console.Out.WriteLine("KL(" + key + "||" + node + ") = " + nf.Format(kl) + "\t" + "support(" + key + ") = " + support2);
double score = kl * support2;
answers.Add(new Pair <IList <string>, double>(key, score));
allScores.SetCount(key, score);
}
}
System.Console.Out.WriteLine("----");
System.Console.Out.WriteLine("Sorted descending support * KL");
answers.Sort(null);
foreach (Pair <IList <string>, double> answer in answers)
{
Pair p = (Pair)answer;
double psd = ((double)p.Second());
System.Console.Out.WriteLine(p.First() + ": " + nf.Format(psd));
if (psd >= Cutoffs[0])
{
IList lst = (IList)p.First();
string nd = (string)lst[0];
string par = (string)lst[1];
for (int j = 0; j < Cutoffs.Length; j++)
{
if (psd >= Cutoffs[j])
{
javaSB[j].Append("\"").Append(nd).Append("^");
javaSB[j].Append(par).Append("\", ");
}
}
}
}
System.Console.Out.WriteLine();
}
/*
* // do value of parent with info gain -- yet to finish this
* for (Iterator it = nodeRules.entrySet().iterator(); it.hasNext(); ) {
* Map.Entry pair = (Map.Entry) it.next();
* String node = (String) pair.getKey();
* Counter cntr = (Counter) pair.getValue();
* double support = (cntr.totalCount());
* System.out.println("Node " + node + " support is " + support);
* ArrayList dtrs = new ArrayList();
* for (Iterator it2 = pRules.entrySet().iterator(); it2.hasNext();) {
* HashMap annotated = new HashMap();
* Map.Entry pair2 = (Map.Entry) it2.next();
* List node2 = (List) pair2.getKey();
* Counter cntr2 = (Counter) pair2.getValue();
* if (node2.get(0).equals(node)) { // only do it if they match
* annotated.put(node2, cntr2);
* }
* }
*
* // upto
*
* List answers = new ArrayList();
* System.out.println("----");
* System.out.println("Sorted descending support * KL");
* Collections.sort(answers,
* new Comparator() {
* public int compare(Object o1, Object o2) {
* Pair p1 = (Pair) o1;
* Pair p2 = (Pair) o2;
* Double p12 = (Double) p1.second();
* Double p22 = (Double) p2.second();
* return p22.compareTo(p12);
* }
* });
* for (int i = 0, size = answers.size(); i < size; i++) {
* Pair p = (Pair) answers.get(i);
* double psd = ((Double) p.second()).doubleValue();
* System.out.println(p.first() + ": " + nf.format(psd));
* if (psd >= CUTOFFS[0]) {
* List lst = (List) p.first();
* String nd = (String) lst.get(0);
* String par = (String) lst.get(1);
* for (int j=0; j < CUTOFFS.length; j++) {
* if (psd >= CUTOFFS[j]) {
* javaSB[j].append("\"").append(nd).append("^");
* javaSB[j].append(par).append("\", ");
* }
* }
* }
* }
* System.out.println();
* }
*/
// do value of grandparent
foreach (IList <string> node_1 in pRules.Keys)
{
List <Pair <IList <string>, double> > answers = Generics.NewArrayList();
ClassicCounter <IList <string> > cntr = pRules[node_1];
double support = (cntr.TotalCount());
if (support < Suppcutoff)
{
continue;
}
System.Console.Out.WriteLine("Node " + node_1 + " support is " + support);
foreach (IList <string> key in gPRules.Keys)
{
if (key[0].Equals(node_1[0]) && key[1].Equals(node_1[1]))
{
// only do it if they match
ClassicCounter <IList <string> > cntr2 = gPRules[key];
double support2 = (cntr2.TotalCount());
double kl = Counters.KlDivergence(cntr2, cntr);
System.Console.Out.WriteLine("KL(" + key + "||" + node_1 + ") = " + nf.Format(kl) + "\t" + "support(" + key + ") = " + support2);
double score = kl * support2;
answers.Add(Pair.MakePair(key, score));
allScores.SetCount(key, score);
}
}
System.Console.Out.WriteLine("----");
System.Console.Out.WriteLine("Sorted descending support * KL");
answers.Sort(null);
foreach (Pair <IList <string>, double> answer in answers)
{
Pair p = (Pair)answer;
double psd = ((double)p.Second());
System.Console.Out.WriteLine(p.First() + ": " + nf.Format(psd));
if (psd >= Cutoffs[0])
{
IList lst = (IList)p.First();
string nd = (string)lst[0];
string par = (string)lst[1];
string gpar = (string)lst[2];
for (int j = 0; j < Cutoffs.Length; j++)
{
if (psd >= Cutoffs[j])
{
javaSB[j].Append("\"").Append(nd).Append("^");
javaSB[j].Append(par).Append("~");
javaSB[j].Append(gpar).Append("\", ");
}
}
}
}
System.Console.Out.WriteLine();
}
System.Console.Out.WriteLine();
System.Console.Out.WriteLine("All scores:");
IPriorityQueue <IList <string> > pq = Counters.ToPriorityQueue(allScores);
while (!pq.IsEmpty())
{
IList <string> key = pq.GetFirst();
double score = pq.GetPriority(key);
pq.RemoveFirst();
System.Console.Out.WriteLine(key + "\t" + score);
}
System.Console.Out.WriteLine(" // Automatically generated by ParentAnnotationStats -- preferably don't edit");
for (int i_1 = 0; i_1 < Cutoffs.Length; i_1++)
{
int len = javaSB[i_1].Length;
javaSB[i_1].Replace(len - 2, len, "};");
System.Console.Out.WriteLine(javaSB[i_1]);
}
System.Console.Out.Write(" public static HashSet splitters = new HashSet(Arrays.asList(");
for (int i_2 = Cutoffs.Length; i_2 > 0; i_2--)
{
if (i_2 == 1)
{
System.Console.Out.Write("splitters1");
}
else
{
System.Console.Out.Write("selectiveSplit" + i_2 + " ? splitters" + i_2 + " : (");
}
}
// need to print extra one to close other things open
for (int i_3 = Cutoffs.Length; i_3 >= 0; i_3--)
{
System.Console.Out.Write(")");
}
System.Console.Out.WriteLine(";");
}
public virtual double TuneDouble(Func function, double[] initial, long msPerTest, StochasticMinimizer.IPropertySetter <double> ps, double lower, double upper, double Tol)
{
double[] xtest = new double[initial.Length];
this.maxTime = msPerTest;
// check for stochastic derivatives
if (!(function is AbstractStochasticCachingDiffFunction))
{
throw new NotSupportedException();
}
AbstractStochasticCachingDiffFunction dfunction = (AbstractStochasticCachingDiffFunction)function;
IList <Pair <double, double> > res = new List <Pair <double, double> >();
Pair <double, double> best = new Pair <double, double>(lower, double.PositiveInfinity);
//this is set to lower because the first it will always use the lower first, so it has to be best
Pair <double, double> low = new Pair <double, double>(lower, double.PositiveInfinity);
Pair <double, double> high = new Pair <double, double>(upper, double.PositiveInfinity);
Pair <double, double> cur = new Pair <double, double>();
Pair <double, double> tmp = new Pair <double, double>();
IList <double> queue = new List <double>();
queue.Add(lower);
queue.Add(upper);
//queue.add(0.5* (lower + upper));
bool toContinue = true;
this.numPasses = 10000;
do
{
System.Array.Copy(initial, 0, xtest, 0, initial.Length);
if (queue.Count != 0)
{
cur.first = queue.Remove(0);
}
else
{
cur.first = 0.5 * (low.First() + high.First());
}
ps.Set(cur.First());
log.Info(string.Empty);
log.Info("About to test with batch size: " + bSize + " gain: " + gain + " and " + ps.ToString() + " set to " + cur.First());
xtest = this.Minimize(function, 1e-100, xtest);
if (double.IsNaN(xtest[0]))
{
cur.second = double.PositiveInfinity;
}
else
{
cur.second = dfunction.ValueAt(xtest);
}
if (cur.Second() < best.Second())
{
CopyPair(best, tmp);
CopyPair(cur, best);
if (tmp.First() > best.First())
{
CopyPair(tmp, high);
}
else
{
// The old best is now the upper bound
CopyPair(tmp, low);
}
// The old best is now the lower bound
queue.Add(0.5 * (cur.First() + high.First()));
}
else
{
// check in the right interval next
if (cur.First() < best.First())
{
CopyPair(cur, low);
}
else
{
if (cur.First() > best.First())
{
CopyPair(cur, high);
}
}
}
if (System.Math.Abs(low.First() - high.First()) < Tol)
{
toContinue = false;
}
res.Add(new Pair <double, double>(cur.First(), cur.Second()));
log.Info(string.Empty);
log.Info("Final value is: " + nf.Format(cur.Second()));
log.Info("Optimal so far using " + ps.ToString() + " is: " + best.First());
}while (toContinue);
//output the results to screen.
log.Info("-------------");
log.Info(" RESULTS ");
log.Info(ps.GetType().ToString());
log.Info("-------------");
log.Info(" val , function after " + msPerTest + " ms");
foreach (Pair <double, double> re in res)
{
log.Info(re.First() + " , " + re.Second());
}
log.Info(string.Empty);
log.Info(string.Empty);
return(best.First());
}
protected string GetReportingFacilities(object obj)
{
TimeSeriesClasses.PollutantTransfers row = (TimeSeriesClasses.PollutantTransfers)obj;
return(NumberFormat.Format(row.Facilities));
}
public static string ToString(byte[] a, NumberFormat nf)
{
if (a == null)
return null;
if (a.Length == 0)
return @"[]";
StringBuilder b = new StringBuilder();
b.Append('[');
for (int i = 0; i < a.Length - 1; i++)
{
string s;
if (nf == null)
{
s = String.ValueOf(a[i]);
}
else
{
s = nf.Format(a[i]);
}
b.Append(s);
b.Append(@", ");
}
string s;
if (nf == null)
{
s = String.ValueOf(a[a.Length - 1]);
}
else
{
s = nf.Format(a[a.Length - 1]);
}
b.Append(s);
b.Append(']');
return b.ToString();
}