private Dictionary <string, string> DetermineLabels(PrometheusMetricDefinition metricDefinition, ScrapeResult scrapeResult, MeasuredMetric measuredMetric, Dictionary <string, string> defaultLabels)
{
var labels = new Dictionary <string, string>(scrapeResult.Labels.Select(label => new KeyValuePair <string, string>(label.Key.SanitizeForPrometheusLabelKey(), label.Value)));
if (measuredMetric.IsDimensional)
{
labels.Add(measuredMetric.DimensionName.SanitizeForPrometheusLabelKey(), measuredMetric.DimensionValue);
}
if (metricDefinition?.Labels?.Any() == true)
{
foreach (var customLabel in metricDefinition.Labels)
{
var customLabelKey = customLabel.Key.SanitizeForPrometheusLabelKey();
if (labels.ContainsKey(customLabelKey))
{
_logger.LogWarning("Custom label {CustomLabelName} was already specified with value '{LabelValue}' instead of '{CustomLabelValue}'. Ignoring...", customLabel.Key, labels[customLabelKey], customLabel.Value);
continue;
}
labels.Add(customLabelKey, customLabel.Value);
}
}
foreach (var defaultLabel in defaultLabels)
{
var defaultLabelKey = defaultLabel.Key.SanitizeForPrometheusLabelKey();
if (labels.ContainsKey(defaultLabelKey) == false)
{
labels.Add(defaultLabelKey, defaultLabel.Value);
}
}
// Add the tenant id
var metricsDeclaration = _metricsDeclarationProvider.Get(applyDefaults: true);
if (labels.ContainsKey("tenant_id") == false)
{
labels.Add("tenant_id", metricsDeclaration.AzureMetadata.TenantId);
}
// Transform labels, if need be
if (_prometheusConfiguration.CurrentValue.Labels != null)
{
labels = LabelTransformer.TransformLabels(_prometheusConfiguration.CurrentValue.Labels.Transformation, labels);
}
var orderedLabels = labels.OrderBy(kvp => kvp.Key).ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
return(orderedLabels);
}
public void TransformLabels_TransformationToLowercase_LabelValuesTransformedToLowercase()
{
// Arrange
var inputLabels = new Dictionary <string, string>
{
{ BogusGenerator.Name.FirstName(), BogusGenerator.Name.FirstName().ToUpper() },
{ BogusGenerator.Name.FirstName(), BogusGenerator.Name.FirstName() },
{ BogusGenerator.Name.FirstName(), BogusGenerator.Name.FirstName().ToUpperInvariant() }
};
// Act
var transformedLabels = LabelTransformer.TransformLabels(LabelTransformation.Lowercase, inputLabels);
// Assert
Assert.Equal(inputLabels.Count, transformedLabels.Count);
foreach (var inputLabel in inputLabels)
{
Assert.True(transformedLabels.ContainsKey(inputLabel.Key));
Assert.Equal(inputLabel.Value.ToLower(), transformedLabels[inputLabel.Key]);
}
}
public void TransformLabels_NoTransformationWithDifferentCases_NoTransformationWasApplied()
{
// Arrange
var inputLabels = new Dictionary <string, string>
{
{ BogusGenerator.Name.FirstName(), BogusGenerator.Name.FirstName() },
{ BogusGenerator.Name.FirstName(), BogusGenerator.Name.FirstName() },
{ BogusGenerator.Name.FirstName(), BogusGenerator.Name.FirstName() },
{ BogusGenerator.Name.FirstName(), BogusGenerator.Name.FirstName() },
{ BogusGenerator.Name.FirstName(), BogusGenerator.Name.FirstName() },
};
// Act
var transformedLabels = LabelTransformer.TransformLabels(LabelTransformation.None, inputLabels);
// Assert
Assert.Equal(inputLabels.Count, transformedLabels.Count);
foreach (var inputLabel in inputLabels)
{
Assert.True(transformedLabels.ContainsKey(inputLabel.Key));
Assert.Equal(inputLabel.Value, transformedLabels[inputLabel.Key]);
}
}
/// <summary>
/// Run teaching epoch.
/// </summary>
/// <returns></returns>
public override double RunEpoch()
{
var data = DataTransformer.Transform(Path(Config.Train.Data));
int[] permutation = null;
switch (Config.Batch)
{
case NetworkConfig.BatchType.Full:
{
permutation = Combinatorics.GeneratePermutation(data.RowCount);
break;
}
case NetworkConfig.BatchType.Mini:
{
var batchSize = data.RowCount;
if (Config.BatchSize > 0 && Config.BatchSize <= data.RowCount)
{
batchSize = Config.BatchSize;
}
permutation = Combinatorics.GenerateVariation(data.RowCount, batchSize);
break;
}
default:
{
permutation = new int[] { DiscreteUniform.Sample(0, data.RowCount - 1) };
break;
}
}
if (data.ColumnCount != Config.Inputs)
{
throw new Exception("Invalid data format.");
}
accumulatedCost = 0;
accumulatedGradient = new Matrix <double> [layers.Count];
var tasks = new Task[permutation.Length];
for (int i = 0; i < permutation.Length; i++)
{
var index = permutation[i];
tasks[i] = Task.Factory.StartNew((object trainState) =>
{
var state = trainState as TrainState;
Train(
state.input,
state.target,
out double cost,
out List <Matrix <double> > gradient
);
// Console.WriteLine("\nTask cost {0} + {1}", cost, accumulatedCost);
accumulatedCost += cost;
for (int g = 0; g < gradient.Count; g++)
{
if (accumulatedGradient[g] == null)
{
accumulatedGradient[g] = Matrix <double> .Build.Dense(gradient[g].RowCount, gradient[g].ColumnCount);
}
accumulatedGradient[g] += gradient[g];
}
}, new TrainState
{
input = data.Row(index),
target = LabelTransformer.TransformLabels().Row(index)
}, TaskCreationOptions.LongRunning);
//tasks[i].Wait(1);
/*
* Train(
* data.Row(index),
* LabelTransformer.TransformLabels().Row(index),
* out double cost,
* out List<Matrix<double>> gradient
* );
*
* accumulatedCost += cost;
*
* for (int g = 0; g < gradient.Count; g++)
* {
* if (accumulatedGradient[g] == null)
* {
* accumulatedGradient[g] = Matrix<double>.Build.Dense(gradient[g].RowCount, gradient[g].ColumnCount);
* }
* accumulatedGradient[g] += gradient[g];
* }
*/
}
Task.WaitAll(tasks, Timeout.Infinite, CancellationToken.None);
// Go through each layer of the network in opposite direction.
for (int i = layers.Count - 1; i >= 0; i--)
{
var layer = layers[i];
var layerGradient = accumulatedGradient[layers.Count - 1 - i];
if (Program.Debug)
{
Console.WriteLine("accumulated gradient for layer {0}: {1}", i, layerGradient);
}
for (int j = 0; j < layer.Size; j++)
{
for (int k = 0; k < layer.Neurons[j].InputCount; k++)
{
var v = layerGradient.Row(j);
layer.Neurons[j].Weights[k] += layerGradient.Row(j).At(k + 1);
}
layer.Neurons[j].Bias += layerGradient.Row(j).At(0);
}
}
return(accumulatedCost / permutation.Length);
}