private protected override Delegate[] CreateGettersCore(Row input, Func <int, bool> activeCols, out Action disposer)
{
disposer = null;
var getters = new Delegate[3];
if (!activeCols(ClusterIdCol) && !activeCols(SortedClusterCol) && !activeCols(SortedClusterScoreCol))
{
return(getters);
}
long cachedPosition = -1;
VBuffer <Single> scores = default(VBuffer <Single>);
var scoresArr = new Single[_numClusters];
int[] sortedIndices = new int[_numClusters];
var scoreGetter = input.GetGetter <VBuffer <Single> >(ScoreIndex);
Action updateCacheIfNeeded =
() =>
{
if (cachedPosition != input.Position)
{
scoreGetter(ref scores);
scores.CopyTo(scoresArr);
int j = 0;
foreach (var index in Enumerable.Range(0, scoresArr.Length).OrderBy(i => scoresArr[i]))
{
sortedIndices[j++] = index;
}
cachedPosition = input.Position;
}
};
if (activeCols(ClusterIdCol))
{
ValueGetter <uint> assignedFn =
(ref uint dst) =>
{
updateCacheIfNeeded();
dst = (uint)sortedIndices[0] + 1;
};
getters[ClusterIdCol] = assignedFn;
}
if (activeCols(SortedClusterScoreCol))
{
ValueGetter <VBuffer <Single> > topKScoresFn =
(ref VBuffer <Single> dst) =>
{
updateCacheIfNeeded();
var editor = VBufferEditor.Create(ref dst, _numClusters);
for (int i = 0; i < _numClusters; i++)
{
editor.Values[i] = scores.GetItemOrDefault(sortedIndices[i]);
}
dst = editor.Commit();
};
getters[SortedClusterScoreCol] = topKScoresFn;
}
if (activeCols(SortedClusterCol))
{
ValueGetter <VBuffer <uint> > topKClassesFn =
(ref VBuffer <uint> dst) =>
{
updateCacheIfNeeded();
var editor = VBufferEditor.Create(ref dst, _numClusters);
for (int i = 0; i < _numClusters; i++)
{
editor.Values[i] = (uint)sortedIndices[i] + 1;
}
dst = editor.Commit();
};
getters[SortedClusterCol] = topKClassesFn;
}
return(getters);
}
private ValueGetter <VBuffer <ushort> > MakeGetterVec(DataViewRow input, int iinfo)
{
Host.AssertValue(input);
int cv = input.Schema[ColMapNewToOld[iinfo]].Type.GetVectorSize();
Contracts.Assert(cv >= 0);
var getSrc = input.GetGetter <VBuffer <ReadOnlyMemory <char> > >(input.Schema[ColMapNewToOld[iinfo]]);
var src = default(VBuffer <ReadOnlyMemory <char> >);
ValueGetter <VBuffer <ushort> > getterWithStartEndSep = (ref VBuffer <ushort> dst) =>
{
getSrc(ref src);
int len = 0;
var srcValues = src.GetValues();
for (int i = 0; i < srcValues.Length; i++)
{
if (!srcValues[i].IsEmpty)
{
len += srcValues[i].Length;
if (_parent._useMarkerChars)
{
len += TextMarkersCount;
}
}
}
var editor = VBufferEditor.Create(ref dst, len);
if (len > 0)
{
int index = 0;
for (int i = 0; i < srcValues.Length; i++)
{
if (srcValues[i].IsEmpty)
{
continue;
}
if (_parent._useMarkerChars)
{
editor.Values[index++] = TextStartMarker;
}
var span = srcValues[i].Span;
for (int ich = 0; ich < srcValues[i].Length; ich++)
{
editor.Values[index++] = span[ich];
}
if (_parent._useMarkerChars)
{
editor.Values[index++] = TextEndMarker;
}
}
Contracts.Assert(index == len);
}
dst = editor.Commit();
};
ValueGetter <VBuffer <ushort> > getterWithUnitSep = (ref VBuffer <ushort> dst) =>
{
getSrc(ref src);
int len = 0;
var srcValues = src.GetValues();
for (int i = 0; i < srcValues.Length; i++)
{
if (!srcValues[i].IsEmpty)
{
len += srcValues[i].Length;
if (i > 0)
{
len += 1; // add UnitSeparator character to len that will be added
}
}
}
if (_parent._useMarkerChars)
{
len += TextMarkersCount;
}
var editor = VBufferEditor.Create(ref dst, len);
if (len > 0)
{
int index = 0;
// ReadOnlyMemory can be a result of either concatenating text columns together
// or application of word tokenizer before char tokenizer in TextFeaturizingEstimator.
//
// Considering VBuffer<ReadOnlyMemory> as a single text stream.
// Therefore, prepend and append start and end markers only once i.e. at the start and at end of vector.
// Insert UnitSeparator after every piece of text in the vector.
if (_parent._useMarkerChars)
{
editor.Values[index++] = TextStartMarker;
}
for (int i = 0; i < srcValues.Length; i++)
{
if (srcValues[i].IsEmpty)
{
continue;
}
if (i > 0)
{
editor.Values[index++] = UnitSeparator;
}
var span = srcValues[i].Span;
for (int ich = 0; ich < srcValues[i].Length; ich++)
{
editor.Values[index++] = span[ich];
}
}
if (_parent._useMarkerChars)
{
editor.Values[index++] = TextEndMarker;
}
Contracts.Assert(index == len);
}
dst = editor.Commit();
};
return(_parent._isSeparatorStartEnd ? getterWithStartEndSep : getterWithUnitSep);
}
/// <summary>
/// Drops slots from src and populates the dst with the resulting vector. Slots are
/// dropped based on min and max slots that were passed at the constructor.
/// </summary>
public void DropSlots <TDst>(ref VBuffer <TDst> src, ref VBuffer <TDst> dst)
{
if (src.Length <= SlotsMin[0])
{
// There is nothing to drop, just swap buffers.
Utils.Swap(ref src, ref dst);
return;
}
int newLength = DstLength == 0 ? ComputeLength(src.Length) : DstLength;
if (newLength == 0)
{
// All slots dropped.
VBufferUtils.Resize(ref dst, 1, 0);
return;
}
Contracts.Assert(newLength < src.Length);
// End of the trivial cases
// At this point, we need to drop some slots and keep some slots.
VBufferEditor <TDst> editor;
var srcValues = src.GetValues();
if (src.IsDense)
{
editor = VBufferEditor.Create(ref dst, newLength);
int iDst = 0;
int iSrc = 0;
for (int i = 0; i < SlotsMax.Length && iSrc < src.Length; i++)
{
var lim = Math.Min(SlotsMin[i], src.Length);
while (iSrc < lim)
{
Contracts.Assert(iDst <= iSrc);
editor.Values[iDst++] = srcValues[iSrc++];
}
iSrc = SlotsMax[i] + 1;
}
while (iSrc < src.Length)
{
Contracts.Assert(iDst <= iSrc);
editor.Values[iDst++] = srcValues[iSrc++];
}
Contracts.Assert(iDst == newLength);
dst = editor.Commit();
return;
}
// Sparse case.
// Approximate new count is min(#indices, newLength).
var newCount = Math.Min(srcValues.Length, newLength);
var indices = dst.GetIndices();
var srcIndices = src.GetIndices();
Contracts.Assert(newCount <= src.Length);
editor = VBufferEditor.Create(
ref dst,
newLength,
newCount,
requireIndicesOnDense: true);
int iiDst = 0;
int iiSrc = 0;
int iOffset = 0;
int iRange = 0;
int min = SlotsMin[iRange];
// REVIEW: Consider using a BitArray with the slots to keep instead of SlotsMax. It would
// only make sense when the number of ranges is greater than the number of slots divided by 32.
int max = SlotsMax[iRange];
while (iiSrc < srcValues.Length)
{
// Copy (with offset) the elements before the current range.
var index = srcIndices[iiSrc];
if (index < min)
{
Contracts.Assert(iiDst <= iiSrc);
editor.Indices[iiDst] = index - iOffset;
editor.Values[iiDst++] = srcValues[iiSrc++];
continue;
}
if (index <= max)
{
// Skip elements in the current range.
iiSrc++;
continue;
}
// Find the next range.
const int threshold1 = 20;
const int threshold2 = 10;
while (++iRange < SlotsMax.Length && SlotsMax[iRange] < index)
{
if (SlotsMax.Length - iRange >= threshold1 &&
SlotsMax[iRange + threshold2] < index)
{
iRange = SlotsMax.FindIndexSorted(iRange + threshold2, SlotsMax.Length, index);
Contracts.Assert(iRange == SlotsMax.Length ||
iRange > 0 && SlotsMax[iRange - 1] < index && index <= SlotsMax[iRange]);
break;
}
}
if (iRange < SlotsMax.Length)
{
min = SlotsMin[iRange];
max = SlotsMax[iRange];
}
else
{
min = max = src.Length;
}
if (iRange > 0)
{
iOffset = _lengthReduction[iRange - 1];
}
Contracts.Assert(index <= max);
}
dst = editor.CommitTruncated(iiDst);
}
private protected override sealed void TransformCore(ref TInput input, FixedSizeQueue <TInput> windowedBuffer, long iteration, ref VBuffer <Double> dst)
{
var outputLength = Parent.OutputLength;
Host.Assert(outputLength >= 2);
var result = VBufferEditor.Create(ref dst, outputLength);
float rawScore = 0;
for (int i = 0; i < outputLength; ++i)
{
result.Values[i] = Double.NaN;
}
// Step 1: Computing the raw anomaly score
result.Values[1] = ComputeRawAnomalyScore(ref input, windowedBuffer, iteration);
if (Double.IsNaN(result.Values[1]))
{
result.Values[0] = 0;
}
else
{
if (WindowSize > 0)
{
// Step 2: Computing the p-value score
rawScore = (float)result.Values[1];
if (Parent.ThresholdScore == AlertingScore.RawScore)
{
switch (Parent.Side)
{
case AnomalySide.Negative:
rawScore = (float)(-result.Values[1]);
break;
case AnomalySide.Positive:
break;
default:
rawScore = (float)Math.Abs(result.Values[1]);
break;
}
}
else
{
result.Values[2] = ComputeKernelPValue(rawScore);
switch (Parent.Side)
{
case AnomalySide.Negative:
result.Values[2] = 1 - result.Values[2];
break;
case AnomalySide.Positive:
break;
default:
result.Values[2] = Math.Min(result.Values[2], 1 - result.Values[2]);
break;
}
// Keeping the p-value in the safe range
if (result.Values[2] < SequentialAnomalyDetectionTransformBase <TInput, TState> .MinPValue)
{
result.Values[2] = SequentialAnomalyDetectionTransformBase <TInput, TState> .MinPValue;
}
else if (result.Values[2] > SequentialAnomalyDetectionTransformBase <TInput, TState> .MaxPValue)
{
result.Values[2] = SequentialAnomalyDetectionTransformBase <TInput, TState> .MaxPValue;
}
RawScoreBuffer.AddLast(rawScore);
// Step 3: Computing the martingale value
if (Parent.Martingale != MartingaleType.None && Parent.ThresholdScore == AlertingScore.MartingaleScore)
{
Double martingaleUpdate = 0;
switch (Parent.Martingale)
{
case MartingaleType.Power:
martingaleUpdate = Parent.LogPowerMartigaleBettingFunc(result.Values[2], Parent.PowerMartingaleEpsilon);
break;
case MartingaleType.Mixture:
martingaleUpdate = Parent.LogMixtureMartigaleBettingFunc(result.Values[2]);
break;
}
if (LogMartingaleUpdateBuffer.Count == 0)
{
for (int i = 0; i < LogMartingaleUpdateBuffer.Capacity; ++i)
{
LogMartingaleUpdateBuffer.AddLast(martingaleUpdate);
}
_logMartingaleValue += LogMartingaleUpdateBuffer.Capacity * martingaleUpdate;
}
else
{
_logMartingaleValue += martingaleUpdate;
_logMartingaleValue -= LogMartingaleUpdateBuffer.PeekFirst();
LogMartingaleUpdateBuffer.AddLast(martingaleUpdate);
}
result.Values[3] = Math.Exp(_logMartingaleValue);
}
}
}
// Generating alert
bool alert = false;
if (RawScoreBuffer.IsFull) // No alert until the buffer is completely full.
{
switch (Parent.ThresholdScore)
{
case AlertingScore.RawScore:
alert = rawScore >= Parent.AlertThreshold;
break;
case AlertingScore.PValueScore:
alert = result.Values[2] <= Parent.AlertThreshold;
break;
case AlertingScore.MartingaleScore:
alert = (Parent.Martingale != MartingaleType.None) && (result.Values[3] >= Parent.AlertThreshold);
if (alert)
{
if (_martingaleAlertCounter > 0)
{
alert = false;
}
else
{
_martingaleAlertCounter = Parent.WindowSize;
}
}
_martingaleAlertCounter--;
_martingaleAlertCounter = _martingaleAlertCounter < 0 ? 0 : _martingaleAlertCounter;
break;
}
}
result.Values[0] = Convert.ToDouble(alert);
}
dst = result.Commit();
}
internal static DataViewSchema GetModelSchema(IExceptionContext ectx, Graph graph, string opType = null)
{
var schemaBuilder = new DataViewSchema.Builder();
foreach (Operation op in graph)
{
if (opType != null && opType != op.OpType)
{
continue;
}
var tfType = op.OutputType(0);
// Determine element type in Tensorflow tensor. For example, a vector of floats may get NumberType.R4 here.
var mlType = Tf2MlNetTypeOrNull(tfType);
// If the type is not supported in ML.NET then we cannot represent it as a column in an Schema.
// We also cannot output it with a TensorFlowTransform, so we skip it.
// Furthermore, operators which have NumOutputs <= 0 needs to be filtered.
// The 'GetTensorShape' method crashes TensorFlow runtime
// (https://github.com/dotnet/machinelearning/issues/2156) when the operator has no outputs.
if (mlType == null || op.NumOutputs <= 0)
{
continue;
}
// Construct the final ML.NET type of a Tensorflow variable.
var tensorShape = op.output.TensorShape.dims;
var columnType = new VectorDataViewType(mlType);
if (!(Utils.Size(tensorShape) == 1 && tensorShape[0] <= 0) &&
(Utils.Size(tensorShape) > 0 && tensorShape.Skip(1).All(x => x > 0)))
{
columnType = new VectorDataViewType(mlType, tensorShape[0] > 0 ? tensorShape : tensorShape.Skip(1).ToArray());
}
// There can be at most two metadata fields.
// 1. The first field always presents. Its value is this operator's type. For example,
// if an output is produced by an "Softmax" operator, the value of this field should be "Softmax".
// 2. The second field stores operators whose outputs are consumed by this operator. In other words,
// these values are names of some upstream operators which should be evaluated before executing
// the current operator. It's possible that one operator doesn't need any input, so this field
// can be missing.
var metadataBuilder = new DataViewSchema.Annotations.Builder();
// Create the first metadata field.
metadataBuilder.Add(TensorflowOperatorTypeKind, TextDataViewType.Instance, (ref ReadOnlyMemory <char> value) => value = op.OpType.AsMemory());
if (op.NumInputs > 0)
{
// Put upstream operators' names to an array (type: VBuffer) of string (type: ReadOnlyMemory<char>).
VBuffer <ReadOnlyMemory <char> > upstreamOperatorNames = default;
var bufferEditor = VBufferEditor.Create(ref upstreamOperatorNames, op.NumInputs);
for (int i = 0; i < op.NumInputs; ++i)
{
bufferEditor.Values[i] = op.inputs[i].op.name.AsMemory();
}
upstreamOperatorNames = bufferEditor.Commit(); // Used in metadata's getter.
// Create the second metadata field.
metadataBuilder.Add(TensorflowUpstreamOperatorsKind, new VectorDataViewType(TextDataViewType.Instance, op.NumInputs),
(ref VBuffer <ReadOnlyMemory <char> > value) => { upstreamOperatorNames.CopyTo(ref value); });
}
schemaBuilder.AddColumn(op.name, columnType, metadataBuilder.ToAnnotations());
}
return(schemaBuilder.ToSchema());
}
/// <summary>
/// Getter generator for inputs of type <typeparamref name="TSrc"/>, where output type is a vector of hashes
/// </summary>
/// <typeparam name="TSrc">Input type. Must be a vector</typeparam>
/// <param name="input">Row input</param>
/// <param name="iinfo">Index of the getter</param>
private ValueGetter <VBuffer <uint> > ComposeGetterVecToVec <TSrc>(Row input, int iinfo)
{
Host.AssertValue(input);
Host.Assert(Infos[iinfo].TypeSrc.IsVector);
var getSrc = GetSrcGetter <VBuffer <TSrc> >(input, iinfo);
var hashFunction = ComposeHashDelegate <TSrc>();
var src = default(VBuffer <TSrc>);
int n = _exes[iinfo].OutputValueCount;
int expectedSrcLength = Infos[iinfo].TypeSrc.VectorSize;
int[][] slotMap = _exes[iinfo].SlotMap;
// REVIEW: consider adding a fix-zero functionality (subtract emptyTextHash from all hashes)
var mask = (1U << _exes[iinfo].HashBits) - 1;
var hashSeed = _exes[iinfo].HashSeed;
bool ordered = _exes[iinfo].Ordered;
TSrc[] denseValues = null;
return
((ref VBuffer <uint> dst) =>
{
getSrc(ref src);
Host.Check(src.Length == expectedSrcLength);
ReadOnlySpan <TSrc> values;
// force-densify the input
// REVIEW: this performs poorly if only a fraction of sparse vector is used for hashing.
// This scenario was unlikely at the time of writing. Regardless of performance, the hash value
// needs to be consistent across equivalent representations - sparse vs dense.
if (src.IsDense)
{
values = src.GetValues();
}
else
{
if (denseValues == null)
{
denseValues = new TSrc[expectedSrcLength];
}
src.CopyTo(denseValues);
values = denseValues;
}
var hashes = VBufferEditor.Create(ref dst, n);
for (int i = 0; i < n; i++)
{
uint hash = hashSeed;
foreach (var srcSlot in slotMap[i])
{
// REVIEW: some legacy code hashes 0 for srcSlot in ord- case, do we need to preserve this behavior?
if (ordered)
{
hash = Hashing.MurmurRound(hash, (uint)srcSlot);
}
hash = hashFunction(in values[srcSlot], hash);
}
hashes.Values[i] = (Hashing.MixHash(hash) & mask) + 1; // +1 to offset from zero, which has special meaning for KeyType
}
dst = hashes.Commit();
});
}
// This converts in place.
private static void FillValues(IExceptionContext ectx, ref VBuffer <Float> buffer)
{
int size = buffer.Length;
ectx.Check(0 <= size & size < int.MaxValue / 2);
var values = buffer.GetValues();
var editor = VBufferEditor.Create(ref buffer, size * 2, values.Length);
int iivDst = 0;
if (buffer.IsDense)
{
// Currently, it's dense. We always produce sparse.
for (int ivSrc = 0; ivSrc < values.Length; ivSrc++)
{
ectx.Assert(iivDst <= ivSrc);
var val = values[ivSrc];
if (val == 0)
{
continue;
}
if (Float.IsNaN(val))
{
editor.Values[iivDst] = 1;
editor.Indices[iivDst] = 2 * ivSrc + 1;
}
else
{
editor.Values[iivDst] = val;
editor.Indices[iivDst] = 2 * ivSrc;
}
iivDst++;
}
}
else
{
// Currently, it's sparse.
var indices = buffer.GetIndices();
int ivPrev = -1;
for (int iivSrc = 0; iivSrc < values.Length; iivSrc++)
{
ectx.Assert(iivDst <= iivSrc);
var val = values[iivSrc];
if (val == 0)
{
continue;
}
int iv = indices[iivSrc];
ectx.Assert(ivPrev < iv & iv < size);
ivPrev = iv;
if (Float.IsNaN(val))
{
editor.Values[iivDst] = 1;
editor.Indices[iivDst] = 2 * iv + 1;
}
else
{
editor.Values[iivDst] = val;
editor.Indices[iivDst] = 2 * iv;
}
iivDst++;
}
}
ectx.Assert(0 <= iivDst & iivDst <= values.Length);
buffer = editor.CommitTruncated(iivDst);
}
private void GetSlotNames(int iinfo, ref VBuffer <ReadOnlyMemory <char> > dst)
{
Host.Assert(0 <= iinfo && iinfo < Infos.Length);
int size = _types[iinfo].VectorSize;
if (size == 0)
{
throw MetadataUtils.ExceptGetMetadata();
}
var editor = VBufferEditor.Create(ref dst, size);
var type = Infos[iinfo].TypeSrc;
if (!type.IsVector)
{
Host.Assert(_types[iinfo].VectorSize == 2);
var columnName = Source.Schema.GetColumnName(Infos[iinfo].Source);
editor.Values[0] = columnName.AsMemory();
editor.Values[1] = (columnName + IndicatorSuffix).AsMemory();
}
else
{
Host.Assert(type.IsKnownSizeVector);
Host.Assert(size == 2 * type.VectorSize);
// REVIEW: Do we need to verify that there is metadata or should we just call GetMetadata?
var typeNames = Source.Schema.GetMetadataTypeOrNull(MetadataUtils.Kinds.SlotNames, Infos[iinfo].Source);
if (typeNames == null || typeNames.VectorSize != type.VectorSize || !typeNames.ItemType.IsText)
{
throw MetadataUtils.ExceptGetMetadata();
}
var names = default(VBuffer <ReadOnlyMemory <char> >);
Source.Schema.GetMetadata(MetadataUtils.Kinds.SlotNames, Infos[iinfo].Source, ref names);
// We both assert and check. If this fails, there is a bug somewhere (possibly in this code
// but more likely in the implementation of Base. On the other hand, we don't want to proceed
// if we've received garbage.
Host.Check(names.Length == type.VectorSize, "Unexpected slot name vector size");
var sb = new StringBuilder();
int slot = 0;
foreach (var kvp in names.Items(all: true))
{
Host.Assert(0 <= slot && slot < size);
Host.Assert(slot % 2 == 0);
sb.Clear();
if (kvp.Value.IsEmpty)
{
sb.Append('[').Append(slot / 2).Append(']');
}
else
{
sb.AppendMemory(kvp.Value);
}
int len = sb.Length;
sb.Append(IndicatorSuffix);
var str = sb.ToString();
editor.Values[slot++] = str.AsMemory().Slice(0, len);
editor.Values[slot++] = str.AsMemory();
}
Host.Assert(slot == size);
}
dst = editor.Commit();
}
