CalculateMatchingScoresPerLabelPerPunchedCard(
IDictionary <string, IDictionary <IBitVector, IReadOnlyCollection <IBitVector> > >
punchedCardsCollection,
IBitVector input,
IPuncher <string, IBitVector, IBitVector> puncher,
IDictionary <int, IAdjacencyMatrix> adjacencyMatrices)
{
var matchingScoresPerLabelPerPunchedCard = new Dictionary <IBitVector, IDictionary <string, double> >();
foreach (var punchedCardsCollectionItem in punchedCardsCollection)
{
var punchedInput = puncher.Punch(punchedCardsCollectionItem.Key, input).Input;
foreach (var label in punchedCardsCollectionItem.Value)
{
ProcessTheSpecificLabel(
matchingScoresPerLabelPerPunchedCard,
punchedCardsCollectionItem.Key,
label.Key,
label.Value.Count,
adjacencyMatrices[GetAdjacencyMatrixKey(punchedCardsCollectionItem.Key, label.Key)],
punchedInput);
}
}
return(matchingScoresPerLabelPerPunchedCard);
}
private double[] CalculateLossesPerLabel(IBitVector bitVector)
{
var lossesPerLabel = _labels.Select(currentLabel => CalculateLossPerLabel(bitVector, currentLabel)).ToArray();
//Softmax(lossesPerLabel);
return(lossesPerLabel);
}
CalculateLossPerLabelPerPunchedCard(
IReadOnlyDictionary <string, IReadOnlyDictionary <IBitVector, IReadOnlyCollection <IBitVector> > > punchedCardsCollection,
IBitVector input,
IPuncher <string, IBitVector, IBitVector> puncher,
IReadOnlyDictionary <string, IExpert> experts)
{
var lossPerLabelPerPunchedCard = new Dictionary <IBitVector, IReadOnlyDictionary <string, double> >();
foreach (var punchedCardsCollectionItem in punchedCardsCollection)
{
var expert = experts[punchedCardsCollectionItem.Key];
var punchedInput = puncher.Punch(punchedCardsCollectionItem.Key, input).Input;
var losses = expert.CalculateLosses(punchedInput);
foreach (var label in punchedCardsCollectionItem.Value)
{
if (!lossPerLabelPerPunchedCard.TryGetValue(label.Key, out var dictionary))
{
dictionary = new Dictionary <string, double>();
lossPerLabelPerPunchedCard.Add(label.Key, dictionary);
}
((Dictionary <string, double>)dictionary).Add(punchedCardsCollectionItem.Key, losses[label.Key]);
}
}
return(lossPerLabelPerPunchedCard);
}
public void BitwiseAnd(IBitVector u)
{
BitVectorAlloc v = (BitVectorAlloc)u;
Ensure.That(v).IsNotNull();
Ensure.That(Size).Is(v.Size);
if (!allocated && !v.allocated)
{
return;
}
if (allocated && !v.allocated)
{
Deallocate();
return;
}
if (!allocated && v.allocated)
{
return;
}
bool isEmpty = true;
unsafe
{
fixed(UInt64 *items_ptr = items)
{
fixed(UInt64 *v_items_ptr = v.items)
{
UInt64 *items_last = items_ptr + items.Length;
UInt64 *v2items_last = v_items_ptr + v.items.Length;
UInt64 *ptr = items_ptr;
UInt64 *v_ptr = v_items_ptr;
for (; ptr < items_last;)
{
(*ptr) &= (*v_ptr);
if ((*ptr) != 0)
{
isEmpty = false;
}
ptr++;
v_ptr++;
}
}
}
}
if (isEmpty)
{
Deallocate();
}
}
public int XorCardinality(IBitVector bitVector)
{
if (Count != bitVector.Count)
{
throw new Exception("Counts does not match!");
}
return((int)_roaringBitmap.XorCardinality(((BitVectorRoaringBitmap)bitVector)._roaringBitmap));
}
private static double CalculateMatchingScore(IBitVector firstBitVector, IBitVector secondBitVector)
{
if (firstBitVector.Count != secondBitVector.Count)
{
throw new Exception("Counts does not match!");
}
return(firstBitVector.AndCardinality(secondBitVector) - firstBitVector.XorCardinality(secondBitVector) / 10d);
}
public void BitwiseSubtract(IBitVector u)
{
BitVectorAsSet v = (BitVectorAsSet)u;
Ensure.That(v).IsNotNull();
foreach (ulong b in v.bits)
{
bits.Remove(b);
}
}
public void BitwiseOr(IBitVector u)
{
BitVectorAsSet v = (BitVectorAsSet)u;
Ensure.That(v).IsNotNull();
foreach (ulong b in v.bits)
{
bits.Add(b);
}
}
/// <summary>
/// Creates a new Bloom filter, using the optimal size for the underlying data structure based on the desired capacity and error rate, as well as the optimal number of hash functions.
/// </summary>
/// <param name="capacity">The anticipated number of items to be added to the filter. More than this number of items can be added, but the error rate will exceed what is expected.</param>
/// <param name="errorRate">The accepable false-positive rate (e.g., 0.01F = 1%)</param>
/// <param name="hashFunction">The function to hash the input values. Do not use GetHashCode(). If it is null, and T is string or int a hash function will be provided for you.</param>
/// <param name="vectorFactory">vectorFactory</param>
public BloomFilter(int capacity, float errorRate, Func <uint, IBitVector> vectorFactory, HashFunction hashFunction)
{
if (capacity < 1)
{
throw new ArgumentOutOfRangeException("capacity", capacity, "capacity must be > 0");
}
if (errorRate >= 1 || errorRate <= 0)
{
throw new ArgumentOutOfRangeException("errorRate", errorRate, string.Format("errorRate must be between 0 and 1, exclusive. Was {0}", errorRate));
}
uint m = BloomFilter.ComputeM(capacity, errorRate);
if (m < 1)
{
throw new OverflowException(string.Format("The provided capacity and errorRate values would result in an array of length > int.MaxValue. Please reduce either of these values. Capacity: {0}, Error rate: {1}", capacity, errorRate));
}
if (vectorFactory == null)
{
_bitVector = new BitVector(m);
}
else
{
_bitVector = vectorFactory(m);
}
if (hashFunction == null)
{
if (typeof(T) == typeof(String))
{
_getHashSecondary = hashString;
}
else if (typeof(T) == typeof(int))
{
_getHashSecondary = hashInt32;
}
else
{
throw new ArgumentNullException("hashFunction", "Please provide a hash function for your type T, when T is not a string or int.");
}
}
else
{
_getHashSecondary = hashFunction;
}
int k = BloomFilter.ComputeK(capacity, errorRate);
if (k < 1)
{
throw new OverflowException(string.Format("The provided capacity and errorRate values would result in an array of length > int.MaxValue. Please reduce either of these values. Capacity: {0}, Error rate: {1}", capacity, errorRate));
}
_hashFunctionCount = k;
}
public IBitVector BitwiseSubtract(IBitVector x, IBitVector y)
{
BitVectorAsSet v1 = (BitVectorAsSet)x;
BitVectorAsSet v2 = (BitVectorAsSet)y;
Ensure.That(v1).IsNotNull();
Ensure.That(v2).IsNotNull();
SortedSet <ulong> zBits = new(v1.bits.Except(v2.bits));
return(new BitVectorAsSet(zBits));
}
private double CalculateLossPerLabel(IBitVector bitVector, IBitVector label)
{
var maxSpanningTreeWeightLoss = 0;
foreach (var edge in _maxSpanningTreesEdges[label].SelectMany(edge => edge))
{
var edgeIndex = GetEdgeIndexByVertexValues(bitVector.IsActive(edge.Item1), bitVector.IsActive(edge.Item2));
if (edgeIndex != edge.Item3)
{
maxSpanningTreeWeightLoss += edge.Item4;
}
}
return((double)maxSpanningTreeWeightLoss / _maxSpanningTreesWeightSums[label]);
}
private static double CalculateMatchingScore(IBitVector punchedInput,
IEnumerable <Tuple <IBitVector, int> > labelPunchedInputs)
{
var matchingScoreSum = 0d;
var count = 0;
foreach (var labelPunchedInput in labelPunchedInputs)
{
matchingScoreSum += CalculateMatchingScore(punchedInput, labelPunchedInput.Item1) *
labelPunchedInput.Item2;
count += labelPunchedInput.Item2;
}
return(matchingScoreSum / count);
}
private static void ProcessTheSpecificLabel(
IDictionary <IBitVector, IDictionary <string, double> > matchingScoresPerLabelPerPunchedCard,
string punchedCardKey,
KeyValuePair <IBitVector, IReadOnlyCollection <Tuple <IBitVector, int> > > label,
IBitVector punchedInput)
{
var matchingScorePerLabel = CalculateMatchingScore(punchedInput, label.Value);
if (!matchingScoresPerLabelPerPunchedCard.TryGetValue(label.Key, out var dictionary))
{
dictionary = new Dictionary <string, double>();
matchingScoresPerLabelPerPunchedCard[label.Key] = dictionary;
}
dictionary.Add(punchedCardKey, matchingScorePerLabel);
}
private static void ProcessTheSpecificLabel(
IDictionary <IBitVector, IDictionary <string, double> > matchingScoresPerLabelPerPunchedCard,
string punchedCardKey,
IBitVector key,
int samplesCount,
IAdjacencyMatrix adjacencyMatrix,
IBitVector punchedInput)
{
var matchingScorePerLabel = CalculateMatchingScore(punchedInput, adjacencyMatrix, samplesCount);
if (!matchingScoresPerLabelPerPunchedCard.TryGetValue(key, out var dictionary))
{
dictionary = new Dictionary <string, double>();
matchingScoresPerLabelPerPunchedCard[key] = dictionary;
}
dictionary.Add(punchedCardKey, matchingScorePerLabel);
}
public IBitVector BitwiseSubtract(IBitVector x, IBitVector y)
{
BitVector v1 = (BitVector)x;
BitVector v2 = (BitVector)y;
Ensure.That(v1).IsNotNull();
Ensure.That(v2).IsNotNull();
Ensure.That(v1.Size).Is(v2.Size);
BitVector result = new(v1.Size);
for (ulong i = 0; i < (ulong)v1.items.Length; i++)
{
result.items[i] = v1.items[i] & (~v2.items[i]);
}
return(result);
}
CalculateMatchingScoresPerLabelPerPunchedCard(
IDictionary <string, IDictionary <IBitVector, IReadOnlyCollection <Tuple <IBitVector, int> > > >
punchedCardsCollection,
IBitVector input,
IPuncher <string, IBitVector, IBitVector> puncher)
{
var matchingScoresPerLabelPerPunchedCard = new Dictionary <IBitVector, IDictionary <string, double> >();
foreach (var punchedCardsCollectionItem in punchedCardsCollection)
{
var punchedInput = puncher.Punch(punchedCardsCollectionItem.Key, input).Input;
foreach (var label in punchedCardsCollectionItem.Value)
{
ProcessTheSpecificLabel(matchingScoresPerLabelPerPunchedCard, punchedCardsCollectionItem.Key,
label, punchedInput);
}
}
return(matchingScoresPerLabelPerPunchedCard);
}
public double CalculateLoss(IBitVector bitVector, IBitVector label)
{
return(CalculateLossPerLabel(bitVector, label));
}
public IBitVector CreateVector(IBitVector v)
{
return(new BitVector((BitVector)v));
}
public IReadOnlyDictionary <IBitVector, double> CalculateLosses(IBitVector bitVector)
{
var lossesPerLabel = CalculateLossesPerLabel(bitVector);
return(Enumerable.Range(0, _labels.Count).ToDictionary(index => _labels[index], index => lossesPerLabel[index]));
}
internal static double CalculateMaxLoss(KeyValuePair <string, IReadOnlyDictionary <IBitVector, IReadOnlyCollection <IBitVector> > > punchedCardPerLabel, IReadOnlyDictionary <string, IExpert> experts, IBitVector label)
{
var expert = experts[punchedCardPerLabel.Key];
return(punchedCardPerLabel.Value[label].Max(input => expert.CalculateLoss(input, label)));
}
public IBitVector CreateVector(IBitVector v)
{
return(new BitVectorAsSet((BitVectorAsSet)v));
}
private static int GetAdjacencyMatrixKey(string punchedCardsCollectionItemKey, IBitVector labelKey)
{
return(HashCode.Combine(punchedCardsCollectionItemKey, labelKey));
}
public void BitwiseSubtract(IBitVector u)
{
throw new InvalidOperationException();
}
public IBitVector CreateVector(IBitVector v)
{
return(new BitVectorAlloc((BitVectorAlloc)v));
}
public IBitVector BitwiseAnd(IBitVector x, IBitVector y)
{
BitVectorAlloc v1 = (BitVectorAlloc)x;
BitVectorAlloc v2 = (BitVectorAlloc)y;
Ensure.That(v1).IsNotNull();
Ensure.That(v2).IsNotNull();
Ensure.That(v1.Size).Is(v2.Size);
BitVectorAlloc result = new(v1.Size);
if (!v1.allocated && !v2.allocated)
{
return(result);
}
if (v1.allocated && !v2.allocated)
{
return(result);
}
if (!v1.allocated && v2.allocated)
{
return(result);
}
result.Allocate();
unsafe
{
fixed(UInt64 *r_items_ptr = result.items)
{
fixed(UInt64 *v1_items_ptr = v1.items)
{
fixed(UInt64 *v2_items_ptr = v2.items)
{
UInt64 *r_items_last = r_items_ptr + result.items.Length;
UInt64 *v1_items_last = v1_items_ptr + v1.items.Length;
UInt64 *v2_items_last = v2_items_ptr + v2.items.Length;
UInt64 *r_ptr = r_items_ptr;
UInt64 *v1_ptr = v1_items_ptr;
UInt64 *v2_ptr = v2_items_ptr;
for (; v1_ptr < v1_items_last;)
{
(*r_ptr) = (*v1_ptr) & (*v2_ptr);
r_ptr++;
v1_ptr++;
v2_ptr++;
}
}
}
}
}
if (result.IsEmpty())
{
return(new BitVectorAlloc(v1.Size));
}
return(result);
}
internal static double CalculateMatchingScore(IBitVector punchedInput, IAdjacencyMatrix adjacencyMatrix, int samplesCount)
{
return((double)CalculateAdjacencyMatrixScore(
adjacencyMatrix,
punchedInput.ActiveBitIndices) / samplesCount);
}
