public void Initialize(Feature feature, IEnumerable <Tuple <Instance, double> > instances)
{
_instances = instances;
if (Model == null)
{
throw new InvalidOperationException("Model is null");
}
if (ClassFeature.FeatureType != FeatureType.Nominal)
{
throw new InvalidOperationException("Cannot use this iterator on non-nominal class");
}
NominalFeature classFeature = (NominalFeature)ClassFeature;
if (feature.FeatureType != FeatureType.Nominal)
{
throw new InvalidOperationException("Cannot use this iterator on non-nominal feature");
}
_numClasses = classFeature.Values.Length;
_feature = feature;
_perValueDistribution = new Dictionary <double, double[]>();
_totalDistribution = new double[_numClasses];
foreach (var instance in _instances)
{
double value = instance.Item1[feature];
if (FeatureValue.IsMissing(value))
{
continue;
}
double[] current;
if (!_perValueDistribution.TryGetValue(value, out current))
{
_perValueDistribution.Add(value, current = new double[_numClasses]);
}
int classIdx = (int)instance.Item1[ClassFeature];
current[classIdx] += instance.Item2;
_totalDistribution[classIdx] += instance.Item2;
}
CurrentDistribution = new double[2][];
_valuesCount = _perValueDistribution.Count;
existingValues = _perValueDistribution.Keys.ToArray();
_iteratingTwoValues = (_valuesCount == 2);
_valueIndex = -1;
_twoValuesIterated = false;
_initialized = true;
}
public static double[] CalculateSupports(double[] data, Feature classFeature)
{
NominalFeature feat = (NominalFeature)classFeature;
var featureInformation = (feat.FeatureInformation as NominalFeatureInformation);
double[] result = (double[])data.Clone();
for (int i = 0; i < result.Length; i++)
{
result[i] = featureInformation.Distribution[i] != 0
? result[i] / featureInformation.Distribution[i]
: 0;
}
return(result);
}
private static void CreateNomminalFeature(IEnumerable <string> values, string[] headers, int index, InstanceModel model, List <string[]> matrix,
Instance[] result)
{
string[] casted = values.Where(v => !string.IsNullOrWhiteSpace(v)).Distinct().ToArray();
var newModel = new NominalFeature(TextToId(headers[index]), index)
{
Values = casted,
};
model.Features[index] = newModel;
for (int i = 0; i < matrix.Count; i++)
{
string asStr = matrix[i][index];
if (string.IsNullOrWhiteSpace(asStr))
{
asStr = null;
}
result[i].SetNominalValue(newModel, asStr);
}
}
public List <List <Instance> > FindClusters(InstanceModel model, List <Instance> instances, out List <IEmergingPattern> selectedPatterns)
{
NominalFeature classFeature = null;
FeatureInformation backupFeatureInformation = null;
string[] backupClassValues = null;
double[] backupClassByInstance = null;
bool isClassPresent = true;
if (model.ClassFeature() == null)
{
isClassPresent = false;
classFeature = new NominalFeature("class", model.Features.Length);
var backupFeatures = model.Features;
model.Features = new Feature[backupFeatures.Length + 1];
for (int i = 0; i < backupFeatures.Length; i++)
{
model.Features[i] = backupFeatures[i];
}
model.Features[backupFeatures.Length] = classFeature;
}
else
{
classFeature = model.ClassFeature() as NominalFeature;
backupFeatureInformation = classFeature.FeatureInformation;
backupClassValues = classFeature.Values;
backupClassByInstance = new double[instances.Count];
for (int i = 0; i < instances.Count; i++)
{
backupClassByInstance[i] = instances[i][classFeature];
instances[i][classFeature] = 0;
}
}
classFeature.FeatureInformation = new NominalFeatureInformation()
{
Distribution = new double[] { 1, 1, 1, 1, 1 },
Ratio = new double[] { 1, 1, 1, 1, 1 },
ValueProbability = new double[] { 1, 1, 1, 1, 1 }
};
classFeature.Values = new string[1] {
"Unknown"
};
var Miner = new UnsupervisedRandomForestMiner()
{
ClusterCount = ClusterCount, TreeCount = 100
};
var patterns = Miner.Mine(model, instances, classFeature);
var instIdx = new Dictionary <Instance, int>();
for (int i = 0; i < instances.Count; i++)
{
instIdx.Add(instances[i], i);
}
int[,] similarityMatrix = new int[instances.Count, instances.Count + 1];
var coverSetByPattern = new Dictionary <IEmergingPattern, HashSet <Instance> >();
foreach (var pattern in patterns)
{
if (pattern != null)
{
var currentCluster = new List <int>();
var currentCoverSet = new HashSet <Instance>();
for (int i = 0; i < instances.Count; i++)
{
if (pattern.IsMatch(instances[i]))
{
currentCluster.Add(i);
currentCoverSet.Add(instances[i]);
}
}
for (int i = 0; i < currentCluster.Count; i++)
{
for (int j = 0; j < currentCluster.Count; j++)
{
similarityMatrix[currentCluster[i], currentCluster[j]] += 1;
similarityMatrix[currentCluster[i], instances.Count] += 1;
}
}
coverSetByPattern.Add(pattern, currentCoverSet);
}
}
var kmeans = new KMeans()
{
K = ClusterCount, classFeature = classFeature, similarityMatrix = similarityMatrix, instIdx = instIdx
};
var clusterList = kmeans.FindClusters(instances);
var patternClusterList = new List <List <IEmergingPattern> >();
for (int i = 0; i < ClusterCount; i++)
{
patternClusterList.Add(new List <IEmergingPattern>());
}
foreach (var pattern in patterns)
{
if (pattern != null)
{
var bestIdx = 0;
var maxCoverCount = int.MinValue;
pattern.Supports = new double[ClusterCount];
pattern.Counts = new double[ClusterCount];
HashSet <Instance> bestCover = null;
for (int i = 0; i < ClusterCount; i++)
{
HashSet <Instance> currentCover = new HashSet <Instance>(coverSetByPattern[pattern].Intersect(clusterList[i]));
var currentCoverCount = currentCover.Count;
pattern.Counts[i] = currentCoverCount;
pattern.Supports[i] = 1.0 * currentCoverCount / clusterList[i].Count;
if (currentCoverCount > maxCoverCount)
{
maxCoverCount = currentCoverCount;
bestIdx = i;
bestCover = currentCover;
}
}
coverSetByPattern[pattern] = bestCover;
patternClusterList[bestIdx].Add(pattern);
}
}
selectedPatterns = FilterPatterns(instances, patternClusterList);
if (isClassPresent)
{
classFeature.FeatureInformation = backupFeatureInformation;
classFeature.Values = backupClassValues;
for (int i = 0; i < instances.Count; i++)
{
instances[i][classFeature] = backupClassByInstance[i];
}
}
return(clusterList);
}
