public void Create(RevealedVectorSet trainingSet, GroupCache groupCache)
{
do
{
Initialize();
} while (!UpdateClassifierUnits(trainingSet.Accepted, trainingSet, groupCache));
}
public IDictionary <BigInteger, string> RevealSuspiciousElements(
ICollection <BigInteger> actuallyAcceptedVectors,
IList <SuspiciousNode> suspiciousNodes,
EncodingResult encodingResult, RevealedVectorSet trainingSet)
{
var additionalAcceptedVector2GroupPath = new Dictionary <BigInteger, string>();
foreach (var t in suspiciousNodes)
{
if (actuallyAcceptedVectors.Contains(t.Vector))
{
additionalAcceptedVector2GroupPath.Add(t.Vector, t.GroupKey);
trainingSet.Accepted.Add(t.Vector, t.GroupKey);
}
else
{
trainingSet.Rejected.Add(t.Vector, t.GroupKey);
}
}
Console.WriteLine(
"Accepted: " + trainingSet.AcceptedCount + " + "
+ additionalAcceptedVector2GroupPath.Count
+ " / " + encodingResult.IdealVectorSet.AcceptedCount);
Console.WriteLine(
"Rejected: " + trainingSet.RejectedCount + " + "
+ (suspiciousNodes.Count - additionalAcceptedVector2GroupPath.Count)
+ " / " + encodingResult.IdealVectorSet.RejectedCount);
return(additionalAcceptedVector2GroupPath);
}
public EncodingResult() {
_vector2Node = new Dictionary<BigInteger, CstNode>();
Vector2Count = new Dictionary<BigInteger, int>();
IdealVectorSet = new RevealedVectorSet();
SeedVectorSet = new RevealedVectorSet();
}
private UpdateResult UpdateAcceptingClassifierUnits(
IDictionary <BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet,
GroupCache groupCache)
{
if (additionalAcceptedSet.Count == 0)
{
var rejectedGroupInfo = FindWronglyAcceptedGroupInfo(trainingSet, groupCache);
if (rejectedGroupInfo != null)
{
return(new UpdateResult {
Rejected = rejectedGroupInfo,
Accepted = null,
});
}
return(null);
}
foreach (var vectorAndGroupPath in additionalAcceptedSet)
{
var vector = vectorAndGroupPath.Key;
var groupIndex = groupCache.GetGroupIndex(vector);
Units[groupIndex].Accepting &= vector;
var rejectedGroupInfo = FindWronglyAcceptedGroupInfo(trainingSet, groupCache);
if (rejectedGroupInfo != null)
{
return(new UpdateResult {
Rejected = rejectedGroupInfo,
Accepted = new GroupInfo {
Index = groupIndex, Path = vectorAndGroupPath.Value
},
});
}
}
return(null);
}
public EncodingResult()
{
_vector2Node = new Dictionary <BigInteger, CstNode>();
Vector2Count = new Dictionary <BigInteger, int>();
IdealVectorSet = new RevealedVectorSet();
SeedVectorSet = new RevealedVectorSet();
}
public bool Update(
IDictionary <BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet,
GroupCache groupCache)
{
var originalClassifierUnit = Units.Select(c => c.Clone()).ToImmutableList();
if (!UpdateClassifierUnits(additionalAcceptedSet, trainingSet, groupCache))
{
Create(trainingSet, groupCache);
}
return(!Units.SequenceEqual(originalClassifierUnit));
}
private GroupInfo FindWronglyAcceptedGroupInfo(
RevealedVectorSet trainingSet, GroupCache groupCache)
{
foreach (var featureAndGroupPath in trainingSet.Rejected)
{
var feature = featureAndGroupPath.Key;
var groupPath = featureAndGroupPath.Value;
var groupIndex = groupCache.GetGroupIndex(feature);
var accepted = IsAccepted(feature, groupIndex) && !IsRejected(feature, groupIndex);
if (accepted)
{
return(new GroupInfo {
Index = groupIndex, Path = groupPath
});
}
}
return(null);
}
private bool UpdateClassifierUnits(
IDictionary <BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet,
GroupCache groupCache)
{
UpdateRejectingClassifierUnits(additionalAcceptedSet.Keys, groupCache);
var result = UpdateAcceptingClassifierUnits(additionalAcceptedSet, trainingSet,
groupCache);
if (result == null)
{
return(true);
}
var count = GroupPaths.Count;
var updated = false;
if (result.Accepted != null &&
result.Accepted.Path != GroupPaths[result.Accepted.Index])
{
updated |= AddNewGroup(result.Accepted);
}
if (result.Rejected.Path != GroupPaths[result.Rejected.Index])
{
updated |= AddNewGroup(result.Rejected);
}
if (updated)
{
groupCache.UpdateGroupCache(this);
}
// ReSharper disable once PossibleUnintendedReferenceComparison
if (GroupPaths.Count == count &&
additionalAcceptedSet == trainingSet.Accepted)
{
throw new Exception("Fail to learn rules");
}
Console.WriteLine("Failed to create Classifier units.");
Console.WriteLine("Groups: " + GroupPaths.Count + " (" + count + ")");
return(false);
}
public void WriteResult(StreamWriter writer, RevealedVectorSet trainingSet = null)
{
if (writer == null)
{
return;
}
trainingSet = trainingSet ?? new RevealedVectorSet();
writer.Write(IdealAcceptedVector2GroupPath.Concat(IdealRejectedVector2GroupPath)
.Sum(f => Vector2Count[f.Key]));
writer.Write(",");
writer.Write(trainingSet.Accepted.Concat(trainingSet.Rejected)
.Sum(f => Vector2Count[f.Key]));
writer.Write(",");
writer.Write(IdealVectorSet.Count);
writer.Write(",");
writer.Write(trainingSet.Count);
writer.Write(",");
writer.Write(SeedNodeCount);
writer.Write(",");
writer.Write(SeedVectorCount);
writer.Write(",");
writer.Write(SeedAcceptedNodeCount);
writer.Write(",");
}
private bool UpdateClassifierUnits(
IDictionary<BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet,
GroupCache groupCache) {
UpdateRejectingClassifierUnits(additionalAcceptedSet.Keys, groupCache);
var result = UpdateAcceptingClassifierUnits(additionalAcceptedSet, trainingSet,
groupCache);
if (result == null) {
return true;
}
var count = GroupPaths.Count;
var updated = false;
if (result.Accepted != null
&& result.Accepted.Path != GroupPaths[result.Accepted.Index]) {
updated |= AddNewGroup(result.Accepted);
}
if (result.Rejected.Path != GroupPaths[result.Rejected.Index]) {
updated |= AddNewGroup(result.Rejected);
}
if (updated) {
groupCache.UpdateGroupCache(this);
}
// ReSharper disable once PossibleUnintendedReferenceComparison
if (GroupPaths.Count == count
&& additionalAcceptedSet == trainingSet.Accepted) {
throw new Exception("Fail to learn rules");
}
Console.WriteLine("Failed to create Classifier units.");
Console.WriteLine("Groups: " + GroupPaths.Count + " (" + count + ")");
return false;
}
public void Create(RevealedVectorSet trainingSet, GroupCache groupCache) {
do {
Initialize();
} while (!UpdateClassifierUnits(trainingSet.Accepted, trainingSet, groupCache));
}
public bool Update(
IDictionary<BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet,
GroupCache groupCache) {
var originalClassifierUnit = Units.Select(c => c.Clone()).ToImmutableList();
if (!UpdateClassifierUnits(additionalAcceptedSet, trainingSet, groupCache)) {
Create(trainingSet, groupCache);
}
return !Units.SequenceEqual(originalClassifierUnit);
}
public ClassificationResult Classify(
int count, Classifier classifier, GroupCache groupCache,
EncodingResult encodingResult,
RevealedVectorSet trainingSet = null)
{
trainingSet = trainingSet ?? new RevealedVectorSet();
var correctlyAccepted = 0;
var correctlyRejected = 0;
var wronglyAccepted = 0;
var wronglyRejected = 0;
var correctlyRejectedInRejecting = 0;
var wronglyRejectedInRejecting = 0;
var rejectAccept = new List <List <SuspiciousNode> >();
var rejectReject = new List <List <SuspiciousNode> >();
var acceptAccept = new List <List <SuspiciousNode> >();
var acceptReject = new List <List <SuspiciousNode> >();
for (int i = 0; i < classifier.GroupCount; i++)
{
rejectAccept.Add(new List <SuspiciousNode>());
rejectReject.Add(new List <SuspiciousNode>());
acceptAccept.Add(new List <SuspiciousNode>());
acceptReject.Add(new List <SuspiciousNode>());
}
var wrongNodeCount = 0;
var wronglyRejectedFeatures = new List <BigInteger>();
var wronglyAcceptedFeatures = new List <BigInteger>();
foreach (var vectorAndGroupPath in encodingResult.IdealVectorSet.Accepted)
{
var vector = vectorAndGroupPath.Key;
var groupPath = vectorAndGroupPath.Value;
var groupIndex = groupCache.GetGroupIndex(vector);
var rejected = classifier.IsRejected(vector, groupIndex);
var accepted = classifier.IsAccepted(vector, groupIndex);
if (!trainingSet.Accepted.ContainsKey(vector) &&
!trainingSet.Rejected.ContainsKey(vector))
{
var target = new SuspiciousNode {
Vector = vector,
GroupKey = groupPath,
Used = false,
};
if (accepted)
{
if (!rejected)
{
acceptAccept[groupIndex].Add(target);
}
else
{
acceptReject[groupIndex].Add(target);
}
}
else
{
if (!rejected)
{
rejectAccept[groupIndex].Add(target);
}
else
{
rejectReject[groupIndex].Add(target);
}
}
}
if (!accepted)
{
wronglyRejected++;
wrongNodeCount += encodingResult.Vector2Count[vector];
wronglyRejectedFeatures.Add(vector);
}
else if (!rejected)
{
correctlyAccepted++;
}
else
{
wronglyRejectedInRejecting++;
wrongNodeCount += encodingResult.Vector2Count[vector];
wronglyRejectedFeatures.Add(vector);
}
}
foreach (var vectorAndGroupPath in encodingResult.IdealVectorSet.Rejected)
{
var vector = vectorAndGroupPath.Key;
var groupPath = vectorAndGroupPath.Value;
var groupIndex = groupCache.GetGroupIndex(vector);
var rejected = classifier.IsRejected(vector, groupIndex);
var accepted = classifier.IsAccepted(vector, groupIndex);
if (!trainingSet.Accepted.ContainsKey(vector) &&
!trainingSet.Rejected.ContainsKey(vector))
{
var target = new SuspiciousNode {
Vector = vector,
GroupKey = groupPath,
Used = false,
};
if (accepted)
{
if (!rejected)
{
acceptAccept[groupIndex].Add(target);
}
else
{
acceptReject[groupIndex].Add(target);
}
}
else
{
if (!rejected)
{
rejectAccept[groupIndex].Add(target);
}
else
{
rejectReject[groupIndex].Add(target);
}
}
}
if (!accepted)
{
correctlyRejected++;
}
else if (!rejected)
{
wronglyAccepted++;
wrongNodeCount += encodingResult.Vector2Count[vector];
wronglyAcceptedFeatures.Add(vector);
}
else
{
correctlyRejectedInRejecting++;
}
}
Console.WriteLine("done");
Console.WriteLine(
"WA: " + wronglyAccepted + ", WR: " + wronglyRejected + "/"
+ wronglyRejectedInRejecting + ", CA: "
+ correctlyAccepted + ", CR: " + correctlyRejected + "/"
+ correctlyRejectedInRejecting);
Console.WriteLine(
"TR: "
+ trainingSet.Count + ", AF: "
+ String.Join(", ", classifier.CountAcceptingFeatures())
+ ", RF: "
+ String.Join(", ", classifier.CountRejectingFeatures()));
var wrongFeatureCount = wronglyAccepted + wronglyRejected
+ wronglyRejectedInRejecting;
var selector = new SuspiciousNodeSelector(classifier.AcceptingFeatureBitMask,
classifier.RejectingFeatureBitMask);
var suspiciousNodes = selector.SelectSuspiciousNodes(count, classifier, acceptAccept,
acceptReject, rejectAccept, rejectReject);
return(new ClassificationResult(suspiciousNodes, wronglyAcceptedFeatures,
wronglyRejectedFeatures,
wrongFeatureCount, wrongNodeCount, encodingResult));
}
public IDictionary<BigInteger, string> RevealSuspiciousElements(
ICollection<BigInteger> actuallyAcceptedVectors,
IList<SuspiciousNode> suspiciousNodes,
EncodingResult encodingResult, RevealedVectorSet trainingSet) {
var additionalAcceptedVector2GroupPath = new Dictionary<BigInteger, string>();
foreach (var t in suspiciousNodes) {
if (actuallyAcceptedVectors.Contains(t.Vector)) {
additionalAcceptedVector2GroupPath.Add(t.Vector, t.GroupKey);
trainingSet.Accepted.Add(t.Vector, t.GroupKey);
} else {
trainingSet.Rejected.Add(t.Vector, t.GroupKey);
}
}
Console.WriteLine(
"Accepted: " + trainingSet.AcceptedCount + " + "
+ additionalAcceptedVector2GroupPath.Count
+ " / " + encodingResult.IdealVectorSet.AcceptedCount);
Console.WriteLine(
"Rejected: " + trainingSet.RejectedCount + " + "
+ (suspiciousNodes.Count - additionalAcceptedVector2GroupPath.Count)
+ " / " + encodingResult.IdealVectorSet.RejectedCount);
return additionalAcceptedVector2GroupPath;
}
public void WriteResult(StreamWriter writer, RevealedVectorSet trainingSet = null) {
if (writer == null) {
return;
}
trainingSet = trainingSet ?? new RevealedVectorSet();
writer.Write(IdealAcceptedVector2GroupPath.Concat(IdealRejectedVector2GroupPath)
.Sum(f => Vector2Count[f.Key]));
writer.Write(",");
writer.Write(trainingSet.Accepted.Concat(trainingSet.Rejected)
.Sum(f => Vector2Count[f.Key]));
writer.Write(",");
writer.Write(IdealVectorSet.Count);
writer.Write(",");
writer.Write(trainingSet.Count);
writer.Write(",");
writer.Write(SeedNodeCount);
writer.Write(",");
writer.Write(SeedVectorCount);
writer.Write(",");
writer.Write(SeedAcceptedNodeCount);
writer.Write(",");
}
private UpdateResult UpdateAcceptingClassifierUnits(
IDictionary<BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet,
GroupCache groupCache) {
if (additionalAcceptedSet.Count == 0) {
var rejectedGroupInfo = FindWronglyAcceptedGroupInfo(trainingSet, groupCache);
if (rejectedGroupInfo != null) {
return new UpdateResult {
Rejected = rejectedGroupInfo,
Accepted = null,
};
}
return null;
}
foreach (var vectorAndGroupPath in additionalAcceptedSet) {
var vector = vectorAndGroupPath.Key;
var groupIndex = groupCache.GetGroupIndex(vector);
Units[groupIndex].Accepting &= vector;
var rejectedGroupInfo = FindWronglyAcceptedGroupInfo(trainingSet, groupCache);
if (rejectedGroupInfo != null) {
return new UpdateResult {
Rejected = rejectedGroupInfo,
Accepted = new GroupInfo {
Index = groupIndex, Path = vectorAndGroupPath.Value
},
};
}
}
return null;
}
private GroupInfo FindWronglyAcceptedGroupInfo(
RevealedVectorSet trainingSet, GroupCache groupCache) {
foreach (var featureAndGroupPath in trainingSet.Rejected) {
var feature = featureAndGroupPath.Key;
var groupPath = featureAndGroupPath.Value;
var groupIndex = groupCache.GetGroupIndex(feature);
var accepted = IsAccepted(feature, groupIndex) && !IsRejected(feature, groupIndex);
if (accepted) {
return new GroupInfo { Index = groupIndex, Path = groupPath };
}
}
return null;
}
public ClassificationResult Classify(
int count, Classifier classifier, GroupCache groupCache,
EncodingResult encodingResult,
RevealedVectorSet trainingSet = null) {
trainingSet = trainingSet ?? new RevealedVectorSet();
var correctlyAccepted = 0;
var correctlyRejected = 0;
var wronglyAccepted = 0;
var wronglyRejected = 0;
var correctlyRejectedInRejecting = 0;
var wronglyRejectedInRejecting = 0;
var rejectAccept = new List<List<SuspiciousNode>>();
var rejectReject = new List<List<SuspiciousNode>>();
var acceptAccept = new List<List<SuspiciousNode>>();
var acceptReject = new List<List<SuspiciousNode>>();
for (int i = 0; i < classifier.GroupCount; i++) {
rejectAccept.Add(new List<SuspiciousNode>());
rejectReject.Add(new List<SuspiciousNode>());
acceptAccept.Add(new List<SuspiciousNode>());
acceptReject.Add(new List<SuspiciousNode>());
}
var wrongNodeCount = 0;
var wronglyRejectedFeatures = new List<BigInteger>();
var wronglyAcceptedFeatures = new List<BigInteger>();
foreach (var vectorAndGroupPath in encodingResult.IdealVectorSet.Accepted) {
var vector = vectorAndGroupPath.Key;
var groupPath = vectorAndGroupPath.Value;
var groupIndex = groupCache.GetGroupIndex(vector);
var rejected = classifier.IsRejected(vector, groupIndex);
var accepted = classifier.IsAccepted(vector, groupIndex);
if (!trainingSet.Accepted.ContainsKey(vector)
&& !trainingSet.Rejected.ContainsKey(vector)) {
var target = new SuspiciousNode {
Vector = vector,
GroupKey = groupPath,
Used = false,
};
if (accepted) {
if (!rejected) {
acceptAccept[groupIndex].Add(target);
} else {
acceptReject[groupIndex].Add(target);
}
} else {
if (!rejected) {
rejectAccept[groupIndex].Add(target);
} else {
rejectReject[groupIndex].Add(target);
}
}
}
if (!accepted) {
wronglyRejected++;
wrongNodeCount += encodingResult.Vector2Count[vector];
wronglyRejectedFeatures.Add(vector);
} else if (!rejected) {
correctlyAccepted++;
} else {
wronglyRejectedInRejecting++;
wrongNodeCount += encodingResult.Vector2Count[vector];
wronglyRejectedFeatures.Add(vector);
}
}
foreach (var vectorAndGroupPath in encodingResult.IdealVectorSet.Rejected) {
var vector = vectorAndGroupPath.Key;
var groupPath = vectorAndGroupPath.Value;
var groupIndex = groupCache.GetGroupIndex(vector);
var rejected = classifier.IsRejected(vector, groupIndex);
var accepted = classifier.IsAccepted(vector, groupIndex);
if (!trainingSet.Accepted.ContainsKey(vector)
&& !trainingSet.Rejected.ContainsKey(vector)) {
var target = new SuspiciousNode {
Vector = vector,
GroupKey = groupPath,
Used = false,
};
if (accepted) {
if (!rejected) {
acceptAccept[groupIndex].Add(target);
} else {
acceptReject[groupIndex].Add(target);
}
} else {
if (!rejected) {
rejectAccept[groupIndex].Add(target);
} else {
rejectReject[groupIndex].Add(target);
}
}
}
if (!accepted) {
correctlyRejected++;
} else if (!rejected) {
wronglyAccepted++;
wrongNodeCount += encodingResult.Vector2Count[vector];
wronglyAcceptedFeatures.Add(vector);
} else {
correctlyRejectedInRejecting++;
}
}
Console.WriteLine("done");
Console.WriteLine(
"WA: " + wronglyAccepted + ", WR: " + wronglyRejected + "/"
+ wronglyRejectedInRejecting + ", CA: "
+ correctlyAccepted + ", CR: " + correctlyRejected + "/"
+ correctlyRejectedInRejecting);
Console.WriteLine(
"TR: "
+ trainingSet.Count + ", AF: "
+ String.Join(", ", classifier.CountAcceptingFeatures())
+ ", RF: "
+ String.Join(", ", classifier.CountRejectingFeatures()));
var wrongFeatureCount = wronglyAccepted + wronglyRejected
+ wronglyRejectedInRejecting;
var selector = new SuspiciousNodeSelector(classifier.AcceptingFeatureBitMask,
classifier.RejectingFeatureBitMask);
var suspiciousNodes = selector.SelectSuspiciousNodes(count, classifier, acceptAccept,
acceptReject, rejectAccept, rejectReject);
return new ClassificationResult(suspiciousNodes, wronglyAcceptedFeatures,
wronglyRejectedFeatures,
wrongFeatureCount, wrongNodeCount, encodingResult);
}