private UpdateResult UpdateAcceptingClassifierUnits(
IDictionary <BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet,
GroupCache groupCache)
{
if (additionalAcceptedSet.Count == 0)
{
var rejectedGroupInfo = FindWronglyAcceptedGroupInfo(trainingSet, groupCache);
if (rejectedGroupInfo != null)
{
return(new UpdateResult {
Rejected = rejectedGroupInfo,
Accepted = null,
});
}
return(null);
}
foreach (var vectorAndGroupPath in additionalAcceptedSet)
{
var vector = vectorAndGroupPath.Key;
var groupIndex = groupCache.GetGroupIndex(vector);
Units[groupIndex].Accepting &= vector;
var rejectedGroupInfo = FindWronglyAcceptedGroupInfo(trainingSet, groupCache);
if (rejectedGroupInfo != null)
{
return(new UpdateResult {
Rejected = rejectedGroupInfo,
Accepted = new GroupInfo {
Index = groupIndex, Path = vectorAndGroupPath.Value
},
});
}
}
return(null);
}
/// <summary>
/// 分類結果に影響を与えないacceptingFeatureを取り除く。
/// </summary>
/// <param name="rejectedVectors"></param>
/// <param name="groupCache"></param>
public void Optimize(IEnumerable <BigInteger> rejectedVectors, GroupCache groupCache)
{
var groupIndex2RejectedVectors = new List <List <BigInteger> >();
for (int i = 0; i < Units.Count; i++)
{
groupIndex2RejectedVectors.Add(new List <BigInteger>());
}
foreach (var rejectedVector in rejectedVectors)
{
var groupIndex = groupCache.GetGroupIndex(rejectedVector);
var acceptingClassifier = Units[groupIndex].Accepting;
var rejectingClassifier = Units[groupIndex].Rejecting;
var accepted = (rejectedVector & acceptingClassifier) == acceptingClassifier;
var rejected = (rejectedVector & rejectingClassifier) != BigInteger.Zero;
if (!accepted /*&& !rejected*/)
{
groupIndex2RejectedVectors[groupIndex].Add(rejectedVector);
}
}
foreach (var unitAndVectors in Units.Zip(groupIndex2RejectedVectors))
{
var unit = unitAndVectors.Item1;
var vectors = unitAndVectors.Item2;
var bit = BigInteger.One;
for (int i = 0; i < AcceptingFeatureCount; i++)
{
if ((unit.Accepting & bit) != 0)
{
unit.Accepting ^= bit;
var acceptingClassifier = unit.Accepting;
var removable = false;
foreach (var rejectedVector in vectors)
{
var accepted = (rejectedVector & acceptingClassifier)
== acceptingClassifier;
// 全てが (rejectedVector & bit) == 0 なら削除しない
// 一つでも (rejectedVector & bit) != 0 なら削除できる
removable |= (rejectedVector & bit) != 0;
if (accepted)
{
removable = false;
break;
}
}
if (!removable)
{
unit.Accepting ^= bit;
}
}
bit <<= 1;
}
}
}
/// <summary>
/// RejectedVectorが持つAcceptingFeatuerを削除する。
/// </summary>
/// <param name="rejectedVectors"></param>
/// <param name="groupCache"></param>
public void OptimizeWeakly(ICollection <BigInteger> rejectedVectors, GroupCache groupCache)
{
var usedRejectedFeatures =
Enumerable.Repeat(RejectingFeatureBitMask, Units.Count).ToList();
foreach (var rejectedVector in rejectedVectors)
{
var groupIndex = groupCache.GetGroupIndex(rejectedVector);
usedRejectedFeatures[groupIndex] |= rejectedVector;
}
foreach (var unitAndFeature in Units.Zip(usedRejectedFeatures))
{
unitAndFeature.Item1.Accepting &= AllFeatureBitMask ^ unitAndFeature.Item2;
}
}
private GroupInfo FindWronglyAcceptedGroupInfo(
RevealedVectorSet trainingSet, GroupCache groupCache)
{
foreach (var featureAndGroupPath in trainingSet.Rejected)
{
var feature = featureAndGroupPath.Key;
var groupPath = featureAndGroupPath.Value;
var groupIndex = groupCache.GetGroupIndex(feature);
var accepted = IsAccepted(feature, groupIndex) && !IsRejected(feature, groupIndex);
if (accepted)
{
return(new GroupInfo {
Index = groupIndex, Path = groupPath
});
}
}
return(null);
}
/// <summary>
/// 分類結果に影響を与えないacceptingFeatureを取り除く。
/// </summary>
/// <param name="rejectedVectors"></param>
/// <param name="groupCache"></param>
public void Optimize(IEnumerable<BigInteger> rejectedVectors, GroupCache groupCache) {
var groupIndex2RejectedVectors = new List<List<BigInteger>>();
for (int i = 0; i < Units.Count; i++) {
groupIndex2RejectedVectors.Add(new List<BigInteger>());
}
foreach (var rejectedVector in rejectedVectors) {
var groupIndex = groupCache.GetGroupIndex(rejectedVector);
var acceptingClassifier = Units[groupIndex].Accepting;
var rejectingClassifier = Units[groupIndex].Rejecting;
var accepted = (rejectedVector & acceptingClassifier) == acceptingClassifier;
var rejected = (rejectedVector & rejectingClassifier) != BigInteger.Zero;
if (!accepted /*&& !rejected*/) {
groupIndex2RejectedVectors[groupIndex].Add(rejectedVector);
}
}
foreach (var unitAndVectors in Units.Zip(groupIndex2RejectedVectors)) {
var unit = unitAndVectors.Item1;
var vectors = unitAndVectors.Item2;
var bit = BigInteger.One;
for (int i = 0; i < AcceptingFeatureCount; i++) {
if ((unit.Accepting & bit) != 0) {
unit.Accepting ^= bit;
var acceptingClassifier = unit.Accepting;
var removable = false;
foreach (var rejectedVector in vectors) {
var accepted = (rejectedVector & acceptingClassifier)
== acceptingClassifier;
// 全てが (rejectedVector & bit) == 0 なら削除しない
// 一つでも (rejectedVector & bit) != 0 なら削除できる
removable |= (rejectedVector & bit) != 0;
if (accepted) {
removable = false;
break;
}
}
if (!removable) {
unit.Accepting ^= bit;
}
}
bit <<= 1;
}
}
}
private void UpdateRejectingClassifierUnits(
IEnumerable <BigInteger> acceptedVectors, GroupCache groupCache)
{
var count = Units.Count;
for (int i = 0; i < count; i++)
{
Units[i].Rejecting ^= RejectingFeatureBitMask;
}
foreach (var vector in acceptedVectors)
{
var iGroupKey = groupCache.GetGroupIndex(vector);
Units[iGroupKey].Rejecting |= vector;
}
for (int i = 0; i < count; i++)
{
Units[i].Rejecting ^= RejectingFeatureBitMask;
Units[i].Rejecting &= RejectingFeatureBitMask;
}
}
/// <summary>
/// RejectedVectorが持つAcceptingFeatuerを削除する。
/// </summary>
/// <param name="rejectedVectors"></param>
/// <param name="groupCache"></param>
public void OptimizeWeakly(ICollection<BigInteger> rejectedVectors, GroupCache groupCache) {
var usedRejectedFeatures =
Enumerable.Repeat(RejectingFeatureBitMask, Units.Count).ToList();
foreach (var rejectedVector in rejectedVectors) {
var groupIndex = groupCache.GetGroupIndex(rejectedVector);
usedRejectedFeatures[groupIndex] |= rejectedVector;
}
foreach (var unitAndFeature in Units.Zip(usedRejectedFeatures)) {
unitAndFeature.Item1.Accepting &= AllFeatureBitMask ^ unitAndFeature.Item2;
}
}
private GroupInfo FindWronglyAcceptedGroupInfo(
RevealedVectorSet trainingSet, GroupCache groupCache) {
foreach (var featureAndGroupPath in trainingSet.Rejected) {
var feature = featureAndGroupPath.Key;
var groupPath = featureAndGroupPath.Value;
var groupIndex = groupCache.GetGroupIndex(feature);
var accepted = IsAccepted(feature, groupIndex) && !IsRejected(feature, groupIndex);
if (accepted) {
return new GroupInfo { Index = groupIndex, Path = groupPath };
}
}
return null;
}
private UpdateResult UpdateAcceptingClassifierUnits(
IDictionary<BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet,
GroupCache groupCache) {
if (additionalAcceptedSet.Count == 0) {
var rejectedGroupInfo = FindWronglyAcceptedGroupInfo(trainingSet, groupCache);
if (rejectedGroupInfo != null) {
return new UpdateResult {
Rejected = rejectedGroupInfo,
Accepted = null,
};
}
return null;
}
foreach (var vectorAndGroupPath in additionalAcceptedSet) {
var vector = vectorAndGroupPath.Key;
var groupIndex = groupCache.GetGroupIndex(vector);
Units[groupIndex].Accepting &= vector;
var rejectedGroupInfo = FindWronglyAcceptedGroupInfo(trainingSet, groupCache);
if (rejectedGroupInfo != null) {
return new UpdateResult {
Rejected = rejectedGroupInfo,
Accepted = new GroupInfo {
Index = groupIndex, Path = vectorAndGroupPath.Value
},
};
}
}
return null;
}
private void UpdateRejectingClassifierUnits(
IEnumerable<BigInteger> acceptedVectors, GroupCache groupCache) {
var count = Units.Count;
for (int i = 0; i < count; i++) {
Units[i].Rejecting ^= RejectingFeatureBitMask;
}
foreach (var vector in acceptedVectors) {
var iGroupKey = groupCache.GetGroupIndex(vector);
Units[iGroupKey].Rejecting |= vector;
}
for (int i = 0; i < count; i++) {
Units[i].Rejecting ^= RejectingFeatureBitMask;
Units[i].Rejecting &= RejectingFeatureBitMask;
}
}
public ClassificationResult Classify(
int count, Classifier classifier, GroupCache groupCache,
EncodingResult encodingResult,
RevealedVectorSet trainingSet = null)
{
trainingSet = trainingSet ?? new RevealedVectorSet();
var correctlyAccepted = 0;
var correctlyRejected = 0;
var wronglyAccepted = 0;
var wronglyRejected = 0;
var correctlyRejectedInRejecting = 0;
var wronglyRejectedInRejecting = 0;
var rejectAccept = new List <List <SuspiciousNode> >();
var rejectReject = new List <List <SuspiciousNode> >();
var acceptAccept = new List <List <SuspiciousNode> >();
var acceptReject = new List <List <SuspiciousNode> >();
for (int i = 0; i < classifier.GroupCount; i++)
{
rejectAccept.Add(new List <SuspiciousNode>());
rejectReject.Add(new List <SuspiciousNode>());
acceptAccept.Add(new List <SuspiciousNode>());
acceptReject.Add(new List <SuspiciousNode>());
}
var wrongNodeCount = 0;
var wronglyRejectedFeatures = new List <BigInteger>();
var wronglyAcceptedFeatures = new List <BigInteger>();
foreach (var vectorAndGroupPath in encodingResult.IdealVectorSet.Accepted)
{
var vector = vectorAndGroupPath.Key;
var groupPath = vectorAndGroupPath.Value;
var groupIndex = groupCache.GetGroupIndex(vector);
var rejected = classifier.IsRejected(vector, groupIndex);
var accepted = classifier.IsAccepted(vector, groupIndex);
if (!trainingSet.Accepted.ContainsKey(vector) &&
!trainingSet.Rejected.ContainsKey(vector))
{
var target = new SuspiciousNode {
Vector = vector,
GroupKey = groupPath,
Used = false,
};
if (accepted)
{
if (!rejected)
{
acceptAccept[groupIndex].Add(target);
}
else
{
acceptReject[groupIndex].Add(target);
}
}
else
{
if (!rejected)
{
rejectAccept[groupIndex].Add(target);
}
else
{
rejectReject[groupIndex].Add(target);
}
}
}
if (!accepted)
{
wronglyRejected++;
wrongNodeCount += encodingResult.Vector2Count[vector];
wronglyRejectedFeatures.Add(vector);
}
else if (!rejected)
{
correctlyAccepted++;
}
else
{
wronglyRejectedInRejecting++;
wrongNodeCount += encodingResult.Vector2Count[vector];
wronglyRejectedFeatures.Add(vector);
}
}
foreach (var vectorAndGroupPath in encodingResult.IdealVectorSet.Rejected)
{
var vector = vectorAndGroupPath.Key;
var groupPath = vectorAndGroupPath.Value;
var groupIndex = groupCache.GetGroupIndex(vector);
var rejected = classifier.IsRejected(vector, groupIndex);
var accepted = classifier.IsAccepted(vector, groupIndex);
if (!trainingSet.Accepted.ContainsKey(vector) &&
!trainingSet.Rejected.ContainsKey(vector))
{
var target = new SuspiciousNode {
Vector = vector,
GroupKey = groupPath,
Used = false,
};
if (accepted)
{
if (!rejected)
{
acceptAccept[groupIndex].Add(target);
}
else
{
acceptReject[groupIndex].Add(target);
}
}
else
{
if (!rejected)
{
rejectAccept[groupIndex].Add(target);
}
else
{
rejectReject[groupIndex].Add(target);
}
}
}
if (!accepted)
{
correctlyRejected++;
}
else if (!rejected)
{
wronglyAccepted++;
wrongNodeCount += encodingResult.Vector2Count[vector];
wronglyAcceptedFeatures.Add(vector);
}
else
{
correctlyRejectedInRejecting++;
}
}
Console.WriteLine("done");
Console.WriteLine(
"WA: " + wronglyAccepted + ", WR: " + wronglyRejected + "/"
+ wronglyRejectedInRejecting + ", CA: "
+ correctlyAccepted + ", CR: " + correctlyRejected + "/"
+ correctlyRejectedInRejecting);
Console.WriteLine(
"TR: "
+ trainingSet.Count + ", AF: "
+ String.Join(", ", classifier.CountAcceptingFeatures())
+ ", RF: "
+ String.Join(", ", classifier.CountRejectingFeatures()));
var wrongFeatureCount = wronglyAccepted + wronglyRejected
+ wronglyRejectedInRejecting;
var selector = new SuspiciousNodeSelector(classifier.AcceptingFeatureBitMask,
classifier.RejectingFeatureBitMask);
var suspiciousNodes = selector.SelectSuspiciousNodes(count, classifier, acceptAccept,
acceptReject, rejectAccept, rejectReject);
return(new ClassificationResult(suspiciousNodes, wronglyAcceptedFeatures,
wronglyRejectedFeatures,
wrongFeatureCount, wrongNodeCount, encodingResult));
}
public ClassificationResult Classify(
int count, Classifier classifier, GroupCache groupCache,
EncodingResult encodingResult,
RevealedVectorSet trainingSet = null) {
trainingSet = trainingSet ?? new RevealedVectorSet();
var correctlyAccepted = 0;
var correctlyRejected = 0;
var wronglyAccepted = 0;
var wronglyRejected = 0;
var correctlyRejectedInRejecting = 0;
var wronglyRejectedInRejecting = 0;
var rejectAccept = new List<List<SuspiciousNode>>();
var rejectReject = new List<List<SuspiciousNode>>();
var acceptAccept = new List<List<SuspiciousNode>>();
var acceptReject = new List<List<SuspiciousNode>>();
for (int i = 0; i < classifier.GroupCount; i++) {
rejectAccept.Add(new List<SuspiciousNode>());
rejectReject.Add(new List<SuspiciousNode>());
acceptAccept.Add(new List<SuspiciousNode>());
acceptReject.Add(new List<SuspiciousNode>());
}
var wrongNodeCount = 0;
var wronglyRejectedFeatures = new List<BigInteger>();
var wronglyAcceptedFeatures = new List<BigInteger>();
foreach (var vectorAndGroupPath in encodingResult.IdealVectorSet.Accepted) {
var vector = vectorAndGroupPath.Key;
var groupPath = vectorAndGroupPath.Value;
var groupIndex = groupCache.GetGroupIndex(vector);
var rejected = classifier.IsRejected(vector, groupIndex);
var accepted = classifier.IsAccepted(vector, groupIndex);
if (!trainingSet.Accepted.ContainsKey(vector)
&& !trainingSet.Rejected.ContainsKey(vector)) {
var target = new SuspiciousNode {
Vector = vector,
GroupKey = groupPath,
Used = false,
};
if (accepted) {
if (!rejected) {
acceptAccept[groupIndex].Add(target);
} else {
acceptReject[groupIndex].Add(target);
}
} else {
if (!rejected) {
rejectAccept[groupIndex].Add(target);
} else {
rejectReject[groupIndex].Add(target);
}
}
}
if (!accepted) {
wronglyRejected++;
wrongNodeCount += encodingResult.Vector2Count[vector];
wronglyRejectedFeatures.Add(vector);
} else if (!rejected) {
correctlyAccepted++;
} else {
wronglyRejectedInRejecting++;
wrongNodeCount += encodingResult.Vector2Count[vector];
wronglyRejectedFeatures.Add(vector);
}
}
foreach (var vectorAndGroupPath in encodingResult.IdealVectorSet.Rejected) {
var vector = vectorAndGroupPath.Key;
var groupPath = vectorAndGroupPath.Value;
var groupIndex = groupCache.GetGroupIndex(vector);
var rejected = classifier.IsRejected(vector, groupIndex);
var accepted = classifier.IsAccepted(vector, groupIndex);
if (!trainingSet.Accepted.ContainsKey(vector)
&& !trainingSet.Rejected.ContainsKey(vector)) {
var target = new SuspiciousNode {
Vector = vector,
GroupKey = groupPath,
Used = false,
};
if (accepted) {
if (!rejected) {
acceptAccept[groupIndex].Add(target);
} else {
acceptReject[groupIndex].Add(target);
}
} else {
if (!rejected) {
rejectAccept[groupIndex].Add(target);
} else {
rejectReject[groupIndex].Add(target);
}
}
}
if (!accepted) {
correctlyRejected++;
} else if (!rejected) {
wronglyAccepted++;
wrongNodeCount += encodingResult.Vector2Count[vector];
wronglyAcceptedFeatures.Add(vector);
} else {
correctlyRejectedInRejecting++;
}
}
Console.WriteLine("done");
Console.WriteLine(
"WA: " + wronglyAccepted + ", WR: " + wronglyRejected + "/"
+ wronglyRejectedInRejecting + ", CA: "
+ correctlyAccepted + ", CR: " + correctlyRejected + "/"
+ correctlyRejectedInRejecting);
Console.WriteLine(
"TR: "
+ trainingSet.Count + ", AF: "
+ String.Join(", ", classifier.CountAcceptingFeatures())
+ ", RF: "
+ String.Join(", ", classifier.CountRejectingFeatures()));
var wrongFeatureCount = wronglyAccepted + wronglyRejected
+ wronglyRejectedInRejecting;
var selector = new SuspiciousNodeSelector(classifier.AcceptingFeatureBitMask,
classifier.RejectingFeatureBitMask);
var suspiciousNodes = selector.SelectSuspiciousNodes(count, classifier, acceptAccept,
acceptReject, rejectAccept, rejectReject);
return new ClassificationResult(suspiciousNodes, wronglyAcceptedFeatures,
wronglyRejectedFeatures,
wrongFeatureCount, wrongNodeCount, encodingResult);
}