public void Create(RevealedVectorSet trainingSet, GroupCache groupCache)
{
do
{
Initialize();
} while (!UpdateClassifierUnits(trainingSet.Accepted, trainingSet, groupCache));
}
private UpdateResult UpdateAcceptingClassifierUnits(
IDictionary <BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet,
GroupCache groupCache)
{
if (additionalAcceptedSet.Count == 0)
{
var rejectedGroupInfo = FindWronglyAcceptedGroupInfo(trainingSet, groupCache);
if (rejectedGroupInfo != null)
{
return(new UpdateResult {
Rejected = rejectedGroupInfo,
Accepted = null,
});
}
return(null);
}
foreach (var vectorAndGroupPath in additionalAcceptedSet)
{
var vector = vectorAndGroupPath.Key;
var groupIndex = groupCache.GetGroupIndex(vector);
Units[groupIndex].Accepting &= vector;
var rejectedGroupInfo = FindWronglyAcceptedGroupInfo(trainingSet, groupCache);
if (rejectedGroupInfo != null)
{
return(new UpdateResult {
Rejected = rejectedGroupInfo,
Accepted = new GroupInfo {
Index = groupIndex, Path = vectorAndGroupPath.Value
},
});
}
}
return(null);
}
/// <summary>
/// 分類結果に影響を与えないacceptingFeatureを取り除く。
/// </summary>
/// <param name="rejectedVectors"></param>
/// <param name="groupCache"></param>
public void Optimize(IEnumerable <BigInteger> rejectedVectors, GroupCache groupCache)
{
var groupIndex2RejectedVectors = new List <List <BigInteger> >();
for (int i = 0; i < Units.Count; i++)
{
groupIndex2RejectedVectors.Add(new List <BigInteger>());
}
foreach (var rejectedVector in rejectedVectors)
{
var groupIndex = groupCache.GetGroupIndex(rejectedVector);
var acceptingClassifier = Units[groupIndex].Accepting;
var rejectingClassifier = Units[groupIndex].Rejecting;
var accepted = (rejectedVector & acceptingClassifier) == acceptingClassifier;
var rejected = (rejectedVector & rejectingClassifier) != BigInteger.Zero;
if (!accepted /*&& !rejected*/)
{
groupIndex2RejectedVectors[groupIndex].Add(rejectedVector);
}
}
foreach (var unitAndVectors in Units.Zip(groupIndex2RejectedVectors))
{
var unit = unitAndVectors.Item1;
var vectors = unitAndVectors.Item2;
var bit = BigInteger.One;
for (int i = 0; i < AcceptingFeatureCount; i++)
{
if ((unit.Accepting & bit) != 0)
{
unit.Accepting ^= bit;
var acceptingClassifier = unit.Accepting;
var removable = false;
foreach (var rejectedVector in vectors)
{
var accepted = (rejectedVector & acceptingClassifier)
== acceptingClassifier;
// 全てが (rejectedVector & bit) == 0 なら削除しない
// 一つでも (rejectedVector & bit) != 0 なら削除できる
removable |= (rejectedVector & bit) != 0;
if (accepted)
{
removable = false;
break;
}
}
if (!removable)
{
unit.Accepting ^= bit;
}
}
bit <<= 1;
}
}
}
public bool Update(
IDictionary <BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet,
GroupCache groupCache)
{
var originalClassifierUnit = Units.Select(c => c.Clone()).ToImmutableList();
if (!UpdateClassifierUnits(additionalAcceptedSet, trainingSet, groupCache))
{
Create(trainingSet, groupCache);
}
return(!Units.SequenceEqual(originalClassifierUnit));
}
/// <summary>
/// RejectedVectorが持つAcceptingFeatuerを削除する。
/// </summary>
/// <param name="rejectedVectors"></param>
/// <param name="groupCache"></param>
public void OptimizeWeakly(ICollection <BigInteger> rejectedVectors, GroupCache groupCache)
{
var usedRejectedFeatures =
Enumerable.Repeat(RejectingFeatureBitMask, Units.Count).ToList();
foreach (var rejectedVector in rejectedVectors)
{
var groupIndex = groupCache.GetGroupIndex(rejectedVector);
usedRejectedFeatures[groupIndex] |= rejectedVector;
}
foreach (var unitAndFeature in Units.Zip(usedRejectedFeatures))
{
unitAndFeature.Item1.Accepting &= AllFeatureBitMask ^ unitAndFeature.Item2;
}
}
public ClassificationResult Apply(
ICollection<string> codePaths, string searchPattern, FeatureEncoder featureEncoder,
Classifier classifier, StreamWriter writer = null) {
var allCsts = GenerateValidCsts(codePaths);
var encodingResult = featureEncoder.Encode(codePaths, allCsts, this);
var time = Environment.TickCount;
var groupCache = new GroupCache(encodingResult, classifier);
var result = Classify(int.MaxValue, classifier, groupCache, encodingResult);
Console.WriteLine("Time: " + (Environment.TickCount - time));
if (writer != null) {
encodingResult.WriteResult(writer);
}
return result;
}
private GroupInfo FindWronglyAcceptedGroupInfo(
RevealedVectorSet trainingSet, GroupCache groupCache)
{
foreach (var featureAndGroupPath in trainingSet.Rejected)
{
var feature = featureAndGroupPath.Key;
var groupPath = featureAndGroupPath.Value;
var groupIndex = groupCache.GetGroupIndex(feature);
var accepted = IsAccepted(feature, groupIndex) && !IsRejected(feature, groupIndex);
if (accepted)
{
return(new GroupInfo {
Index = groupIndex, Path = groupPath
});
}
}
return(null);
}
/// <summary>
/// 分類結果に影響を与えないacceptingFeatureを取り除く。
/// </summary>
/// <param name="rejectedVectors"></param>
/// <param name="groupCache"></param>
public void Optimize(IEnumerable<BigInteger> rejectedVectors, GroupCache groupCache) {
var groupIndex2RejectedVectors = new List<List<BigInteger>>();
for (int i = 0; i < Units.Count; i++) {
groupIndex2RejectedVectors.Add(new List<BigInteger>());
}
foreach (var rejectedVector in rejectedVectors) {
var groupIndex = groupCache.GetGroupIndex(rejectedVector);
var acceptingClassifier = Units[groupIndex].Accepting;
var rejectingClassifier = Units[groupIndex].Rejecting;
var accepted = (rejectedVector & acceptingClassifier) == acceptingClassifier;
var rejected = (rejectedVector & rejectingClassifier) != BigInteger.Zero;
if (!accepted /*&& !rejected*/) {
groupIndex2RejectedVectors[groupIndex].Add(rejectedVector);
}
}
foreach (var unitAndVectors in Units.Zip(groupIndex2RejectedVectors)) {
var unit = unitAndVectors.Item1;
var vectors = unitAndVectors.Item2;
var bit = BigInteger.One;
for (int i = 0; i < AcceptingFeatureCount; i++) {
if ((unit.Accepting & bit) != 0) {
unit.Accepting ^= bit;
var acceptingClassifier = unit.Accepting;
var removable = false;
foreach (var rejectedVector in vectors) {
var accepted = (rejectedVector & acceptingClassifier)
== acceptingClassifier;
// 全てが (rejectedVector & bit) == 0 なら削除しない
// 一つでも (rejectedVector & bit) != 0 なら削除できる
removable |= (rejectedVector & bit) != 0;
if (accepted) {
removable = false;
break;
}
}
if (!removable) {
unit.Accepting ^= bit;
}
}
bit <<= 1;
}
}
}
public ClassificationResult Apply(
ICollection <string> codePaths, string searchPattern, FeatureEncoder featureEncoder,
Classifier classifier, StreamWriter writer = null)
{
var allCsts = GenerateValidCsts(codePaths);
var encodingResult = featureEncoder.Encode(codePaths, allCsts, this);
var time = Environment.TickCount;
var groupCache = new GroupCache(encodingResult, classifier);
var result = Classify(int.MaxValue, classifier, groupCache, encodingResult);
Console.WriteLine("Time: " + (Environment.TickCount - time));
if (writer != null)
{
encodingResult.WriteResult(writer);
}
return(result);
}
private void UpdateRejectingClassifierUnits(
IEnumerable <BigInteger> acceptedVectors, GroupCache groupCache)
{
var count = Units.Count;
for (int i = 0; i < count; i++)
{
Units[i].Rejecting ^= RejectingFeatureBitMask;
}
foreach (var vector in acceptedVectors)
{
var iGroupKey = groupCache.GetGroupIndex(vector);
Units[iGroupKey].Rejecting |= vector;
}
for (int i = 0; i < count; i++)
{
Units[i].Rejecting ^= RejectingFeatureBitMask;
Units[i].Rejecting &= RejectingFeatureBitMask;
}
}
private bool UpdateClassifierUnits(
IDictionary <BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet,
GroupCache groupCache)
{
UpdateRejectingClassifierUnits(additionalAcceptedSet.Keys, groupCache);
var result = UpdateAcceptingClassifierUnits(additionalAcceptedSet, trainingSet,
groupCache);
if (result == null)
{
return(true);
}
var count = GroupPaths.Count;
var updated = false;
if (result.Accepted != null &&
result.Accepted.Path != GroupPaths[result.Accepted.Index])
{
updated |= AddNewGroup(result.Accepted);
}
if (result.Rejected.Path != GroupPaths[result.Rejected.Index])
{
updated |= AddNewGroup(result.Rejected);
}
if (updated)
{
groupCache.UpdateGroupCache(this);
}
// ReSharper disable once PossibleUnintendedReferenceComparison
if (GroupPaths.Count == count &&
additionalAcceptedSet == trainingSet.Accepted)
{
throw new Exception("Fail to learn rules");
}
Console.WriteLine("Failed to create Classifier units.");
Console.WriteLine("Groups: " + GroupPaths.Count + " (" + count + ")");
return(false);
}
private void UpdateRejectingClassifierUnits(
IEnumerable<BigInteger> acceptedVectors, GroupCache groupCache) {
var count = Units.Count;
for (int i = 0; i < count; i++) {
Units[i].Rejecting ^= RejectingFeatureBitMask;
}
foreach (var vector in acceptedVectors) {
var iGroupKey = groupCache.GetGroupIndex(vector);
Units[iGroupKey].Rejecting |= vector;
}
for (int i = 0; i < count; i++) {
Units[i].Rejecting ^= RejectingFeatureBitMask;
Units[i].Rejecting &= RejectingFeatureBitMask;
}
}
private bool UpdateClassifierUnits(
IDictionary<BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet,
GroupCache groupCache) {
UpdateRejectingClassifierUnits(additionalAcceptedSet.Keys, groupCache);
var result = UpdateAcceptingClassifierUnits(additionalAcceptedSet, trainingSet,
groupCache);
if (result == null) {
return true;
}
var count = GroupPaths.Count;
var updated = false;
if (result.Accepted != null
&& result.Accepted.Path != GroupPaths[result.Accepted.Index]) {
updated |= AddNewGroup(result.Accepted);
}
if (result.Rejected.Path != GroupPaths[result.Rejected.Index]) {
updated |= AddNewGroup(result.Rejected);
}
if (updated) {
groupCache.UpdateGroupCache(this);
}
// ReSharper disable once PossibleUnintendedReferenceComparison
if (GroupPaths.Count == count
&& additionalAcceptedSet == trainingSet.Accepted) {
throw new Exception("Fail to learn rules");
}
Console.WriteLine("Failed to create Classifier units.");
Console.WriteLine("Groups: " + GroupPaths.Count + " (" + count + ")");
return false;
}
public void Create(RevealedVectorSet trainingSet, GroupCache groupCache) {
do {
Initialize();
} while (!UpdateClassifierUnits(trainingSet.Accepted, trainingSet, groupCache));
}
public bool Update(
IDictionary<BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet,
GroupCache groupCache) {
var originalClassifierUnit = Units.Select(c => c.Clone()).ToImmutableList();
if (!UpdateClassifierUnits(additionalAcceptedSet, trainingSet, groupCache)) {
Create(trainingSet, groupCache);
}
return !Units.SequenceEqual(originalClassifierUnit);
}
public LearningResult Learn(
ICollection <string> seedPaths, ICollection <string> codePaths, string searchPattern,
StreamWriter writer = null)
{
var allCsts = GenerateValidCsts(codePaths);
var seedCsts = GenerateValidCsts(seedPaths).ToList();
var seedNodes = seedCsts
.SelectMany(
cst => LearningExperimentUtil.GetUppermostNodesByNames(cst, OracleNames))
.Where(ProtectedIsAcceptedUsingOracle)
.ToList();
var seedCst = seedCsts.First();
var seedCode = seedCst.Code;
var structuredCode = new StructuredCode(seedCode);
var acceptingFragments = ConstructAcceptingFragments(structuredCode, seedCst, seedNodes);
var rejectingFragments = ConstructRejectingFragments(structuredCode, seedCst);
SeedNodeSet.Create(acceptingFragments, this);
var preparingTime = Environment.TickCount;
var extractor = CreateExtractor();
var seedNodeSet = new SeedNodeSet(acceptingFragments.Select(f => f.Node), seedCsts, this);
Console.WriteLine("#Accepted seed nodes: " + seedNodeSet.AcceptedNodes.Count
+ " (" + acceptingFragments.Count + ")");
Console.WriteLine("#Rejected seed nodes: " + seedNodeSet.RejectedNodes.Count
+ " (" + rejectingFragments.Count + ")");
var featureSet = new FeatuerSet(seedNodeSet, extractor, acceptingFragments, rejectingFragments);
var groupPaths = seedNodeSet.SelectedNodeNames.Select(n => ">" + n + ">");
var classifier = new Classifier(groupPaths, featureSet);
Console.WriteLine(
"#Features: " + featureSet.AcceptingFeatureCount + ", "
+ featureSet.RejectingFeatureCount);
Console.WriteLine("Inner: " + extractor.IsInner);
var featureEncoder = new FeatureEncoder(seedNodeSet.SelectedNodeNames, extractor,
featureSet);
var encodingResult = featureEncoder.Encode(codePaths, allCsts, this, seedNodeSet);
Console.WriteLine("#Unique Elements: " + encodingResult.VectorCount);
if (encodingResult.IdealAcceptedVector2GroupPath.Keys.ToHashSet()
.Overlaps(encodingResult.IdealRejectedVector2GroupPath.Keys.ToHashSet()))
{
var others = encodingResult.IdealRejectedVector2GroupPath;
var vector =
encodingResult.IdealAcceptedVector2GroupPath.Keys.First(others.ContainsKey);
foreach (var featureString in featureEncoder.GetFeatureStringsByVector(vector))
{
Console.WriteLine(Experiment.Beautify(featureString));
}
throw new Exception("Master predicates can't classify elements!");
}
var groupCache = new GroupCache(encodingResult, classifier);
var trainingSet = encodingResult.CreateTrainingVectorSet();
classifier.Create(trainingSet, groupCache);
Experiment.WriteFeatureStrings(Console.Out, classifier, featureEncoder);
Console.WriteLine("Preparing time: " + (Environment.TickCount - preparingTime));
var count = 0;
var sumTime = Environment.TickCount;
ClassificationResult classificationResult;
while (true)
{
var time = Environment.TickCount;
classificationResult = Classify(count, classifier, groupCache, encodingResult,
trainingSet);
if (classificationResult.SuspiciousNodes == null)
{
break;
}
var additionalAcceptedSet = RevealSuspiciousElements(
encodingResult.IdealAcceptedVector2GroupPath.Keys,
classificationResult.SuspiciousNodes, encodingResult, trainingSet);
if (!classifier.Update(additionalAcceptedSet, trainingSet, groupCache))
{
count++;
}
else
{
count = 0;
}
Console.WriteLine("Time: " + (Environment.TickCount - time));
}
classifier.MakeImmutable();
Console.WriteLine();
Console.WriteLine("Sum time: " + (Environment.TickCount - sumTime));
var trainingVectorCount = trainingSet.Count;
var idealVectorCount = encodingResult.IdealVectorSet.Count;
Console.WriteLine("#Required vectors: " + trainingVectorCount + " / " + idealVectorCount);
if (writer != null)
{
encodingResult.WriteResult(writer, trainingSet);
}
foreach (var groupPath in classifier.GroupPaths)
{
Console.WriteLine(groupPath);
}
classifier.Optimize(encodingResult.IdealRejectedVector2GroupPath.Keys, groupCache);
return(new LearningResult {
ClassificationResult = classificationResult,
Classifier = classifier,
EncodingResult = encodingResult,
FeatureEncoder = featureEncoder,
});
}
public LearningResult Learn(
ICollection<string> seedPaths, ICollection<string> codePaths, string searchPattern,
StreamWriter writer = null) {
var allCsts = GenerateValidCsts(codePaths);
var seedCsts = GenerateValidCsts(seedPaths).ToList();
var seedNodes = seedCsts
.SelectMany(
cst => LearningExperimentUtil.GetUppermostNodesByNames(cst, OracleNames))
.Where(ProtectedIsAcceptedUsingOracle)
.ToList();
var seedCst = seedCsts.First();
var seedCode = seedCst.Code;
var structuredCode = new StructuredCode(seedCode);
var acceptingFragments = ConstructAcceptingFragments(structuredCode, seedCst, seedNodes);
var rejectingFragments = ConstructRejectingFragments(structuredCode, seedCst);
SeedNodeSet.Create(acceptingFragments, this);
var preparingTime = Environment.TickCount;
var extractor = CreateExtractor();
var seedNodeSet = new SeedNodeSet(acceptingFragments.Select(f => f.Node), seedCsts, this);
Console.WriteLine("#Accepted seed nodes: " + seedNodeSet.AcceptedNodes.Count
+ " (" + acceptingFragments.Count + ")");
Console.WriteLine("#Rejected seed nodes: " + seedNodeSet.RejectedNodes.Count
+ " (" + rejectingFragments.Count + ")");
var featureSet = new FeatuerSet(seedNodeSet, extractor, acceptingFragments, rejectingFragments);
var groupPaths = seedNodeSet.SelectedNodeNames.Select(n => ">" + n + ">");
var classifier = new Classifier(groupPaths, featureSet);
Console.WriteLine(
"#Features: " + featureSet.AcceptingFeatureCount + ", "
+ featureSet.RejectingFeatureCount);
Console.WriteLine("Inner: " + extractor.IsInner);
var featureEncoder = new FeatureEncoder(seedNodeSet.SelectedNodeNames, extractor,
featureSet);
var encodingResult = featureEncoder.Encode(codePaths, allCsts, this, seedNodeSet);
Console.WriteLine("#Unique Elements: " + encodingResult.VectorCount);
if (encodingResult.IdealAcceptedVector2GroupPath.Keys.ToHashSet()
.Overlaps(encodingResult.IdealRejectedVector2GroupPath.Keys.ToHashSet())) {
var others = encodingResult.IdealRejectedVector2GroupPath;
var vector =
encodingResult.IdealAcceptedVector2GroupPath.Keys.First(others.ContainsKey);
foreach (var featureString in featureEncoder.GetFeatureStringsByVector(vector)) {
Console.WriteLine(Experiment.Beautify(featureString));
}
throw new Exception("Master predicates can't classify elements!");
}
var groupCache = new GroupCache(encodingResult, classifier);
var trainingSet = encodingResult.CreateTrainingVectorSet();
classifier.Create(trainingSet, groupCache);
Experiment.WriteFeatureStrings(Console.Out, classifier, featureEncoder);
Console.WriteLine("Preparing time: " + (Environment.TickCount - preparingTime));
var count = 0;
var sumTime = Environment.TickCount;
ClassificationResult classificationResult;
while (true) {
var time = Environment.TickCount;
classificationResult = Classify(count, classifier, groupCache, encodingResult,
trainingSet);
if (classificationResult.SuspiciousNodes == null) {
break;
}
var additionalAcceptedSet = RevealSuspiciousElements(
encodingResult.IdealAcceptedVector2GroupPath.Keys,
classificationResult.SuspiciousNodes, encodingResult, trainingSet);
if (!classifier.Update(additionalAcceptedSet, trainingSet, groupCache)) {
count++;
} else {
count = 0;
}
Console.WriteLine("Time: " + (Environment.TickCount - time));
}
classifier.MakeImmutable();
Console.WriteLine();
Console.WriteLine("Sum time: " + (Environment.TickCount - sumTime));
var trainingVectorCount = trainingSet.Count;
var idealVectorCount = encodingResult.IdealVectorSet.Count;
Console.WriteLine("#Required vectors: " + trainingVectorCount + " / " + idealVectorCount);
if (writer != null) {
encodingResult.WriteResult(writer, trainingSet);
}
foreach (var groupPath in classifier.GroupPaths) {
Console.WriteLine(groupPath);
}
classifier.Optimize(encodingResult.IdealRejectedVector2GroupPath.Keys, groupCache);
return new LearningResult {
ClassificationResult = classificationResult,
Classifier = classifier,
EncodingResult = encodingResult,
FeatureEncoder = featureEncoder,
};
}
private UpdateResult UpdateAcceptingClassifierUnits(
IDictionary<BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet,
GroupCache groupCache) {
if (additionalAcceptedSet.Count == 0) {
var rejectedGroupInfo = FindWronglyAcceptedGroupInfo(trainingSet, groupCache);
if (rejectedGroupInfo != null) {
return new UpdateResult {
Rejected = rejectedGroupInfo,
Accepted = null,
};
}
return null;
}
foreach (var vectorAndGroupPath in additionalAcceptedSet) {
var vector = vectorAndGroupPath.Key;
var groupIndex = groupCache.GetGroupIndex(vector);
Units[groupIndex].Accepting &= vector;
var rejectedGroupInfo = FindWronglyAcceptedGroupInfo(trainingSet, groupCache);
if (rejectedGroupInfo != null) {
return new UpdateResult {
Rejected = rejectedGroupInfo,
Accepted = new GroupInfo {
Index = groupIndex, Path = vectorAndGroupPath.Value
},
};
}
}
return null;
}
private GroupInfo FindWronglyAcceptedGroupInfo(
RevealedVectorSet trainingSet, GroupCache groupCache) {
foreach (var featureAndGroupPath in trainingSet.Rejected) {
var feature = featureAndGroupPath.Key;
var groupPath = featureAndGroupPath.Value;
var groupIndex = groupCache.GetGroupIndex(feature);
var accepted = IsAccepted(feature, groupIndex) && !IsRejected(feature, groupIndex);
if (accepted) {
return new GroupInfo { Index = groupIndex, Path = groupPath };
}
}
return null;
}
public ClassificationResult Classify(
int count, Classifier classifier, GroupCache groupCache,
EncodingResult encodingResult,
RevealedVectorSet trainingSet = null)
{
trainingSet = trainingSet ?? new RevealedVectorSet();
var correctlyAccepted = 0;
var correctlyRejected = 0;
var wronglyAccepted = 0;
var wronglyRejected = 0;
var correctlyRejectedInRejecting = 0;
var wronglyRejectedInRejecting = 0;
var rejectAccept = new List <List <SuspiciousNode> >();
var rejectReject = new List <List <SuspiciousNode> >();
var acceptAccept = new List <List <SuspiciousNode> >();
var acceptReject = new List <List <SuspiciousNode> >();
for (int i = 0; i < classifier.GroupCount; i++)
{
rejectAccept.Add(new List <SuspiciousNode>());
rejectReject.Add(new List <SuspiciousNode>());
acceptAccept.Add(new List <SuspiciousNode>());
acceptReject.Add(new List <SuspiciousNode>());
}
var wrongNodeCount = 0;
var wronglyRejectedFeatures = new List <BigInteger>();
var wronglyAcceptedFeatures = new List <BigInteger>();
foreach (var vectorAndGroupPath in encodingResult.IdealVectorSet.Accepted)
{
var vector = vectorAndGroupPath.Key;
var groupPath = vectorAndGroupPath.Value;
var groupIndex = groupCache.GetGroupIndex(vector);
var rejected = classifier.IsRejected(vector, groupIndex);
var accepted = classifier.IsAccepted(vector, groupIndex);
if (!trainingSet.Accepted.ContainsKey(vector) &&
!trainingSet.Rejected.ContainsKey(vector))
{
var target = new SuspiciousNode {
Vector = vector,
GroupKey = groupPath,
Used = false,
};
if (accepted)
{
if (!rejected)
{
acceptAccept[groupIndex].Add(target);
}
else
{
acceptReject[groupIndex].Add(target);
}
}
else
{
if (!rejected)
{
rejectAccept[groupIndex].Add(target);
}
else
{
rejectReject[groupIndex].Add(target);
}
}
}
if (!accepted)
{
wronglyRejected++;
wrongNodeCount += encodingResult.Vector2Count[vector];
wronglyRejectedFeatures.Add(vector);
}
else if (!rejected)
{
correctlyAccepted++;
}
else
{
wronglyRejectedInRejecting++;
wrongNodeCount += encodingResult.Vector2Count[vector];
wronglyRejectedFeatures.Add(vector);
}
}
foreach (var vectorAndGroupPath in encodingResult.IdealVectorSet.Rejected)
{
var vector = vectorAndGroupPath.Key;
var groupPath = vectorAndGroupPath.Value;
var groupIndex = groupCache.GetGroupIndex(vector);
var rejected = classifier.IsRejected(vector, groupIndex);
var accepted = classifier.IsAccepted(vector, groupIndex);
if (!trainingSet.Accepted.ContainsKey(vector) &&
!trainingSet.Rejected.ContainsKey(vector))
{
var target = new SuspiciousNode {
Vector = vector,
GroupKey = groupPath,
Used = false,
};
if (accepted)
{
if (!rejected)
{
acceptAccept[groupIndex].Add(target);
}
else
{
acceptReject[groupIndex].Add(target);
}
}
else
{
if (!rejected)
{
rejectAccept[groupIndex].Add(target);
}
else
{
rejectReject[groupIndex].Add(target);
}
}
}
if (!accepted)
{
correctlyRejected++;
}
else if (!rejected)
{
wronglyAccepted++;
wrongNodeCount += encodingResult.Vector2Count[vector];
wronglyAcceptedFeatures.Add(vector);
}
else
{
correctlyRejectedInRejecting++;
}
}
Console.WriteLine("done");
Console.WriteLine(
"WA: " + wronglyAccepted + ", WR: " + wronglyRejected + "/"
+ wronglyRejectedInRejecting + ", CA: "
+ correctlyAccepted + ", CR: " + correctlyRejected + "/"
+ correctlyRejectedInRejecting);
Console.WriteLine(
"TR: "
+ trainingSet.Count + ", AF: "
+ String.Join(", ", classifier.CountAcceptingFeatures())
+ ", RF: "
+ String.Join(", ", classifier.CountRejectingFeatures()));
var wrongFeatureCount = wronglyAccepted + wronglyRejected
+ wronglyRejectedInRejecting;
var selector = new SuspiciousNodeSelector(classifier.AcceptingFeatureBitMask,
classifier.RejectingFeatureBitMask);
var suspiciousNodes = selector.SelectSuspiciousNodes(count, classifier, acceptAccept,
acceptReject, rejectAccept, rejectReject);
return(new ClassificationResult(suspiciousNodes, wronglyAcceptedFeatures,
wronglyRejectedFeatures,
wrongFeatureCount, wrongNodeCount, encodingResult));
}
/// <summary>
/// RejectedVectorが持つAcceptingFeatuerを削除する。
/// </summary>
/// <param name="rejectedVectors"></param>
/// <param name="groupCache"></param>
public void OptimizeWeakly(ICollection<BigInteger> rejectedVectors, GroupCache groupCache) {
var usedRejectedFeatures =
Enumerable.Repeat(RejectingFeatureBitMask, Units.Count).ToList();
foreach (var rejectedVector in rejectedVectors) {
var groupIndex = groupCache.GetGroupIndex(rejectedVector);
usedRejectedFeatures[groupIndex] |= rejectedVector;
}
foreach (var unitAndFeature in Units.Zip(usedRejectedFeatures)) {
unitAndFeature.Item1.Accepting &= AllFeatureBitMask ^ unitAndFeature.Item2;
}
}
public ClassificationResult Classify(
int count, Classifier classifier, GroupCache groupCache,
EncodingResult encodingResult,
RevealedVectorSet trainingSet = null) {
trainingSet = trainingSet ?? new RevealedVectorSet();
var correctlyAccepted = 0;
var correctlyRejected = 0;
var wronglyAccepted = 0;
var wronglyRejected = 0;
var correctlyRejectedInRejecting = 0;
var wronglyRejectedInRejecting = 0;
var rejectAccept = new List<List<SuspiciousNode>>();
var rejectReject = new List<List<SuspiciousNode>>();
var acceptAccept = new List<List<SuspiciousNode>>();
var acceptReject = new List<List<SuspiciousNode>>();
for (int i = 0; i < classifier.GroupCount; i++) {
rejectAccept.Add(new List<SuspiciousNode>());
rejectReject.Add(new List<SuspiciousNode>());
acceptAccept.Add(new List<SuspiciousNode>());
acceptReject.Add(new List<SuspiciousNode>());
}
var wrongNodeCount = 0;
var wronglyRejectedFeatures = new List<BigInteger>();
var wronglyAcceptedFeatures = new List<BigInteger>();
foreach (var vectorAndGroupPath in encodingResult.IdealVectorSet.Accepted) {
var vector = vectorAndGroupPath.Key;
var groupPath = vectorAndGroupPath.Value;
var groupIndex = groupCache.GetGroupIndex(vector);
var rejected = classifier.IsRejected(vector, groupIndex);
var accepted = classifier.IsAccepted(vector, groupIndex);
if (!trainingSet.Accepted.ContainsKey(vector)
&& !trainingSet.Rejected.ContainsKey(vector)) {
var target = new SuspiciousNode {
Vector = vector,
GroupKey = groupPath,
Used = false,
};
if (accepted) {
if (!rejected) {
acceptAccept[groupIndex].Add(target);
} else {
acceptReject[groupIndex].Add(target);
}
} else {
if (!rejected) {
rejectAccept[groupIndex].Add(target);
} else {
rejectReject[groupIndex].Add(target);
}
}
}
if (!accepted) {
wronglyRejected++;
wrongNodeCount += encodingResult.Vector2Count[vector];
wronglyRejectedFeatures.Add(vector);
} else if (!rejected) {
correctlyAccepted++;
} else {
wronglyRejectedInRejecting++;
wrongNodeCount += encodingResult.Vector2Count[vector];
wronglyRejectedFeatures.Add(vector);
}
}
foreach (var vectorAndGroupPath in encodingResult.IdealVectorSet.Rejected) {
var vector = vectorAndGroupPath.Key;
var groupPath = vectorAndGroupPath.Value;
var groupIndex = groupCache.GetGroupIndex(vector);
var rejected = classifier.IsRejected(vector, groupIndex);
var accepted = classifier.IsAccepted(vector, groupIndex);
if (!trainingSet.Accepted.ContainsKey(vector)
&& !trainingSet.Rejected.ContainsKey(vector)) {
var target = new SuspiciousNode {
Vector = vector,
GroupKey = groupPath,
Used = false,
};
if (accepted) {
if (!rejected) {
acceptAccept[groupIndex].Add(target);
} else {
acceptReject[groupIndex].Add(target);
}
} else {
if (!rejected) {
rejectAccept[groupIndex].Add(target);
} else {
rejectReject[groupIndex].Add(target);
}
}
}
if (!accepted) {
correctlyRejected++;
} else if (!rejected) {
wronglyAccepted++;
wrongNodeCount += encodingResult.Vector2Count[vector];
wronglyAcceptedFeatures.Add(vector);
} else {
correctlyRejectedInRejecting++;
}
}
Console.WriteLine("done");
Console.WriteLine(
"WA: " + wronglyAccepted + ", WR: " + wronglyRejected + "/"
+ wronglyRejectedInRejecting + ", CA: "
+ correctlyAccepted + ", CR: " + correctlyRejected + "/"
+ correctlyRejectedInRejecting);
Console.WriteLine(
"TR: "
+ trainingSet.Count + ", AF: "
+ String.Join(", ", classifier.CountAcceptingFeatures())
+ ", RF: "
+ String.Join(", ", classifier.CountRejectingFeatures()));
var wrongFeatureCount = wronglyAccepted + wronglyRejected
+ wronglyRejectedInRejecting;
var selector = new SuspiciousNodeSelector(classifier.AcceptingFeatureBitMask,
classifier.RejectingFeatureBitMask);
var suspiciousNodes = selector.SelectSuspiciousNodes(count, classifier, acceptAccept,
acceptReject, rejectAccept, rejectReject);
return new ClassificationResult(suspiciousNodes, wronglyAcceptedFeatures,
wronglyRejectedFeatures,
wrongFeatureCount, wrongNodeCount, encodingResult);
}
