public void Create(RevealedVectorSet trainingSet, GroupCache groupCache) { do { Initialize(); } while (!UpdateClassifierUnits(trainingSet.Accepted, trainingSet, groupCache)); }
private UpdateResult UpdateAcceptingClassifierUnits( IDictionary <BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet, GroupCache groupCache) { if (additionalAcceptedSet.Count == 0) { var rejectedGroupInfo = FindWronglyAcceptedGroupInfo(trainingSet, groupCache); if (rejectedGroupInfo != null) { return(new UpdateResult { Rejected = rejectedGroupInfo, Accepted = null, }); } return(null); } foreach (var vectorAndGroupPath in additionalAcceptedSet) { var vector = vectorAndGroupPath.Key; var groupIndex = groupCache.GetGroupIndex(vector); Units[groupIndex].Accepting &= vector; var rejectedGroupInfo = FindWronglyAcceptedGroupInfo(trainingSet, groupCache); if (rejectedGroupInfo != null) { return(new UpdateResult { Rejected = rejectedGroupInfo, Accepted = new GroupInfo { Index = groupIndex, Path = vectorAndGroupPath.Value }, }); } } return(null); }
/// <summary> /// 分類結果に影響を与えないacceptingFeatureを取り除く。 /// </summary> /// <param name="rejectedVectors"></param> /// <param name="groupCache"></param> public void Optimize(IEnumerable <BigInteger> rejectedVectors, GroupCache groupCache) { var groupIndex2RejectedVectors = new List <List <BigInteger> >(); for (int i = 0; i < Units.Count; i++) { groupIndex2RejectedVectors.Add(new List <BigInteger>()); } foreach (var rejectedVector in rejectedVectors) { var groupIndex = groupCache.GetGroupIndex(rejectedVector); var acceptingClassifier = Units[groupIndex].Accepting; var rejectingClassifier = Units[groupIndex].Rejecting; var accepted = (rejectedVector & acceptingClassifier) == acceptingClassifier; var rejected = (rejectedVector & rejectingClassifier) != BigInteger.Zero; if (!accepted /*&& !rejected*/) { groupIndex2RejectedVectors[groupIndex].Add(rejectedVector); } } foreach (var unitAndVectors in Units.Zip(groupIndex2RejectedVectors)) { var unit = unitAndVectors.Item1; var vectors = unitAndVectors.Item2; var bit = BigInteger.One; for (int i = 0; i < AcceptingFeatureCount; i++) { if ((unit.Accepting & bit) != 0) { unit.Accepting ^= bit; var acceptingClassifier = unit.Accepting; var removable = false; foreach (var rejectedVector in vectors) { var accepted = (rejectedVector & acceptingClassifier) == acceptingClassifier; // 全てが (rejectedVector & bit) == 0 なら削除しない // 一つでも (rejectedVector & bit) != 0 なら削除できる removable |= (rejectedVector & bit) != 0; if (accepted) { removable = false; break; } } if (!removable) { unit.Accepting ^= bit; } } bit <<= 1; } } }
public bool Update( IDictionary <BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet, GroupCache groupCache) { var originalClassifierUnit = Units.Select(c => c.Clone()).ToImmutableList(); if (!UpdateClassifierUnits(additionalAcceptedSet, trainingSet, groupCache)) { Create(trainingSet, groupCache); } return(!Units.SequenceEqual(originalClassifierUnit)); }
/// <summary> /// RejectedVectorが持つAcceptingFeatuerを削除する。 /// </summary> /// <param name="rejectedVectors"></param> /// <param name="groupCache"></param> public void OptimizeWeakly(ICollection <BigInteger> rejectedVectors, GroupCache groupCache) { var usedRejectedFeatures = Enumerable.Repeat(RejectingFeatureBitMask, Units.Count).ToList(); foreach (var rejectedVector in rejectedVectors) { var groupIndex = groupCache.GetGroupIndex(rejectedVector); usedRejectedFeatures[groupIndex] |= rejectedVector; } foreach (var unitAndFeature in Units.Zip(usedRejectedFeatures)) { unitAndFeature.Item1.Accepting &= AllFeatureBitMask ^ unitAndFeature.Item2; } }
public ClassificationResult Apply( ICollection<string> codePaths, string searchPattern, FeatureEncoder featureEncoder, Classifier classifier, StreamWriter writer = null) { var allCsts = GenerateValidCsts(codePaths); var encodingResult = featureEncoder.Encode(codePaths, allCsts, this); var time = Environment.TickCount; var groupCache = new GroupCache(encodingResult, classifier); var result = Classify(int.MaxValue, classifier, groupCache, encodingResult); Console.WriteLine("Time: " + (Environment.TickCount - time)); if (writer != null) { encodingResult.WriteResult(writer); } return result; }
private GroupInfo FindWronglyAcceptedGroupInfo( RevealedVectorSet trainingSet, GroupCache groupCache) { foreach (var featureAndGroupPath in trainingSet.Rejected) { var feature = featureAndGroupPath.Key; var groupPath = featureAndGroupPath.Value; var groupIndex = groupCache.GetGroupIndex(feature); var accepted = IsAccepted(feature, groupIndex) && !IsRejected(feature, groupIndex); if (accepted) { return(new GroupInfo { Index = groupIndex, Path = groupPath }); } } return(null); }
/// <summary> /// 分類結果に影響を与えないacceptingFeatureを取り除く。 /// </summary> /// <param name="rejectedVectors"></param> /// <param name="groupCache"></param> public void Optimize(IEnumerable<BigInteger> rejectedVectors, GroupCache groupCache) { var groupIndex2RejectedVectors = new List<List<BigInteger>>(); for (int i = 0; i < Units.Count; i++) { groupIndex2RejectedVectors.Add(new List<BigInteger>()); } foreach (var rejectedVector in rejectedVectors) { var groupIndex = groupCache.GetGroupIndex(rejectedVector); var acceptingClassifier = Units[groupIndex].Accepting; var rejectingClassifier = Units[groupIndex].Rejecting; var accepted = (rejectedVector & acceptingClassifier) == acceptingClassifier; var rejected = (rejectedVector & rejectingClassifier) != BigInteger.Zero; if (!accepted /*&& !rejected*/) { groupIndex2RejectedVectors[groupIndex].Add(rejectedVector); } } foreach (var unitAndVectors in Units.Zip(groupIndex2RejectedVectors)) { var unit = unitAndVectors.Item1; var vectors = unitAndVectors.Item2; var bit = BigInteger.One; for (int i = 0; i < AcceptingFeatureCount; i++) { if ((unit.Accepting & bit) != 0) { unit.Accepting ^= bit; var acceptingClassifier = unit.Accepting; var removable = false; foreach (var rejectedVector in vectors) { var accepted = (rejectedVector & acceptingClassifier) == acceptingClassifier; // 全てが (rejectedVector & bit) == 0 なら削除しない // 一つでも (rejectedVector & bit) != 0 なら削除できる removable |= (rejectedVector & bit) != 0; if (accepted) { removable = false; break; } } if (!removable) { unit.Accepting ^= bit; } } bit <<= 1; } } }
public ClassificationResult Apply( ICollection <string> codePaths, string searchPattern, FeatureEncoder featureEncoder, Classifier classifier, StreamWriter writer = null) { var allCsts = GenerateValidCsts(codePaths); var encodingResult = featureEncoder.Encode(codePaths, allCsts, this); var time = Environment.TickCount; var groupCache = new GroupCache(encodingResult, classifier); var result = Classify(int.MaxValue, classifier, groupCache, encodingResult); Console.WriteLine("Time: " + (Environment.TickCount - time)); if (writer != null) { encodingResult.WriteResult(writer); } return(result); }
private void UpdateRejectingClassifierUnits( IEnumerable <BigInteger> acceptedVectors, GroupCache groupCache) { var count = Units.Count; for (int i = 0; i < count; i++) { Units[i].Rejecting ^= RejectingFeatureBitMask; } foreach (var vector in acceptedVectors) { var iGroupKey = groupCache.GetGroupIndex(vector); Units[iGroupKey].Rejecting |= vector; } for (int i = 0; i < count; i++) { Units[i].Rejecting ^= RejectingFeatureBitMask; Units[i].Rejecting &= RejectingFeatureBitMask; } }
private bool UpdateClassifierUnits( IDictionary <BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet, GroupCache groupCache) { UpdateRejectingClassifierUnits(additionalAcceptedSet.Keys, groupCache); var result = UpdateAcceptingClassifierUnits(additionalAcceptedSet, trainingSet, groupCache); if (result == null) { return(true); } var count = GroupPaths.Count; var updated = false; if (result.Accepted != null && result.Accepted.Path != GroupPaths[result.Accepted.Index]) { updated |= AddNewGroup(result.Accepted); } if (result.Rejected.Path != GroupPaths[result.Rejected.Index]) { updated |= AddNewGroup(result.Rejected); } if (updated) { groupCache.UpdateGroupCache(this); } // ReSharper disable once PossibleUnintendedReferenceComparison if (GroupPaths.Count == count && additionalAcceptedSet == trainingSet.Accepted) { throw new Exception("Fail to learn rules"); } Console.WriteLine("Failed to create Classifier units."); Console.WriteLine("Groups: " + GroupPaths.Count + " (" + count + ")"); return(false); }
private void UpdateRejectingClassifierUnits( IEnumerable<BigInteger> acceptedVectors, GroupCache groupCache) { var count = Units.Count; for (int i = 0; i < count; i++) { Units[i].Rejecting ^= RejectingFeatureBitMask; } foreach (var vector in acceptedVectors) { var iGroupKey = groupCache.GetGroupIndex(vector); Units[iGroupKey].Rejecting |= vector; } for (int i = 0; i < count; i++) { Units[i].Rejecting ^= RejectingFeatureBitMask; Units[i].Rejecting &= RejectingFeatureBitMask; } }
private bool UpdateClassifierUnits( IDictionary<BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet, GroupCache groupCache) { UpdateRejectingClassifierUnits(additionalAcceptedSet.Keys, groupCache); var result = UpdateAcceptingClassifierUnits(additionalAcceptedSet, trainingSet, groupCache); if (result == null) { return true; } var count = GroupPaths.Count; var updated = false; if (result.Accepted != null && result.Accepted.Path != GroupPaths[result.Accepted.Index]) { updated |= AddNewGroup(result.Accepted); } if (result.Rejected.Path != GroupPaths[result.Rejected.Index]) { updated |= AddNewGroup(result.Rejected); } if (updated) { groupCache.UpdateGroupCache(this); } // ReSharper disable once PossibleUnintendedReferenceComparison if (GroupPaths.Count == count && additionalAcceptedSet == trainingSet.Accepted) { throw new Exception("Fail to learn rules"); } Console.WriteLine("Failed to create Classifier units."); Console.WriteLine("Groups: " + GroupPaths.Count + " (" + count + ")"); return false; }
public bool Update( IDictionary<BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet, GroupCache groupCache) { var originalClassifierUnit = Units.Select(c => c.Clone()).ToImmutableList(); if (!UpdateClassifierUnits(additionalAcceptedSet, trainingSet, groupCache)) { Create(trainingSet, groupCache); } return !Units.SequenceEqual(originalClassifierUnit); }
public LearningResult Learn( ICollection <string> seedPaths, ICollection <string> codePaths, string searchPattern, StreamWriter writer = null) { var allCsts = GenerateValidCsts(codePaths); var seedCsts = GenerateValidCsts(seedPaths).ToList(); var seedNodes = seedCsts .SelectMany( cst => LearningExperimentUtil.GetUppermostNodesByNames(cst, OracleNames)) .Where(ProtectedIsAcceptedUsingOracle) .ToList(); var seedCst = seedCsts.First(); var seedCode = seedCst.Code; var structuredCode = new StructuredCode(seedCode); var acceptingFragments = ConstructAcceptingFragments(structuredCode, seedCst, seedNodes); var rejectingFragments = ConstructRejectingFragments(structuredCode, seedCst); SeedNodeSet.Create(acceptingFragments, this); var preparingTime = Environment.TickCount; var extractor = CreateExtractor(); var seedNodeSet = new SeedNodeSet(acceptingFragments.Select(f => f.Node), seedCsts, this); Console.WriteLine("#Accepted seed nodes: " + seedNodeSet.AcceptedNodes.Count + " (" + acceptingFragments.Count + ")"); Console.WriteLine("#Rejected seed nodes: " + seedNodeSet.RejectedNodes.Count + " (" + rejectingFragments.Count + ")"); var featureSet = new FeatuerSet(seedNodeSet, extractor, acceptingFragments, rejectingFragments); var groupPaths = seedNodeSet.SelectedNodeNames.Select(n => ">" + n + ">"); var classifier = new Classifier(groupPaths, featureSet); Console.WriteLine( "#Features: " + featureSet.AcceptingFeatureCount + ", " + featureSet.RejectingFeatureCount); Console.WriteLine("Inner: " + extractor.IsInner); var featureEncoder = new FeatureEncoder(seedNodeSet.SelectedNodeNames, extractor, featureSet); var encodingResult = featureEncoder.Encode(codePaths, allCsts, this, seedNodeSet); Console.WriteLine("#Unique Elements: " + encodingResult.VectorCount); if (encodingResult.IdealAcceptedVector2GroupPath.Keys.ToHashSet() .Overlaps(encodingResult.IdealRejectedVector2GroupPath.Keys.ToHashSet())) { var others = encodingResult.IdealRejectedVector2GroupPath; var vector = encodingResult.IdealAcceptedVector2GroupPath.Keys.First(others.ContainsKey); foreach (var featureString in featureEncoder.GetFeatureStringsByVector(vector)) { Console.WriteLine(Experiment.Beautify(featureString)); } throw new Exception("Master predicates can't classify elements!"); } var groupCache = new GroupCache(encodingResult, classifier); var trainingSet = encodingResult.CreateTrainingVectorSet(); classifier.Create(trainingSet, groupCache); Experiment.WriteFeatureStrings(Console.Out, classifier, featureEncoder); Console.WriteLine("Preparing time: " + (Environment.TickCount - preparingTime)); var count = 0; var sumTime = Environment.TickCount; ClassificationResult classificationResult; while (true) { var time = Environment.TickCount; classificationResult = Classify(count, classifier, groupCache, encodingResult, trainingSet); if (classificationResult.SuspiciousNodes == null) { break; } var additionalAcceptedSet = RevealSuspiciousElements( encodingResult.IdealAcceptedVector2GroupPath.Keys, classificationResult.SuspiciousNodes, encodingResult, trainingSet); if (!classifier.Update(additionalAcceptedSet, trainingSet, groupCache)) { count++; } else { count = 0; } Console.WriteLine("Time: " + (Environment.TickCount - time)); } classifier.MakeImmutable(); Console.WriteLine(); Console.WriteLine("Sum time: " + (Environment.TickCount - sumTime)); var trainingVectorCount = trainingSet.Count; var idealVectorCount = encodingResult.IdealVectorSet.Count; Console.WriteLine("#Required vectors: " + trainingVectorCount + " / " + idealVectorCount); if (writer != null) { encodingResult.WriteResult(writer, trainingSet); } foreach (var groupPath in classifier.GroupPaths) { Console.WriteLine(groupPath); } classifier.Optimize(encodingResult.IdealRejectedVector2GroupPath.Keys, groupCache); return(new LearningResult { ClassificationResult = classificationResult, Classifier = classifier, EncodingResult = encodingResult, FeatureEncoder = featureEncoder, }); }
public LearningResult Learn( ICollection<string> seedPaths, ICollection<string> codePaths, string searchPattern, StreamWriter writer = null) { var allCsts = GenerateValidCsts(codePaths); var seedCsts = GenerateValidCsts(seedPaths).ToList(); var seedNodes = seedCsts .SelectMany( cst => LearningExperimentUtil.GetUppermostNodesByNames(cst, OracleNames)) .Where(ProtectedIsAcceptedUsingOracle) .ToList(); var seedCst = seedCsts.First(); var seedCode = seedCst.Code; var structuredCode = new StructuredCode(seedCode); var acceptingFragments = ConstructAcceptingFragments(structuredCode, seedCst, seedNodes); var rejectingFragments = ConstructRejectingFragments(structuredCode, seedCst); SeedNodeSet.Create(acceptingFragments, this); var preparingTime = Environment.TickCount; var extractor = CreateExtractor(); var seedNodeSet = new SeedNodeSet(acceptingFragments.Select(f => f.Node), seedCsts, this); Console.WriteLine("#Accepted seed nodes: " + seedNodeSet.AcceptedNodes.Count + " (" + acceptingFragments.Count + ")"); Console.WriteLine("#Rejected seed nodes: " + seedNodeSet.RejectedNodes.Count + " (" + rejectingFragments.Count + ")"); var featureSet = new FeatuerSet(seedNodeSet, extractor, acceptingFragments, rejectingFragments); var groupPaths = seedNodeSet.SelectedNodeNames.Select(n => ">" + n + ">"); var classifier = new Classifier(groupPaths, featureSet); Console.WriteLine( "#Features: " + featureSet.AcceptingFeatureCount + ", " + featureSet.RejectingFeatureCount); Console.WriteLine("Inner: " + extractor.IsInner); var featureEncoder = new FeatureEncoder(seedNodeSet.SelectedNodeNames, extractor, featureSet); var encodingResult = featureEncoder.Encode(codePaths, allCsts, this, seedNodeSet); Console.WriteLine("#Unique Elements: " + encodingResult.VectorCount); if (encodingResult.IdealAcceptedVector2GroupPath.Keys.ToHashSet() .Overlaps(encodingResult.IdealRejectedVector2GroupPath.Keys.ToHashSet())) { var others = encodingResult.IdealRejectedVector2GroupPath; var vector = encodingResult.IdealAcceptedVector2GroupPath.Keys.First(others.ContainsKey); foreach (var featureString in featureEncoder.GetFeatureStringsByVector(vector)) { Console.WriteLine(Experiment.Beautify(featureString)); } throw new Exception("Master predicates can't classify elements!"); } var groupCache = new GroupCache(encodingResult, classifier); var trainingSet = encodingResult.CreateTrainingVectorSet(); classifier.Create(trainingSet, groupCache); Experiment.WriteFeatureStrings(Console.Out, classifier, featureEncoder); Console.WriteLine("Preparing time: " + (Environment.TickCount - preparingTime)); var count = 0; var sumTime = Environment.TickCount; ClassificationResult classificationResult; while (true) { var time = Environment.TickCount; classificationResult = Classify(count, classifier, groupCache, encodingResult, trainingSet); if (classificationResult.SuspiciousNodes == null) { break; } var additionalAcceptedSet = RevealSuspiciousElements( encodingResult.IdealAcceptedVector2GroupPath.Keys, classificationResult.SuspiciousNodes, encodingResult, trainingSet); if (!classifier.Update(additionalAcceptedSet, trainingSet, groupCache)) { count++; } else { count = 0; } Console.WriteLine("Time: " + (Environment.TickCount - time)); } classifier.MakeImmutable(); Console.WriteLine(); Console.WriteLine("Sum time: " + (Environment.TickCount - sumTime)); var trainingVectorCount = trainingSet.Count; var idealVectorCount = encodingResult.IdealVectorSet.Count; Console.WriteLine("#Required vectors: " + trainingVectorCount + " / " + idealVectorCount); if (writer != null) { encodingResult.WriteResult(writer, trainingSet); } foreach (var groupPath in classifier.GroupPaths) { Console.WriteLine(groupPath); } classifier.Optimize(encodingResult.IdealRejectedVector2GroupPath.Keys, groupCache); return new LearningResult { ClassificationResult = classificationResult, Classifier = classifier, EncodingResult = encodingResult, FeatureEncoder = featureEncoder, }; }
private UpdateResult UpdateAcceptingClassifierUnits( IDictionary<BigInteger, string> additionalAcceptedSet, RevealedVectorSet trainingSet, GroupCache groupCache) { if (additionalAcceptedSet.Count == 0) { var rejectedGroupInfo = FindWronglyAcceptedGroupInfo(trainingSet, groupCache); if (rejectedGroupInfo != null) { return new UpdateResult { Rejected = rejectedGroupInfo, Accepted = null, }; } return null; } foreach (var vectorAndGroupPath in additionalAcceptedSet) { var vector = vectorAndGroupPath.Key; var groupIndex = groupCache.GetGroupIndex(vector); Units[groupIndex].Accepting &= vector; var rejectedGroupInfo = FindWronglyAcceptedGroupInfo(trainingSet, groupCache); if (rejectedGroupInfo != null) { return new UpdateResult { Rejected = rejectedGroupInfo, Accepted = new GroupInfo { Index = groupIndex, Path = vectorAndGroupPath.Value }, }; } } return null; }
private GroupInfo FindWronglyAcceptedGroupInfo( RevealedVectorSet trainingSet, GroupCache groupCache) { foreach (var featureAndGroupPath in trainingSet.Rejected) { var feature = featureAndGroupPath.Key; var groupPath = featureAndGroupPath.Value; var groupIndex = groupCache.GetGroupIndex(feature); var accepted = IsAccepted(feature, groupIndex) && !IsRejected(feature, groupIndex); if (accepted) { return new GroupInfo { Index = groupIndex, Path = groupPath }; } } return null; }
public ClassificationResult Classify( int count, Classifier classifier, GroupCache groupCache, EncodingResult encodingResult, RevealedVectorSet trainingSet = null) { trainingSet = trainingSet ?? new RevealedVectorSet(); var correctlyAccepted = 0; var correctlyRejected = 0; var wronglyAccepted = 0; var wronglyRejected = 0; var correctlyRejectedInRejecting = 0; var wronglyRejectedInRejecting = 0; var rejectAccept = new List <List <SuspiciousNode> >(); var rejectReject = new List <List <SuspiciousNode> >(); var acceptAccept = new List <List <SuspiciousNode> >(); var acceptReject = new List <List <SuspiciousNode> >(); for (int i = 0; i < classifier.GroupCount; i++) { rejectAccept.Add(new List <SuspiciousNode>()); rejectReject.Add(new List <SuspiciousNode>()); acceptAccept.Add(new List <SuspiciousNode>()); acceptReject.Add(new List <SuspiciousNode>()); } var wrongNodeCount = 0; var wronglyRejectedFeatures = new List <BigInteger>(); var wronglyAcceptedFeatures = new List <BigInteger>(); foreach (var vectorAndGroupPath in encodingResult.IdealVectorSet.Accepted) { var vector = vectorAndGroupPath.Key; var groupPath = vectorAndGroupPath.Value; var groupIndex = groupCache.GetGroupIndex(vector); var rejected = classifier.IsRejected(vector, groupIndex); var accepted = classifier.IsAccepted(vector, groupIndex); if (!trainingSet.Accepted.ContainsKey(vector) && !trainingSet.Rejected.ContainsKey(vector)) { var target = new SuspiciousNode { Vector = vector, GroupKey = groupPath, Used = false, }; if (accepted) { if (!rejected) { acceptAccept[groupIndex].Add(target); } else { acceptReject[groupIndex].Add(target); } } else { if (!rejected) { rejectAccept[groupIndex].Add(target); } else { rejectReject[groupIndex].Add(target); } } } if (!accepted) { wronglyRejected++; wrongNodeCount += encodingResult.Vector2Count[vector]; wronglyRejectedFeatures.Add(vector); } else if (!rejected) { correctlyAccepted++; } else { wronglyRejectedInRejecting++; wrongNodeCount += encodingResult.Vector2Count[vector]; wronglyRejectedFeatures.Add(vector); } } foreach (var vectorAndGroupPath in encodingResult.IdealVectorSet.Rejected) { var vector = vectorAndGroupPath.Key; var groupPath = vectorAndGroupPath.Value; var groupIndex = groupCache.GetGroupIndex(vector); var rejected = classifier.IsRejected(vector, groupIndex); var accepted = classifier.IsAccepted(vector, groupIndex); if (!trainingSet.Accepted.ContainsKey(vector) && !trainingSet.Rejected.ContainsKey(vector)) { var target = new SuspiciousNode { Vector = vector, GroupKey = groupPath, Used = false, }; if (accepted) { if (!rejected) { acceptAccept[groupIndex].Add(target); } else { acceptReject[groupIndex].Add(target); } } else { if (!rejected) { rejectAccept[groupIndex].Add(target); } else { rejectReject[groupIndex].Add(target); } } } if (!accepted) { correctlyRejected++; } else if (!rejected) { wronglyAccepted++; wrongNodeCount += encodingResult.Vector2Count[vector]; wronglyAcceptedFeatures.Add(vector); } else { correctlyRejectedInRejecting++; } } Console.WriteLine("done"); Console.WriteLine( "WA: " + wronglyAccepted + ", WR: " + wronglyRejected + "/" + wronglyRejectedInRejecting + ", CA: " + correctlyAccepted + ", CR: " + correctlyRejected + "/" + correctlyRejectedInRejecting); Console.WriteLine( "TR: " + trainingSet.Count + ", AF: " + String.Join(", ", classifier.CountAcceptingFeatures()) + ", RF: " + String.Join(", ", classifier.CountRejectingFeatures())); var wrongFeatureCount = wronglyAccepted + wronglyRejected + wronglyRejectedInRejecting; var selector = new SuspiciousNodeSelector(classifier.AcceptingFeatureBitMask, classifier.RejectingFeatureBitMask); var suspiciousNodes = selector.SelectSuspiciousNodes(count, classifier, acceptAccept, acceptReject, rejectAccept, rejectReject); return(new ClassificationResult(suspiciousNodes, wronglyAcceptedFeatures, wronglyRejectedFeatures, wrongFeatureCount, wrongNodeCount, encodingResult)); }
/// <summary> /// RejectedVectorが持つAcceptingFeatuerを削除する。 /// </summary> /// <param name="rejectedVectors"></param> /// <param name="groupCache"></param> public void OptimizeWeakly(ICollection<BigInteger> rejectedVectors, GroupCache groupCache) { var usedRejectedFeatures = Enumerable.Repeat(RejectingFeatureBitMask, Units.Count).ToList(); foreach (var rejectedVector in rejectedVectors) { var groupIndex = groupCache.GetGroupIndex(rejectedVector); usedRejectedFeatures[groupIndex] |= rejectedVector; } foreach (var unitAndFeature in Units.Zip(usedRejectedFeatures)) { unitAndFeature.Item1.Accepting &= AllFeatureBitMask ^ unitAndFeature.Item2; } }
public ClassificationResult Classify( int count, Classifier classifier, GroupCache groupCache, EncodingResult encodingResult, RevealedVectorSet trainingSet = null) { trainingSet = trainingSet ?? new RevealedVectorSet(); var correctlyAccepted = 0; var correctlyRejected = 0; var wronglyAccepted = 0; var wronglyRejected = 0; var correctlyRejectedInRejecting = 0; var wronglyRejectedInRejecting = 0; var rejectAccept = new List<List<SuspiciousNode>>(); var rejectReject = new List<List<SuspiciousNode>>(); var acceptAccept = new List<List<SuspiciousNode>>(); var acceptReject = new List<List<SuspiciousNode>>(); for (int i = 0; i < classifier.GroupCount; i++) { rejectAccept.Add(new List<SuspiciousNode>()); rejectReject.Add(new List<SuspiciousNode>()); acceptAccept.Add(new List<SuspiciousNode>()); acceptReject.Add(new List<SuspiciousNode>()); } var wrongNodeCount = 0; var wronglyRejectedFeatures = new List<BigInteger>(); var wronglyAcceptedFeatures = new List<BigInteger>(); foreach (var vectorAndGroupPath in encodingResult.IdealVectorSet.Accepted) { var vector = vectorAndGroupPath.Key; var groupPath = vectorAndGroupPath.Value; var groupIndex = groupCache.GetGroupIndex(vector); var rejected = classifier.IsRejected(vector, groupIndex); var accepted = classifier.IsAccepted(vector, groupIndex); if (!trainingSet.Accepted.ContainsKey(vector) && !trainingSet.Rejected.ContainsKey(vector)) { var target = new SuspiciousNode { Vector = vector, GroupKey = groupPath, Used = false, }; if (accepted) { if (!rejected) { acceptAccept[groupIndex].Add(target); } else { acceptReject[groupIndex].Add(target); } } else { if (!rejected) { rejectAccept[groupIndex].Add(target); } else { rejectReject[groupIndex].Add(target); } } } if (!accepted) { wronglyRejected++; wrongNodeCount += encodingResult.Vector2Count[vector]; wronglyRejectedFeatures.Add(vector); } else if (!rejected) { correctlyAccepted++; } else { wronglyRejectedInRejecting++; wrongNodeCount += encodingResult.Vector2Count[vector]; wronglyRejectedFeatures.Add(vector); } } foreach (var vectorAndGroupPath in encodingResult.IdealVectorSet.Rejected) { var vector = vectorAndGroupPath.Key; var groupPath = vectorAndGroupPath.Value; var groupIndex = groupCache.GetGroupIndex(vector); var rejected = classifier.IsRejected(vector, groupIndex); var accepted = classifier.IsAccepted(vector, groupIndex); if (!trainingSet.Accepted.ContainsKey(vector) && !trainingSet.Rejected.ContainsKey(vector)) { var target = new SuspiciousNode { Vector = vector, GroupKey = groupPath, Used = false, }; if (accepted) { if (!rejected) { acceptAccept[groupIndex].Add(target); } else { acceptReject[groupIndex].Add(target); } } else { if (!rejected) { rejectAccept[groupIndex].Add(target); } else { rejectReject[groupIndex].Add(target); } } } if (!accepted) { correctlyRejected++; } else if (!rejected) { wronglyAccepted++; wrongNodeCount += encodingResult.Vector2Count[vector]; wronglyAcceptedFeatures.Add(vector); } else { correctlyRejectedInRejecting++; } } Console.WriteLine("done"); Console.WriteLine( "WA: " + wronglyAccepted + ", WR: " + wronglyRejected + "/" + wronglyRejectedInRejecting + ", CA: " + correctlyAccepted + ", CR: " + correctlyRejected + "/" + correctlyRejectedInRejecting); Console.WriteLine( "TR: " + trainingSet.Count + ", AF: " + String.Join(", ", classifier.CountAcceptingFeatures()) + ", RF: " + String.Join(", ", classifier.CountRejectingFeatures())); var wrongFeatureCount = wronglyAccepted + wronglyRejected + wronglyRejectedInRejecting; var selector = new SuspiciousNodeSelector(classifier.AcceptingFeatureBitMask, classifier.RejectingFeatureBitMask); var suspiciousNodes = selector.SelectSuspiciousNodes(count, classifier, acceptAccept, acceptReject, rejectAccept, rejectReject); return new ClassificationResult(suspiciousNodes, wronglyAcceptedFeatures, wronglyRejectedFeatures, wrongFeatureCount, wrongNodeCount, encodingResult); }