static OverallDataAccessor()
{
allClusters = AllResults.Load(ConfigurationManager.AppSettings["OverAllKnnGraphBin"]);
allImages = allClusters.ImageEncoding.Reverse();
allPatches = allClusters.PatchEncoding.Reverse();
distanceLookup = allClusters.Rows.GroupBy(x => Tuple.Create <int, int>(x.Query.ImageId, x.Query.PatchId)).ToDictionary(x => x.Key, x => x.First());
}
private static void Print(string newName, AllResults loadedFile)
{
using (var sw = new StreamWriter(newName.Replace(".bin", ".html"), append: false))
{
loadedFile.Render(sw);
}
}
public void RunResultsAreImmutable()
{
// Create run results.
var runResults = new Dictionary <ImmutableGenome, IDictionary <TestInstance, TestResult> >();
var genome1 = new ImmutableGenome(new Genome());
var instance1 = new TestInstance("1");
var results1 = new Dictionary <TestInstance, TestResult> {
{ instance1, new TestResult(1) }
};
runResults.Add(genome1, results1);
// Create message out of them.
var resultMessage = new AllResults <TestInstance, TestResult>(runResults);
Assert.True(runResults.Keys.SequenceEqual(resultMessage.RunResults.Keys));
Assert.True(resultMessage.RunResults[genome1].SequenceEqual(results1));
// Add more results to original dictionary.
results1.Add(new TestInstance("2"), new TestResult());
// Make sure that is not reflected in message.
Assert.False(resultMessage.RunResults[genome1].SequenceEqual(results1));
// Remove all genome results from original results.
runResults.Remove(genome1);
// Make sure that is not reflected in message.
Assert.False(runResults.Keys.SequenceEqual(resultMessage.RunResults.Keys));
}
/// <summary>
/// The GlobalInfoSummery.
/// </summary>
/// <param name="document">The document<see cref="IParentNode"/>.</param>
/// <returns>The <see cref="AllResults"/>.</returns>
private static AllResults GlobalInfoSummery(IDocument document)
{
var selectorAll = document.GetElementsByClassName(MainCounterSelector);// .QuerySelectorAll(".maincounter-number > span");
var result = new AllResults();
for (var i = 0; i < selectorAll.Length; i++)
{
int.TryParse((selectorAll[i].QuerySelector("span").TextContent?.Trim() ?? "0").Replace(",", ""),
out var count);
switch (i)
{
case 0:
result.Cases = count;
break;
case 1:
result.Deaths = count;
break;
default:
result.Recovered = count;
break;
}
}
return(result);
}
private static SearchHit NameToHit(string name, AllResults results)
{
var tokens = name.Split(Delimiters, StringSplitOptions.RemoveEmptyEntries);
var p = NameToPatch(results, tokens);
return(new SearchHit {
Hit = p, Distance = Single.Parse(tokens.Last(), FloatParser)
});
}
private void UpdateForceOutput()
{
List <WallPointResult> cr = null;
if (AllResults != null)
{
if (ReportingPierPointLocation != PierPointLocation.Both)
{
//foreach (var story in SelectedStories)
//{
if (SelectedStory != null & SelectedCombo != null)
{
cr = AllResults.Where(k => k.ComboName == SelectedCombo)
.SelectMany(r => r.PierForces
.Where(f => f.PierName == SelectedPier)
.Where(b => b.Result.PierPointLocation == ReportingPierPointLocation)
.Where(b => b.Result.StoryName == _SelectedStory)
.Select(a => a.Result)).ToList();
}
//}
}
else
{
//foreach (var story in SelectedStories)
//{
if (SelectedStory != null & SelectedCombo != null)
{
cr = AllResults.Where(k => k.ComboName == SelectedCombo)
.SelectMany(r => r.PierForces
.Where(f => f.PierName == SelectedPier)
.Where(b => b.Result.StoryName == _SelectedStory)
.Select(a => a.Result)).ToList();
}
//}
}
if (cr != null)
{
if (cr.Count > 0)
{
V_major_max = cr.Select(r => r.V2_Max).Max();
V_major_min = cr.Select(r => r.V2_Min).Min();
M_major_max = cr.Select(r => r.M3_Max).Max();
M_major_min = cr.Select(r => r.M3_Min).Min();
P_max = cr.Select(r => r.P_Max).Max();
P_min = cr.Select(r => r.P_Min).Min();
V_minor_max = cr.Select(r => r.V3_Max).Max();
V_minor_min = cr.Select(r => r.V3_Min).Min();
M_minor_max = cr.Select(r => r.M2_Max).Max();
M_minor_min = cr.Select(r => r.M2_Min).Min();
}
}
}
}
private static void GraphComponentDecomposition(string path)
{
var loaded = AllResults.Load(path);
var artificalResults = GroupIntoClusters(loaded);
artificalResults.Save(path.Replace(".bin", "-patchClusters.bin"));
using (var sw = new StreamWriter(path.Replace(".bin", "-patchLevelClustersReduced.html"), append: false))
{
artificalResults.Render(sw);
}
}
public static AllResults GroupIntoClusters(AllResults loaded, int imagePerCandidateMaxTreshold = 2000, int imagesPerCandidateMinTreshold = 4)
{
imgColors.Clear();
totalColors = 0;
var directional = loaded.Rows.SelectMany(x => x.Hits.Select(y => new { x.Query, y.Hit })).Distinct().ToList();
neighbours = directional.Union(directional.Select(a => new { Query = a.Hit, Hit = a.Query }))
.ToLookup(x => x.Query, x => x.Hit);
stack = new Stack <Patch>(neighbours.Select(x => x.Key));
while (stack.Count > 0)
{
Visit(stack.Pop());
}
var stats = imgColors.GroupBy(x => x.Value)
.Where(x => x.Select(z => z.Key.ImageId).Distinct().Count() < imagePerCandidateMaxTreshold &&
x.Select(y => y.Key.ImageId).Distinct().Count() >= imagesPerCandidateMinTreshold);
Console.WriteLine("Having " + stats.Count() + " nice clusters");
var rareImgs = new HashSet <Patch>(stats.SelectMany(x => x.Select(y => y.Key)));
var artificalResults = new AllResults {
ImageEncoding = loaded.ImageEncoding, PatchEncoding = loaded.PatchEncoding
};
var colorGroups = loaded.Rows
.Where(r => rareImgs.Contains(r.Query))
.GroupBy(x => imgColors[x.Query]);
foreach (var g in colorGroups)
{
artificalResults.Rows.Add(new ResultsRow
{
Query = g.First().Query,
Hits = g.SelectMany(x => x.Hits.Concat(new[] { new SearchHit {
Hit = x.Query, Distance = g.Max(colorGroup => colorGroup.Hits.Min(h => h.Distance))
} }))
.GroupBy(x => x.Hit)
.Select(x => new SearchHit {
Hit = x.Key, Distance = x.Min(y => y.Distance)
})
.Distinct()
.ToArray()
});
}
artificalResults.RefreshReferenceMap();
return(artificalResults);
}
//private void RefreshEtabsData()
//{
// ErrorMessage = "";
// try
// {
// ETABSModelManager manager = new ETABSModelManager();
// List<string> comboNames = manager.GetModelComboNames();
// AvaliableCombos = new ObservableCollection<string>(comboNames);
// List<string> PierNames = manager.GetModelPierNames();
// AvaliablePiers = new ObservableCollection<string>(PierNames);
// List<string> StoryNames = manager.GetModelStoryNames();
// AvaliableStories = new ObservableCollection<string>(StoryNames);
// }
// catch (Exception)
// {
// ErrorMessage = "Could not connect to ETABS model.";
// }
//}
private void UpdateStoryList()
{
if (SelectedPier != null)
{
if (AllResults != null)
{
if (AvaliableCombos.Count > 0)
{
var resultsForCurrentPier = AllResults.Where(k => k.ComboName == AvaliableCombos[0]).SelectMany(r => r.PierForces.Where(f => f.PierName == SelectedPier).Select(a => a.Result.StoryName)).Distinct().ToList();
AvaliableStories = new ObservableCollection <string>(resultsForCurrentPier);
}
}
}
}
private static string Filter(AllResults loadedFile, string file)
{
for (int i = 0; i < 31; i++)
{
Console.WriteLine($"iteration {i} starting");
loadedFile.RefreshReferenceMap();
loadedFile.RefBasedShrink();
}
var newName = file.Replace(".bin", "-refShrink" + 30 + ".bin");
loadedFile.Save(newName);
return(newName);
}
public static void CreateProtobufFile()
{
Console.WriteLine("Enter name of .csv file");
var path = Console.ReadLine();
Console.WriteLine("Enter number of items to fetch");
var topN = Int32.Parse(Console.ReadLine() ?? "50");
Console.WriteLine("Enter number of neighbours to show");
var localTake = Int32.Parse(Console.ReadLine() ?? "100");
var parseInfo = new AllResults();
int rowsProcessed = 0;
foreach (var line in File.ReadLines(path).Take(topN))
{
var parts = line.Split(';');
var query = parts.Take(1);
var hits = parts
.Skip(1)
.Take(localTake)
.Skip(1)
.Select <string, SearchHit>(x => NameToHit(x, parseInfo))
.ToArray();
var queryObject = NameToPatch(query.First(), parseInfo);
parseInfo.Rows.Add(new ResultsRow {
Hits = hits, Query = queryObject
});
if (rowsProcessed++ % 1000 == 0)
{
Console.WriteLine(rowsProcessed);
}
}
parseInfo.Rows.Sort((first, second) => first.Query.CompareTo(second.Query));
Console.WriteLine(
$"Images = {parseInfo.ImageEncoding.Count}, Patches = {parseInfo.PatchEncoding.Count}, Rows = {parseInfo.Rows.Count}");
var name = Path.GetFileNameWithoutExtension(path) + ".bin";
parseInfo.Save(name);
Console.WriteLine("Done, press enter");
Console.ReadLine();
}
private static void RenderData()
{
var protobufFolder = @"G:\siret\zoot\protobuf";
foreach (var file in Directory.GetFiles(protobufFolder, "*conv4*.bin"))
{
Console.WriteLine(file + " started");
var loadedFile = AllResults.Load(file);
var newName = Filter(loadedFile, file);
Print(newName, loadedFile);
Console.WriteLine($"{file} is done now");
}
}
private static void SparkMasketBasketParsing()
{
Console.WriteLine("Enter pattern of Spark .txt file(s)");
var lines = Directory.EnumerateFiles(@"G:\siret\spark-out\rules_conv_5-occur_5.0-conf_0.1", "part*")
.SelectMany(File.ReadLines);
var results = SparkResults.Parse(lines);
var loadedNameMapping = AllResults.Load(@"G:\siret\zoot\protobuf\local-conv5-cleaned-shrinked.bin").ImageEncoding;
using (var sw = new StreamWriter(@"G:\siret\spark-viz\market-basket-conv5-large-filtered.html"))
{
results.Print(sw, loadedNameMapping.ToDictionary(x => x.Value, x => x.Key),
r => r.Input.Length < 4 && r.Input.Length > 1);
}
Console.WriteLine("Printed. Pres enter...");
Console.ReadLine();
}
void StartNewSearch()
{
lock (SearchStateLock)
{
lock (NewResults)
{
NewResults.Clear();
AllResults.Clear();
IsArchiveNamesComplete = false;
IsFolderNamesComplete = false;
if (OnNewSearch != null)
{
OnNewSearch();
}
}
}
}
public Test_result(IMatrixReadStorage<int, EmptyData> storage, IMatrixReadStorage<int, SolutionData> datastorage, IAlgorithm<int> algorithm, PartitioningParameters parameters)
{
results = new AllResults();
InitializeComponent();
foreach (var matrixData in storage.ReadMatrixes())
{
string name = matrixData.Name;
IMatrix<int> matrix = matrixData.Matrix;
int n = matrixData.Matrix.Size(0);
int m = matrixData.Matrix.Size(1);
int M1 = parameters[0];
int M2 = parameters[1];
Int64 start = Stopwatch.GetTimestamp();
ISolution solution = algorithm.Run(matrix);
double time = (Stopwatch.GetTimestamp() - start) / (double)Stopwatch.Frequency;
double crit = MinMaxCriterium.Calculate(matrix, solution);
double w = Utilities.W(matrix, parameters);
double diff = crit - w;
results.AddElem(new Note(name, n, m, M1, M2, crit, diff, 0, time, false));
}
foreach (var matrixData in datastorage.ReadMatrixes())
{
string name = matrixData.Name;
IMatrix<int> matrix = matrixData.Matrix;
int n = matrixData.Matrix.Size(0);
int m = matrixData.Matrix.Size(1);
PartitioningParameters dataParameters = matrixData.Data.Item1;
int M1 = dataParameters[0];
int M2 = dataParameters[1];
double goodCrit = matrixData.Data.Item3;
Int64 start = Stopwatch.GetTimestamp();
ISolution solution = algorithm.Run(matrix);
double time = (Stopwatch.GetTimestamp() - start) / (double)Stopwatch.Frequency;
double crit = CoreUtilities.Utilities.Max(new SplittedMatrix(matrix, solution));
double w = Utilities.W(matrix, dataParameters);
double diff = crit - w;
double goodDiff = crit - goodCrit;
results.AddElem(new Note(name, n, m, M1, M2, crit, diff, goodDiff, time, true));
}
results.GetGroups(GroupedResults);
}
public Test_result(IMatrixReadStorage <int, EmptyData> storage, IMatrixReadStorage <int, SolutionData> datastorage, IAlgorithm <int> algorithm, PartitioningParameters parameters)
{
results = new AllResults();
InitializeComponent();
foreach (var matrixData in storage.ReadMatrixes())
{
string name = matrixData.Name;
IMatrix <int> matrix = matrixData.Matrix;
int n = matrixData.Matrix.Size(0);
int m = matrixData.Matrix.Size(1);
int M1 = parameters[0];
int M2 = parameters[1];
Int64 start = Stopwatch.GetTimestamp();
ISolution solution = algorithm.Run(matrix);
double time = (Stopwatch.GetTimestamp() - start) / (double)Stopwatch.Frequency;
double crit = MinMaxCriterium.Calculate(matrix, solution);
double w = Utilities.W(matrix, parameters);
double diff = crit - w;
results.AddElem(new Note(name, n, m, M1, M2, crit, diff, 0, time, false));
}
foreach (var matrixData in datastorage.ReadMatrixes())
{
string name = matrixData.Name;
IMatrix <int> matrix = matrixData.Matrix;
int n = matrixData.Matrix.Size(0);
int m = matrixData.Matrix.Size(1);
PartitioningParameters dataParameters = matrixData.Data.Item1;
int M1 = dataParameters[0];
int M2 = dataParameters[1];
double goodCrit = matrixData.Data.Item3;
Int64 start = Stopwatch.GetTimestamp();
ISolution solution = algorithm.Run(matrix);
double time = (Stopwatch.GetTimestamp() - start) / (double)Stopwatch.Frequency;
double crit = CoreUtilities.Utilities.Max(new SplittedMatrix(matrix, solution));
double w = Utilities.W(matrix, dataParameters);
double diff = crit - w;
double goodDiff = crit - goodCrit;
results.AddElem(new Note(name, n, m, M1, M2, crit, diff, goodDiff, time, true));
}
results.GetGroups(GroupedResults);
}
private static void CreateSmallerBinsRoutine()
{
var wl = AllResults.Load(ConfigurationManager.AppSettings["FilteredPatchesBin"]);
var wlie = wl.ImageEncoding.Reverse();
var wlpe = wl.PatchEncoding.Reverse();
var globalSet = OverallDataAccessor.allClusters;
var interestingImagePatches = wl
.Rows.SelectMany(r => r.Hits.Select(h => h.Hit).Concat(new[] { r.Query }))
.Distinct()
.Select(p => new Patch
{
ImageId = globalSet.ImageEncoding[wlie[p.ImageId]], PatchId = globalSet.PatchEncoding[wlpe[p.PatchId]]
})
.ToLookup(x => x);
var removed = globalSet.Rows.RemoveAll(rr => !interestingImagePatches.Contains(rr.Query));
globalSet.Save(ConfigurationManager.AppSettings["OverAllKnnGraphBin"].Replace(".bin", "-essential-knn.bin"));
Console.WriteLine("After filtering {0} rows remaining, {1} was removed", globalSet.Rows.Count, removed);
}
private static Patch NameToPatch(AllResults results, string[] tokens)
{
if (!results.ImageEncoding.TryGetValue(tokens[2], out var imgId))
{
imgId = results.ImageEncoding.Count;
results.ImageEncoding[tokens[2]] = imgId;
}
var patchPart = tokens[0] + tokens[1];
if (!results.PatchEncoding.TryGetValue(patchPart, out var patchId))
{
patchId = results.PatchEncoding.Count;
results.PatchEncoding[patchPart] = patchId;
}
var p = new Patch {
ImageId = imgId, PatchId = patchId
};
return(p);
}
public void RunResultsAreSetCorrectly()
{
// Create results for two genomes and two test instances:
var runResults = new Dictionary <ImmutableGenome, IDictionary <TestInstance, TestResult> >();
// (a) Create genomes and instances.
var genome1 = new ImmutableGenome(new Genome());
var genome2 = new ImmutableGenome(new Genome());
var instance1 = new TestInstance("1");
var instance2 = new TestInstance("2");
// (b) Create results for each pair.
var results1 = new Dictionary <TestInstance, TestResult>();
var results2 = new Dictionary <TestInstance, TestResult>();
results1.Add(instance1, new TestResult(1));
results1.Add(instance2, new TestResult(2));
results2.Add(instance1, new TestResult(3));
results2.Add(instance2, new TestResult(4));
// (c) Add them to dictionary.
runResults.Add(genome1, results1);
runResults.Add(genome2, results2);
// Create all results message.
var resultMessage = new AllResults <TestInstance, TestResult>(runResults);
// Check it has copied over the dictionary correctly.
Assert.Equal(2, resultMessage.RunResults.Count);
Assert.True(
TestUtils.SetsAreEquivalent(resultMessage.RunResults[genome1], results1),
"Results for first genome should have been different.");
Assert.True(
TestUtils.SetsAreEquivalent(resultMessage.RunResults[genome2], results2),
"Results for second genome should have been different.");
}
/// <summary>
/// Handles a <see cref="AllResultsRequest"/> sent by a certain actor.
/// </summary>
/// <param name="sender">The sender.</param>
private void HandleAllResultsRequest(IActorRef sender)
{
var allResults = new AllResults <TInstance, TResult>(this._runResults);
sender.Tell(allResults);
}
public void PleaseWaitSearch()
{
ChangeTracker.SearchString = (ChangeTracker.SearchString == null) ? "" : ChangeTracker.SearchString.Trim();
if (ChangeTracker.SearchString.Length < 3)
{
ShowError(TranslationProvider.Translate(MultistringTags.TERMINAL_FORM_SEARCH_STRING_TOO_SHORT).ToString());
ClearSearchResults("");
return;
}
Repository.FindMatches(AllResults, "", SelectedLanguage, MatchFilter2, delegate(IMatchVw m1, IMatchVw m2) { return(0); });
if (AllResults.Count == 0)
{
ShowError(TranslationProvider.Translate(MultistringTags.TERMINAL_FORM_NO_MATCH_FOUND).ToString());
ClearSearchResults("");
}
else
{
var sportscb = new List <ComboBoxItem>();
var sports = AllResults.Where(x => x.SportView != null).Select(x => x.SportView).Distinct().ToList();
foreach (var group in sports)
{
if (sportscb.Count(x => x.Id == group.LineObject.GroupId) == 0)
{
sportscb.Add(new ComboBoxItem(group.DisplayName, group.LineObject.GroupId)
{
OrderId = AllResults.Count(x => x.SportView.LineObject.GroupId == @group.LineObject.GroupId)
});
}
}
for (int i = 1; i < sportscb.Count;)
{
var comboBoxItem = sportscb[i];
if (sports.Count(x => x.LineObject.GroupId != comboBoxItem.Id) == 0)
{
sportscb.RemoveAt(i);
}
else
{
i++;
}
}
Sports = new ObservableCollection <ComboBoxItem>(sportscb.OrderByDescending(x => x.OrderId).ToList());
ChangeTracker.SearchSelectedSport = Sports[0];
OnPropertyChanged("SelectedSport");
var tournaments = new List <ComboBoxItem>();
var groups = AllResults.Where(x => x.SportView.LineObject.GroupId == ChangeTracker.SearchSelectedSport.Id).Where(x => x.TournamentView != null).Select(x => x.TournamentView).Distinct().ToList();
foreach (var group in groups)
{
if (tournaments.Count(x => x.Id == group.LineObject.GroupId) == 0)
{
tournaments.Add(new ComboBoxItem(group.DisplayName, group.LineObject.GroupId)
{
OrderId = AllResults.Count(x => x.TournamentView.LineObject.GroupId == @group.LineObject.GroupId)
});
}
}
for (int i = 1; i < tournaments.Count;)
{
var comboBoxItem = tournaments[i];
if (groups.Count(x => x.LineObject.GroupId != comboBoxItem.Id) == 0)
{
tournaments.RemoveAt(i);
}
else
{
i++;
}
}
Tournaments = new ObservableCollection <ComboBoxItem>(tournaments.OrderByDescending(x => x.OrderId).ToList());
ChangeTracker.SearchSelectedTournament = Tournaments[0];
OnPropertyChanged("SelectedTournament");
FillMatches();
}
}
public static void Main(string[] args)
{
if (args.Length > 0 && args[0] == "run-offline-stats")
{
StatsCalculationRoutine();
return;
}
if (args.Length > 0 && args[0] == "csv-to-bin")
{
CsvToProtobuf.CreateProtobufFile();
return;
}
if (args.Length > 0 && args[0] == "bin-all-smaller")
{
CreateSmallerBinsRoutine();
return;
}
var zootLabels = ZootLabelProcessingTests.AllRecords;
var zootLabelsByName = zootLabels.CreateIndex(x => new[] { x.id }).Unique();
var filteredClusters = AllResults.Load(ConfigurationManager.AppSettings["FilteredPatchesBin"]);
var fromIdToName = filteredClusters.ImageEncoding.ToDictionary(x => x.Value, x => Path.GetFileNameWithoutExtension(x.Key).ToLower());
var patchIdToname = filteredClusters.PatchEncoding.Reverse();
var imageIdToFullName = filteredClusters.ImageEncoding.Reverse();
//filteredClusters.Rows = filteredClusters.Rows.Take(10).ToList();
Parallel.ForEach(filteredClusters.Rows, cluster =>
{
Console.Write('.');
var involved = cluster.Hits.Select(x => x.Hit).ToList();
involved.Add(cluster.Query);
var withDistancesAndLabels =
(from i in involved.Distinct()
let name = fromIdToName[i.ImageId]
let Zoot = zootLabelsByName[name]
let Distances = OverallDataAccessor.FindHitsInBigFile(imageIdToFullName[i.ImageId],
patchIdToname[i.PatchId])
select new { Patch = i, Zoot, Distances })
.ToList();
var commonLabels = withDistancesAndLabels
.SelectMany(x => x.Zoot.MeaningfulTextAttributes().Select(a => new { x.Patch, Attr = a }))
.GroupBy(x => x.Attr, x => x.Patch)
.OrderByDescending(g => g.Count())
.Where(g => g.Count() > 1 && g.Key != "zoot")
.Take(20)
.ToList();
var pairsOfLabels =
from first in commonLabels
from second in commonLabels.TakeWhile(x => x.Key != first.Key)
from third in commonLabels.TakeWhile(x => x.Key != second.Key)
let intersect = first.Intersect(second).Intersect(third).Count()
where intersect > 10
select new { first, second, third, Name = first.Key + "^" + second.Key + "^" + third.Key, Size = intersect };
var allFoundMatches = withDistancesAndLabels
.SelectMany(x => x.Distances)
.GroupBy(x => x.Img)
.Select(g => new
{
g.Key, MinDist = g.Min(i => i.Distance),
ZootLabel = zootLabelsByName[OverallDataAccessor.GetCleanName(g.Key)]
})
.OrderBy(x => x.MinDist)
.Select(x => Tuple.Create(x.ZootLabel, x.MinDist))
.ToList();
cluster.Labels =
(from cl in pairsOfLabels
let labels = new[] { cl.first.Key, cl.second.Key, cl.third.Key }
let corr = allFoundMatches.PointBiserialCorrelation(zl =>
zl.MeaningfulTextAttributes().Intersect(labels).Count() == labels.Length)
orderby Math.Abs(corr) descending
select new ClusterLabel {
Correlation = corr, Label = cl.Name, Count = cl.Size
}).ToArray();
Console.Write('!');
});
filteredClusters.Rows = filteredClusters.Rows.Where(x => x.Labels.Any()).OrderByDescending(x => x.Labels.Length == 0 ? 0 : x.Labels.Max(l => Math.Abs(l.Correlation))).ToList();
filteredClusters.Save(ConfigurationManager.AppSettings["FilteredPatchesBin"].Replace(".bin", "-with-labels-triples.bin"));
var withLabels = AllResults.Load(ConfigurationManager.AppSettings["FilteredPatchesBin"].Replace(".bin", "-with-labels-triples.bin"));
using (var sw = new StreamWriter(ConfigurationManager.AppSettings["FilteredPatchesBin"]
.Replace(".bin", "-with-labels-triples.html")))
{
withLabels.Render(sw);
}
}
public async Task OnGetAsync()
{
AllResults = await _apiService.GetAll();
Countries = await _apiService.GetCountries();
}
private static Patch NameToPatch(string name, AllResults results)
{
var tokens = name.Split(Delimiters, StringSplitOptions.RemoveEmptyEntries);
return(NameToPatch(results, tokens));
}
private static void CalculateStats(string filename)
{
using (var file = new StreamWriter("filtering-statistics-selection.csv", append: true))
using (var sw = new CompositionWriter(new[] { file, Console.Out }))
{
AllResults loadedFile;
void PrintStats(string stepName)
{
loadedFile.PrintStats(filename, stepName, sw);
}
var smallerFileName = filename.Replace(".bin", "-tresholdBasedCleaned.bin");
if (!File.Exists(smallerFileName))
{
Console.WriteLine("Starting from scatch, no previous save point");
loadedFile = AllResults.Load(filename);
PrintStats("Default-all");
loadedFile.Rows.RemoveAll(r => r.HasNearDuplicates());
GC.Collect();
PrintStats("Near-duplicate-candidates-removed");
loadedFile.Rows.RemoveAll(r => r.HasTooManyCloseMatches());
PrintStats("Too-large-candidates-removed");
loadedFile.Rows.RemoveAll(r => r.IsTooEquidistant());
PrintStats("Equidistant-candidates-removed");
loadedFile.Save(smallerFileName);
}
var combinations = new[]
{
new { File = "conv3-local.bin", Ratio = 0.91, Max = 400, Min = 10 },
new { File = "conv3-local.bin", Ratio = 0.88, Max = 400, Min = 8 },
new { File = "conv3-local.bin", Ratio = 0.96, Max = 50, Min = 10 },
new { File = "conv4-local.bin", Ratio = 0.91, Max = 800, Min = 12 },
new { File = "conv4-local.bin", Ratio = 0.89, Max = 800, Min = 8 },
new { File = "conv4-local.bin", Ratio = 0.94, Max = 400, Min = 12 },
new { File = "conv5-local.bin", Ratio = 0.8, Max = 800, Min = 8 },
new { File = "conv5-local.bin", Ratio = 0.76, Max = 50, Min = 6 },
new { File = "conv5-local.bin", Ratio = 0.92, Max = 200, Min = 8 },
}.ToLookup(x => x.File);
// new[]{5,6,7,9,10,11,13,14,15}
var bigFile = AllResults.Load(filename);
Console.WriteLine(filename + " was big-loaded.");
foreach (var c in combinations[filename])
{
var ratio = c.Ratio;
loadedFile = AllResults.Load(smallerFileName);
loadedFile.Rows.ForEach(r => r.FilterNeigbhoursUsingDistanceDerivative(ratio));
loadedFile.RefreshReferenceMap();
loadedFile.RefBasedShrink();
loadedFile.RefreshReferenceMap();
for (int i = 1; i < 31; i++)
{
var removed = loadedFile.RefBasedShrink();
loadedFile.RefreshReferenceMap();
if (removed == 0)
{
Console.WriteLine($"Nothing removed at iteration {i}, stopping ref-based shrink for {ratio}");
break;
}
}
// foreach (var maxImagesTreshold in new[]{25,50,100,200,400,800,1600})
// foreach (var minImagesTreshold in new[]{2,4,6,8,10,12})
var clustered = ClusterDecomposition.GroupIntoClusters(loadedFile, c.Max, c.Min);
clustered.PrintStats(filename, $"After-clustering;{ratio};{c.Max};{c.Min}", sw);
var clusterName = filename.Replace(".bin", $"deriv_{ratio}-max_{c.Max}-min_{c.Min}.bin");
clustered.Save(clusterName);
Console.WriteLine(clusterName + " was saved.");
using (var htmlw = new StreamWriter(clusterName.Replace("bin", ".html")))
{
clustered.Render(htmlw);
}
Console.WriteLine(clusterName + " was rendered.");
var wlie = clustered.ImageEncoding.Reverse();
var wlpe = clustered.PatchEncoding.Reverse();
var interestingImagePatches = clustered
.Rows.SelectMany(r => r.Hits.Select(h => h.Hit).Concat(new[] { r.Query }))
.Distinct()
.Select(p => new Patch {
ImageId = bigFile.ImageEncoding[wlie[p.ImageId]], PatchId = bigFile.PatchEncoding[wlpe[p.PatchId]]
})
.ToLookup(x => x);
var newBigFile = new AllResults
{
ImageEncoding = bigFile.ImageEncoding,
PatchEncoding = bigFile.PatchEncoding,
Rows = bigFile.Rows.Where(rr => interestingImagePatches.Contains(rr.Query)).ToList()
};
Console.WriteLine(clusterName + " 's essential knn was shrinked.");
newBigFile.Save(clusterName.Replace(".bin", "-essential-knn.bin"));
Console.WriteLine("After filtering of {2} = {0} rows remaining, {1} was removed", newBigFile.Rows.Count, bigFile.Rows.Count - newBigFile.Rows.Count, clusterName);
}
}
Console.WriteLine(filename + " Done");
}
public static void AgglomerativeClustering(AllResults loaded)
{
var imgName = "241666.jpg"; //"159161.jpg";
var imgId = loaded.ImageEncoding[imgName];
var relevantRows = loaded.Rows
//.Where(r => r.Query.ImageId == imgId)
.ToList();
// cluster all of them together? Include query into dissimilarity function then
// Or product by product, filter down to big elements, and offer to transitively load more and more?
var metric = new ResultsRowSetBasedDistance();
var linkage = new AverageLinkage <ResultsRow>(metric);
var algorithm = new AgglomerativeClusteringAlgorithm <ResultsRow>(linkage);
var clusters = algorithm.GetClustering(new HashSet <ResultsRow>(relevantRows));
clusters.SaveToCsv(@"G:\siret\zoot\protobuf\clustertest.csv");
//RenderData();
var dummyResults = new AllResults {
ImageEncoding = loaded.ImageEncoding, PatchEncoding = loaded.PatchEncoding
};
var clusterQueue = new Queue <Cluster <ResultsRow> >(new[] { clusters.SingleCluster });
while (clusterQueue.Count > 0)
{
var item = clusterQueue.Dequeue();
if (item.Dissimilarity <= 0.70 && item.Count < 50)
{
dummyResults.Rows.Add(new ResultsRow
{
Query = item.First().Query,
Hits = item.SelectMany(x => x.Hits)
.GroupBy(x => x.Hit)
.Select(x => new SearchHit {
Hit = x.Key, Distance = x.Min(y => y.Distance)
})
.Concat(item.Select(i => new SearchHit {
Hit = i.Query, Distance = -1
}))
.ToArray()
});
}
else
{
clusterQueue.Enqueue(item.Parent1);
clusterQueue.Enqueue(item.Parent2);
}
}
loaded.RefreshReferenceMap();
foreach (var k in AllResults.ReferenceMap.Keys)
{
AllResults.ReferenceMap[k][k] = 1;
}
using (var sw = new StreamWriter(@"G:\siret\zoot\protobuf\clusteringTestMega.html", append: false))
{
dummyResults.Render(sw);
}
}
public void AddResults(List <OptimizationResult> results)
{
AllResults.AddRange(results);
jQIsZpt75A(results);
LastResults = results;
}
