private static void LoadAndSearch(string pathPrefix)
{
Stopwatch clock;
var world = new SmallWorld <float[], float>(CosineDistance.NonOptimized);
Console.Write("Loading HNSW graph... ");
clock = Stopwatch.StartNew();
BinaryFormatter formatter = new BinaryFormatter();
var sampleVectors = (List <float[]>)formatter.Deserialize(new MemoryStream(File.ReadAllBytes($"{pathPrefix}.{VectorsPathSuffix}")));
world.DeserializeGraph(sampleVectors, File.ReadAllBytes($"{pathPrefix}.{GraphPathSuffix}"));
Console.WriteLine($"Done in {clock.ElapsedMilliseconds} ms.");
Console.Write($"Generating {TestSize} test vectos... ");
clock = Stopwatch.StartNew();
var vectors = RandomVectors(Dimensionality, TestSize);
Console.WriteLine($"Done in {clock.ElapsedMilliseconds} ms.");
Console.WriteLine("Running search agains the graph... ");
using (var listener = new MetricsEventListener(EventSources.GraphSearchEventSource.Instance))
{
clock = Stopwatch.StartNew();
Parallel.ForEach(vectors, (vector) =>
{
world.KNNSearch(vector, 10);
});
Console.WriteLine($"Done in {clock.ElapsedMilliseconds} ms.");
}
}
/// <summary>
/// Entry point to get estimated closest cluster position. Computed from N_CLOSEST_FOR_CLUSTER by default
/// </summary>
/// <param name="from"></param>
/// <returns></returns>
public Vector3 GetClosestApproximatedClusterPosition(Vector3 from, int k = N_CLOSEST_FOR_CLUSTER)
{
// Can't query if not built yet
if (_graph == null)
{
return(from);
}
float[] queryPos = { from.x, from.y, from.z };
IList <SmallWorld <float[], float> .KNNSearchResult> k_closest;
lock ( _lock ) {
k_closest = _graph.KNNSearch(queryPos, k);
}
// Averaging results positions
Vector3 avg = Vector3.zero;
for (int i = 0; i < k_closest.Count; i++)
{
avg += new Vector3(k_closest[i].Item[0], k_closest[i].Item[1], k_closest[i].Item[2]);
}
return(avg / k_closest.Count);
}
public void KNNSearchTestAlgorithm4(bool expandBestSelection, bool keepPrunedConnections)
{
var parameters = new SmallWorld <float[], float> .Parameters()
{
NeighbourHeuristic = NeighbourSelectionHeuristic.SelectHeuristic, ExpandBestSelection = expandBestSelection, KeepPrunedConnections = keepPrunedConnections
};
var graph = new SmallWorld <float[], float>(CosineDistance.NonOptimized, DefaultRandomGenerator.Instance, parameters);
graph.AddItems(vectors);
int bestWrong = 0;
float maxError = float.MinValue;
for (int i = 0; i < vectors.Count; ++i)
{
var result = graph.KNNSearch(vectors[i], 20);
var best = result.OrderBy(r => r.Distance).First();
Assert.AreEqual(20, result.Count);
if (best.Id != i)
{
bestWrong++;
}
maxError = Math.Max(maxError, best.Distance);
}
Assert.AreEqual(0, bestWrong);
Assert.AreEqual(0, maxError, FloatError);
}
public void KNNSearchTest()
{
var parameters = new SmallWorld <float[], float> .Parameters();
var graph = new SmallWorld <float[], float>(CosineDistance.NonOptimized, DefaultRandomGenerator.Instance, parameters);
graph.AddItems(vectors);
int bestWrong = 0;
float maxError = float.MinValue;
for (int i = 0; i < vectors.Count; ++i)
{
var result = graph.KNNSearch(vectors[i], 20);
var best = result.OrderBy(r => r.Distance).First();
Assert.AreEqual(20, result.Count);
if (best.Id != i)
{
bestWrong++;
}
maxError = Math.Max(maxError, best.Distance);
}
Assert.AreEqual(0, bestWrong);
Assert.AreEqual(0, maxError, FloatError);
}
public void KNNSearchTest()
{
var parameters = new SmallWorld <float[], float> .Parameters();
var graph = new SmallWorld <float[], float>(CosineDistance.NonOptimized);
graph.BuildGraph(this.vectors, new Random(42), parameters);
for (int i = 0; i < this.vectors.Count; ++i)
{
var result = graph.KNNSearch(this.vectors[i], 20);
var best = result.OrderBy(r => r.Distance).First();
Assert.AreEqual(20, result.Count);
Assert.AreEqual(i, best.Id);
Assert.AreEqual(0, best.Distance, FloatError);
}
}
static void Main(string[] args)
{
var parameters = new SmallWorld <float[], float> .Parameters()
{
M = 50,
LevelLambda = 1 / Math.Log(15),
};
var r = new Random();
const int dimensions = 100;
var vectors = GetFloatVectors(dimensions, r);
var graph = new SmallWorld <float[], float>(CosineDistance.SIMD);
var stopWatch = new Stopwatch();
stopWatch.Start();
graph.BuildGraph(vectors, new Random(42), parameters);
stopWatch.Stop();
var buildTime = stopWatch.Elapsed;
Console.WriteLine($"graph build for {vectors.Count} items in {buildTime}");
byte[] buffer = graph.SerializeGraph();
Console.WriteLine($"graph serialized in {buffer.Length} bytes");
float[] query = GetRandomVector(dimensions, r);
for (var i = 0; i < 100; i++)
{
stopWatch = new Stopwatch();
stopWatch.Start();
var best20 = graph.KNNSearch(query, 20);
stopWatch.Stop();
buildTime = stopWatch.Elapsed;
Console.WriteLine($"Top 20 items retrieved in {buildTime}");
}
/*foreach (var item in best20)
* {
* Console.WriteLine($"{item.Id} -> {item.Distance}");
* }*/
}