private static void LoadAndSearch(string pathPrefix)
{
Stopwatch clock;
Console.Write("Loading HNSW graph... ");
clock = Stopwatch.StartNew();
BinaryFormatter formatter = new BinaryFormatter();
var sampleVectors = (List <float[]>)formatter.Deserialize(new MemoryStream(File.ReadAllBytes($"{pathPrefix}.{VectorsPathSuffix}")));
SmallWorld <float[], float> world;
using (var f = File.OpenRead($"{pathPrefix}.{GraphPathSuffix}"))
{
world = SmallWorld <float[], float> .DeserializeGraph(sampleVectors, CosineDistance.SIMDForUnits, DefaultRandomGenerator.Instance, f);
}
Console.WriteLine($"Done in {clock.ElapsedMilliseconds} ms.");
Console.Write($"Generating {TestSize} test vectos... ");
clock = Stopwatch.StartNew();
var vectors = RandomVectors(Dimensionality, TestSize);
Console.WriteLine($"Done in {clock.ElapsedMilliseconds} ms.");
Console.WriteLine("Running search agains the graph... ");
using (var listener = new MetricsEventListener(EventSources.GraphSearchEventSource.Instance))
{
clock = Stopwatch.StartNew();
Parallel.ForEach(vectors, (vector) =>
{
world.KNNSearch(vector, 10);
});
Console.WriteLine($"Done in {clock.ElapsedMilliseconds} ms.");
}
}
private static void BuildAndSave(string pathPrefix)
{
Stopwatch clock;
List <float[]> sampleVectors;
var parameters = new Parameters();
parameters.EnableDistanceCacheForConstruction = true;
var world = new SmallWorld <float[], float>(CosineDistance.NonOptimized);
Console.Write($"Generating {SampleSize} sample vectos... ");
clock = Stopwatch.StartNew();
sampleVectors = RandomVectors(Dimensionality, SampleSize);
Console.WriteLine($"Done in {clock.ElapsedMilliseconds} ms.");
Console.WriteLine("Building HNSW graph... ");
using (var listener = new MetricsEventListener(EventSources.GraphBuildEventSource.Instance))
{
clock = Stopwatch.StartNew();
world.BuildGraph(sampleVectors, new Random(42), parameters);
Console.WriteLine($"Done in {clock.ElapsedMilliseconds} ms.");
}
Console.Write($"Saving HNSW graph to '${Path.Combine(Directory.GetCurrentDirectory(), pathPrefix)}'... ");
clock = Stopwatch.StartNew();
BinaryFormatter formatter = new BinaryFormatter();
MemoryStream sampleVectorsStream = new MemoryStream();
formatter.Serialize(sampleVectorsStream, sampleVectors);
File.WriteAllBytes($"{pathPrefix}.{VectorsPathSuffix}", sampleVectorsStream.ToArray());
File.WriteAllBytes($"{pathPrefix}.{GraphPathSuffix}", world.SerializeGraph());
Console.WriteLine($"Done in {clock.ElapsedMilliseconds} ms.");
}
/// <summary>
/// Initializes the ANN class, given all featuresvectors to which
/// a query will be compared.
/// </summary>
/// <param name="database">All featurevectors of its world.</param>
public ANN(IEnumerable <NamedFeatureVector> database)
{
IReadOnlyList <NamedFeatureVector> vectors = database.ToList().AsReadOnly();
world = new SmallWorld <NamedFeatureVector, double>(ANNDistance);
SmallWorld <NamedFeatureVector, double> .Parameters parameters =
new SmallWorld <NamedFeatureVector, double> .Parameters
{
EnableDistanceCacheForConstruction = true
};
using (MetricsEventListener listener =
new MetricsEventListener(EventSources.GraphBuildEventSource.Instance)) {
world.BuildGraph(vectors, RandomUtil.ThreadSafeRandom, parameters);
}
}
private static void BuildAndSave(string pathPrefix)
{
var world = new SmallWorld <float[], float>(CosineDistance.SIMDForUnits, DefaultRandomGenerator.Instance, new Parameters()
{
EnableDistanceCacheForConstruction = true, InitialDistanceCacheSize = SampleSize, NeighbourHeuristic = NeighbourSelectionHeuristic.SelectHeuristic, KeepPrunedConnections = true, ExpandBestSelection = true
});
Console.Write($"Generating {SampleSize} sample vectos... ");
var clock = Stopwatch.StartNew();
var sampleVectors = RandomVectors(Dimensionality, SampleSize);
Console.WriteLine($"Done in {clock.ElapsedMilliseconds} ms.");
Console.WriteLine("Building HNSW graph... ");
using (var listener = new MetricsEventListener(EventSources.GraphBuildEventSource.Instance))
{
clock = Stopwatch.StartNew();
for (int i = 0; i < (SampleSize / SampleIncrSize); i++)
{
world.AddItems(sampleVectors.Skip(i * SampleIncrSize).Take(SampleIncrSize).ToArray());
Console.WriteLine($"\nAt {i+1} of {SampleSize / SampleIncrSize} Elapsed: {clock.ElapsedMilliseconds} ms.\n");
}
Console.WriteLine($"Done in {clock.ElapsedMilliseconds} ms.");
}
Console.Write($"Saving HNSW graph to '${Path.Combine(Directory.GetCurrentDirectory(), pathPrefix)}'... ");
clock = Stopwatch.StartNew();
BinaryFormatter formatter = new BinaryFormatter();
MemoryStream sampleVectorsStream = new MemoryStream();
formatter.Serialize(sampleVectorsStream, sampleVectors);
File.WriteAllBytes($"{pathPrefix}.{VectorsPathSuffix}", sampleVectorsStream.ToArray());
using (var f = File.Open($"{pathPrefix}.{GraphPathSuffix}", FileMode.Create))
{
world.SerializeGraph(f);
}
Console.WriteLine($"Done in {clock.ElapsedMilliseconds} ms.");
}
