public void TestFoo()
{
var rnd = new Random();
Func <Slice, Slice> compress = (input) =>
{
Console.WriteLine("IN [{0}] => {1}", input.Count, input);
var writer = new CompressedBitmapWriter();
int r = WordAlignHybridEncoder.CompressTo(input, writer);
Slice compressed = writer.GetBuffer();
Console.WriteLine("OUT [{0}] => {1} [r={2}]", compressed.Count, compressed, r);
var sb = new StringBuilder();
Console.WriteLine(WordAlignHybridEncoder.DumpCompressed(compressed).ToString());
Console.WriteLine();
return(compressed);
};
compress(Slice.FromString("This is a test of the emergency broadcast system"));
// all zeroes (multiple of 31 bits)
compress(Slice.Repeat(0, 62));
// all zeroes (with padding)
compress(Slice.Repeat(0, 42));
// all ones (multiple of 31 bits)
compress(Slice.Repeat(255, 62));
// all ones (with padding)
compress(Slice.Repeat(255, 42));
// random stuff (multiple of 31 bits)
compress(Slice.Random(rnd, 42));
// random stuff (with padding)
compress(Slice.Random(rnd, 42));
// mostly zeroes
Action <byte[], int> setBit = (b, p) => { b[p >> 3] |= (byte)(1 << (p & 7)); };
Func <int, byte[]> mostlyZeroes = (count) =>
{
var buf = new byte[1024];
for (int i = 0; i < count; i++)
{
setBit(buf, rnd.Next(buf.Length * 8));
}
Console.WriteLine("Mostly zeroes: " + count);
return(buf);
};
compress(Slice.Create(mostlyZeroes(1)));
compress(Slice.Create(mostlyZeroes(10)));
compress(Slice.Create(mostlyZeroes(42)));
compress(Slice.Create(mostlyZeroes(100)));
// mostly ones
Action <byte[], int> clearBit = (b, p) => { b[p >> 3] &= (byte)~(1 << (p & 7)); };
Func <int, byte[]> mostlyOnes = (count) =>
{
var buf = new byte[1024];
for (int i = 0; i < buf.Length; i++)
{
buf[i] = 0xFF;
}
for (int i = 0; i < 10; i++)
{
clearBit(buf, rnd.Next(buf.Length * 8));
}
Console.WriteLine("Mostly ones: " + count);
return(buf);
};
compress(Slice.Create(mostlyOnes(1)));
compress(Slice.Create(mostlyOnes(10)));
compress(Slice.Create(mostlyOnes(42)));
compress(Slice.Create(mostlyOnes(100)));
// progressive
Func <byte[], int, bool> testBit = (b, p) => (b[p >> 3] & (1 << (p & 7))) != 0;
const int VALUES = 8192;
var buffer = new byte[VALUES / 8];
var output = new CompressedBitmapWriter();
WordAlignHybridEncoder.CompressTo(Slice.Create(buffer), output);
Console.WriteLine("{0}\t{1}\t1024", 0, output.Length);
for (int i = 0; i < VALUES / 8; i++)
{
int p;
do
{
p = rnd.Next(VALUES);
}while (testBit(buffer, p));
setBit(buffer, p);
output.Reset();
WordAlignHybridEncoder.CompressTo(Slice.Create(buffer), output);
Console.WriteLine("{0}\t{1}\t1024", 1.0d * (i + 1) / VALUES, output.Length);
}
}
public void Test_Merging_Multiple_Bitmaps()
{
var dataSet = new List <Character>()
{
new Character {
Id = 1, Name = "Spike Spiegel", Gender = "Male", Job = "Bounty_Hunter", Born = new DateTime(2044, 6, 26), Dead = true /* bang! */
},
new Character {
Id = 2, Name = "Jet Black", Gender = "Male", Job = "Bounty_Hunter", Born = new DateTime(2035, 12, 13)
},
new Character {
Id = 3, Name = "Faye Valentine", Gender = "Female", Job = "Bounty_Hunter", Born = new DateTime(1994, 8, 14)
},
new Character {
Id = 4, Name = "Edward Wong Hau Pepelu Tivruski IV", Gender = "Female", Job = "Hacker", Born = new DateTime(2058, 1, 1)
},
new Character {
Id = 5, Name = "Ein", Gender = "Male", Job = "Dog"
},
new Character {
Id = 6, Name = "Vicious", Gender = "Male", Job = "Vilain", Dead = true
},
new Character {
Id = 7, Name = "Julia", Gender = "Female", Job = "Damsel_In_Distress", Dead = true /* It's all a dream */
},
new Character {
Id = 8, Name = "Victoria Tepsichore", Gender = "Female", Job = "Space_Trucker"
},
new Character {
Id = 9, Name = "Punch", Gender = "Male", Job = "TV_Host"
},
new Character {
Id = 10, Name = "Judy", Gender = "Female", Job = "TV_Host"
},
};
// poor man's in memory database
var database = new Dictionary <int, Character>();
var indexByGender = new MemoryIndex <string>(StringComparer.OrdinalIgnoreCase);
var indexByJob = new MemoryIndex <string>(StringComparer.OrdinalIgnoreCase);
var indexOfTheDead = new MemoryIndex <bool>();
// simulate building the indexes one document at a time
var indexers = new[]
{
MakeInserter <Character, string>(indexByGender, (doc) => doc.Id, (doc) => doc.Gender),
MakeInserter <Character, string>(indexByJob, (doc) => doc.Id, (doc) => doc.Job),
MakeInserter <Character, bool>(indexOfTheDead, (doc) => doc.Id, (doc) => doc.Dead),
};
Console.WriteLine("Inserting into database...");
foreach (var character in dataSet)
{
database[character.Id] = character;
foreach (var indexer in indexers)
{
indexer(character);
}
}
// dump the indexes
Console.WriteLine();
DumpIndex("Genders", indexByGender, (s, _) => s);
Console.WriteLine();
DumpIndex("Jobs", indexByJob, (s, _) => s);
Console.WriteLine();
DumpIndex("DeadOrAlive", indexOfTheDead, (s, _) => s);
// Où sont les femmes ?
Console.WriteLine();
Console.WriteLine("indexByGender.Lookup('Female')");
CompressedBitmap females = indexByGender.Lookup("Female");
Assert.That(females, Is.Not.Null);
Console.WriteLine("=> {0}", females.Dump());
DumpIndexQueryResult(database, females);
// R.I.P
Console.WriteLine();
Console.WriteLine("indexOfTheDead.Lookup(dead: true)");
CompressedBitmap deadPeople = indexOfTheDead.Lookup(true);
Assert.That(deadPeople, Is.Not.Null);
Console.WriteLine("=> {0}", deadPeople.Dump());
DumpIndexQueryResult(database, deadPeople);
// combination of both
Console.WriteLine();
Console.WriteLine("indexByGender.Lookup('Female') AND indexOfTheDead.Lookup(dead: true)");
var julia = WordAlignHybridEncoder.And(females, deadPeople);
Console.WriteLine("=> {0}", julia.Dump());
DumpIndexQueryResult(database, julia);
// the crew
Console.WriteLine();
Console.WriteLine("indexByJob.Lookup('Bounty_Hunter' OR 'Hacker' OR 'Dog')");
var bmps = new[] { "Bounty_Hunter", "Hacker", "Dog" }.Select(job => indexByJob.Lookup(job)).ToList();
CompressedBitmap crew = null;
foreach (var bmp in bmps)
{
if (crew == null)
{
crew = bmp;
}
else
{
crew = WordAlignHybridEncoder.Or(crew, bmp);
}
}
crew = crew ?? CompressedBitmap.Empty;
Console.WriteLine("=> {0}", crew.Dump());
DumpIndexQueryResult(database, crew);
}
public void TestFoo()
{
var rnd = new Random();
void Compress(Slice input)
{
Log($"IN [{input.Count}] => {input}");
var writer = new CompressedBitmapWriter();
int r = WordAlignHybridEncoder.CompressTo(input, writer);
var compressed = writer.GetBuffer();
Log($"OUT [{compressed.Count}] => {compressed} [r={r}]");
var sb = new StringBuilder();
Log(WordAlignHybridEncoder.DumpCompressed(compressed).ToString());
Log();
}
Compress(Slice.FromString("This is a test of the emergency broadcast system"));
// all zeroes (multiple of 31 bits)
Compress(Slice.Repeat(0, 62));
// all zeroes (with padding)
Compress(Slice.Repeat(0, 42));
// all ones (multiple of 31 bits)
Compress(Slice.Repeat(255, 62));
// all ones (with padding)
Compress(Slice.Repeat(255, 42));
// random stuff (multiple of 31 bits)
Compress(Slice.Random(rnd, 42));
// random stuff (with padding)
Compress(Slice.Random(rnd, 42));
// mostly zeroes
void SetBit(byte[] b, int p)
{
b[p >> 3] |= (byte)(1 << (p & 7));
}
byte[] MostlyZeroes(int count)
{
var buf = new byte[1024];
for (int i = 0; i < count; i++)
{
SetBit(buf, rnd.Next(buf.Length * 8));
}
Log("Mostly zeroes: " + count);
return(buf);
}
Compress(MostlyZeroes(1).AsSlice());
Compress(MostlyZeroes(10).AsSlice());
Compress(MostlyZeroes(42).AsSlice());
Compress(MostlyZeroes(100).AsSlice());
// mostly ones
void ClearBit(byte[] b, int p)
{
b[p >> 3] &= (byte)~(1 << (p & 7));
}
byte[] MostlyOnes(int count)
{
var buf = new byte[1024];
for (int i = 0; i < buf.Length; i++)
{
buf[i] = 0xFF;
}
for (int i = 0; i < 10; i++)
{
ClearBit(buf, rnd.Next(buf.Length * 8));
}
Log("Mostly ones: " + count);
return(buf);
}
Compress(MostlyOnes(1).AsSlice());
Compress(MostlyOnes(10).AsSlice());
Compress(MostlyOnes(42).AsSlice());
Compress(MostlyOnes(100).AsSlice());
// progressive
bool TestBit(byte[] b, int p) => (b[p >> 3] & (1 << (p & 7))) != 0;
const int VALUES = 8192;
var buffer = new byte[VALUES / 8];
var output = new CompressedBitmapWriter();
WordAlignHybridEncoder.CompressTo(buffer.AsSlice(), output);
Log($"{0}\t{output.Length}\t1024");
for (int i = 0; i < VALUES / 8; i++)
{
int p;
do
{
p = rnd.Next(VALUES);
}while (TestBit(buffer, p));
SetBit(buffer, p);
output.Reset();
WordAlignHybridEncoder.CompressTo(buffer.AsSlice(), output);
Log($"{1.0d * (i + 1) / VALUES}\t{output.Length}\t1024");
}
}