public int Decode(BitStream32 stream, BitStreamCtx ctx)
{
int min = 0;
int galloping = 1;
int check;
int u = 0;
while (true) {
check = (1 << galloping) - 1;
if (stream.Read (ctx)) {
min = check + 1;
++galloping;
} else {
if (galloping == 1) {
if (stream.Read(ctx)) {
u++;
}
return u;
} else {
u += min;
min = 0;
galloping = 1;
}
}
}
}
public long Decode(BitStream32 stream, BitStreamCtx ctx)
{
long min = 0;
long check;
long u = 0;
int galloping = 1;
while (true) {
check = (1L << galloping) - 1L;
if (stream.Read (ctx)) {
min = check + 1L;
++galloping;
} else {
if (galloping == 1) {
if (stream.Read (ctx)) {
u++;
}
return u;
} else {
u += min;
min = 0L;
galloping = 1;
}
}
}
}
public void Encode(BitStream32 stream, int u)
{
var size = this.Huffman.Encode (stream, u);
if (size == 0) {
throw new KeyNotFoundException ("HuffmanCoding.Encode unknown symbol:" + u.ToString());
}
}
public int Decode(BitStream32 Buffer, BitStreamCtx ctx)
{
int u = Buffer.ReadZeros (ctx);
//int u = Buffer.ReadOnes ();
Buffer.Read (ctx);
return u;
}
// int run_len = 0;
public DiffSetRL2_64()
{
this.Samples = new List<long> ();
this.Offsets = new List<long> ();
this.Stream = new BitStream32 ();
this.Coder = new EliasDelta64 ();
}
public void Build(SAT sat)
{
var n = sat.DB.Count;
this.DB = sat.DB;
var cov = new double[n];
var seq = new int[n];
this.root = sat.root.objID;
int nonzeros = 0;
var visit = new Action<SAT.Node,SAT.Node>((parent, child) => {
seq[child.objID] = (parent == null) ? n : parent.objID;
cov[child.objID] = child.cov;
if (child.cov > 0) ++nonzeros;
});
visit (null, sat.root);
this.Build_IterateSAT (sat.root, visit);
var listibuilder = ListIBuilders.GetArray ();
var permbuilder = PermutationBuilders.GetCyclicPerms (1, listibuilder, listibuilder);
var seqbuilder = SequenceBuilders.GetSeqSinglePerm (permbuilder);
this.SEQ = seqbuilder (seq, n + 1);
this.COV = new List<double> (nonzeros);
var cov_zero = new BitStream32 ();
cov_zero.Write (true, n);
for (int objID = 0; objID < n; ++objID) {
if (cov[objID] > 0) {
this.COV.Add(cov[objID]);
cov_zero[objID] = false;
}
}
this.COV_ZERO = new GGMN ();
this.COV_ZERO.Build (cov_zero, 8);
}
public int ArrayGet(BitStream32 Buffer, int pos)
{
long p = pos;
p *= this.NumBits;
var ctx = new BitStreamCtx (p);
int number = (int)Buffer.Read (this.NumBits, ctx);
return number;
}
public DiffSet()
: base()
{
this.Samples = new List<int> ();
this.Offsets = new List<long> ();
this.Stream = new BitStream32 ();
this.Coder = new EliasDelta ();
}
public void Encode(BitStream32 Buffer, int u)
{
if (u < 1) {
throw new ArgumentOutOfRangeException (String.Format ("Invalid range for elias delta coding, u: {0}", u));
}
var log2 = BitAccess.Log2 (u);
gammacoder.Encode (Buffer, log2);
Buffer.Write (u, log2-1);
}
public void Encode(BitStream32 Buffer, int u)
{
if (u < 0) {
throw new ArgumentOutOfRangeException (String.Format ("Invalid range for UnaryCoding, u: {0}", u));
}
Buffer.Write (false, u);
Buffer.Write (true);
//Buffer.Write (true, u);
//Buffer.Write (false);
}
public void Encode(BitStream32 Buffer, int u)
{
if (u < 0) {
throw new ArgumentOutOfRangeException (String.Format ("Invalid range for BlockCoding, u: {0}", u));
}
int skip = u >> this.Power;
this.SkipCoder.Encode (Buffer, skip);
u &= (1 << this.Power) - 1;
Buffer.Write(u, this.Power);
}
public int Decode(BitStream32 Buffer, BitStreamCtx ctx)
{
int numbits = unary.Decode (Buffer, ctx);
if (numbits == 0) {
return 1;
} else {
int number = (int) Buffer.Read (numbits, ctx);
return (1 << numbits) | number;
}
}
public int Decode(BitStream32 Buffer, BitStreamCtx ctx)
{
int len_code = gammacoder.Decode (Buffer, ctx);
len_code--;
if (len_code == 0) {
return 1;
} else {
int output = (int)Buffer.Read (len_code, ctx);
// Console.WriteLine ("Decode> count: {0}, output: {1}", count, output);
return (1 << len_code) + output;
}
}
public static BitStream32 CreateBitStream32(IList<int> L, int n = 0)
{
if (n == 0 && L.Count > 0) {
n = L[L.Count - 1] + 1;
}
var b = new BitStream32 ();
b.Write (false, n);
foreach (var p in L) {
b[p] = true;
}
return b;
}
public void BuildWebGraph(string filename, SequenceBuilder seqbuilder, BitmapFromBitStream bitmapbuilder = null)
{
if (bitmapbuilder == null) {
bitmapbuilder = BitmapBuilders.GetGGMN_wt (12);
}
var len_stream = new BitStream32 ();
var seq = new List<int> ();
int prev_context = -1;
using (var Input = File.OpenText (filename)) {
string line;
int lineno = 0;
int counterlineno = 0;
while (true) {
{
if (lineno % 10000 == 0) {
if (counterlineno % 10 == 0) {
Console.WriteLine ();
Console.Write ("Processing lines: ");
}
++counterlineno;
Console.Write ("{0}, ", lineno);
}
++lineno;
}
line = Input.ReadLine ();
if (line == null) {
break;
}
if (line.StartsWith ("#")) {
continue;
}
var link = line.Split ('\t', ' ');
var start_node = int.Parse (link [0]);
var end_node = int.Parse (link [1]);
// on webgraph format, starting nodes are already sorted, just advance and count
if (start_node != prev_context) {
for (int diffcount = start_node - prev_context; diffcount > 0; --diffcount) {
len_stream.Write (true);
}
prev_context = start_node;
}
len_stream.Write (false);
seq.Add (end_node);
}
// a simple hack simplifying direct-neighbors's retrieval
len_stream.Write (true);
}
this.SEQ = seqbuilder (seq, prev_context + 1);
this.LENS = bitmapbuilder (new FakeBitmap (len_stream));
}
public RankSelectBlocks(BitStream32 bitmap, short superBlockSize, short blockSize)
{
this.SetState (bitmap.GetIList32(), (int)bitmap.CountBits, superBlockSize, blockSize, null, null);
// Checking ranges of our data types
int bs = this.BitBlocks.Count;
this.SuperBlocks = new int[bs / (this.SuperBlockSize * this.BlockSize)];
this.Blocks = new short[bs / this.BlockSize - this.SuperBlocks.Length];
int abs = 0;
short rel = 0;
//Console.WriteLine ("rel-rank: {0}, abs-rank: {1}, bitmap-ints: {2}", this.Blocks.Length, this.SuperBlocks.Length, this.Bitmap.CountUInt32);
var Lbuffer = bitmap.GetIList32 ();
for (int relindex = 0, absindex = 0, index = 0, relcounter = this.SuperBlockSize - 1;
relindex <= this.Blocks.Length;
index += this.BlockSize,relcounter--) {
int count = Math.Min (this.BlockSize, bitmap.Count32 - index);
// Console.WriteLine ("index: {0}, count: {1}, block-size: {2}, super-block-size: {3}, bitmap-uintsize: {4}",
// index, count, this.BlockSize, this.SuperBlockSize, bitmap.CountUInt32);
rel += (short)this.SeqRank1 (Lbuffer, index, count, -1);
// Console.WriteLine ("absindex: {0}, abs: {1}, relcounter", absindex, abs, relcounter);
if (relcounter == 0) {
abs += rel;
if (absindex < this.SuperBlocks.Length) {
// Console.WriteLine ("========> absindex: {0}, abs: {1}, rel: {2}", absindex, abs, rel);
this.SuperBlocks[absindex] = abs;
}
relcounter = this.SuperBlockSize;
absindex++;
rel = 0;
} else {
if (relindex < this.Blocks.Length) {
this.Blocks[relindex] = rel;
}
relindex++;
}
}
/*Console.WriteLine ("---- Precomputed tables");
Console.WriteLine ("rel-rank: {0}, abs-rank: {1}, bitmap-ints: {2}",
this.Blocks.Length, this.SuperBlocks.Length, this.Bitmap.CountUInt32);
Console.WriteLine ("---- Absolute values");
for (int i = 0; i < this.SuperBlocks.Length; i++) {
Console.WriteLine ("TABLE ABS i: {0}, abs_rank: {1}, bit-position: {2}",
i, this.SuperBlocks[i], 32 * (i + 1) * this.BlockSize * this.SuperBlockSize);
}
Console.WriteLine ("---- Relative values");
for (int i = 0; i < this.Blocks.Length; i++) {
Console.WriteLine ("TABLE REL i: {0}, rel_rank: {1}, bit-position: {2}",
i, this.Blocks[i], (i + 1) * 32 * this.BlockSize);
}*/
}
public void Encode(BitStream32 Buffer, long u)
{
if (u < 1) {
throw new ArgumentOutOfRangeException (String.Format ("Invalid range for elias gamma coding, u: {0}", u));
}
var log2 = BitAccess.Log2 (u);
--log2;
unary.Encode (Buffer, log2);
if (log2 <= 32) {
Buffer.Write ((int)u, log2);
} else {
Buffer.Write ((int)u, 32);
Buffer.Write (u >> 32, log2 - 32);
}
}
public List<WTM_Symbol> Encode(int symbol, List<WTM_Symbol> output = null)
{
var coderstream = new BitStream32 ();
this.Coder.Encode (coderstream, symbol);
int numbits = (int)coderstream.CountBits;
var ctx = new BitStreamCtx (0);
if (output == null) {
output = new List<WTM_Symbol> ();
}
for (int i = 0; i < numbits; i+= this.bits_per_code) {
int code = (int)coderstream.Read (this.bits_per_code, ctx);
output.Add(new WTM_Symbol(code, (byte) Math.Min (this.bits_per_code, numbits - i)));
// Console.WriteLine("get-mini symbol: {0}, numbits: {1}, i: {2}, code: {3}", symbol, numbits, i, code);
}
return output;
}
/// <summary>
/// Build the index
/// </summary>
public virtual void Build(MetricDB db, int num_centers, SequenceBuilder seq_builder = null)
{
this.DB = db;
this.CENTERS = RandomSets.GetRandomSubSet (num_centers, this.DB.Count);
Sorting.Sort<int> (this.CENTERS);
BitStream32 IsCenter = new BitStream32 ();
IsCenter.Write (false, db.Count);
var seq = new int[db.Count];
this.COV = new float[num_centers];
for (int i = 0; i < num_centers; i++) {
IsCenter [this.CENTERS [i]] = true;
seq [this.CENTERS [i]] = this.CENTERS.Count;
}
this.BuildInternal (IsCenter, seq, seq_builder);
//this.Save (output_name, invindex);
}
// int alphabet_numbits;
public SA_fss(IList<int> text, int alphabet_size)
{
this.TXT = text;
var n = text.Count;
// this.alphabet_numbits = ListIFS.GetNumBits(alphabet_size);
// this.SA = new int[n];
//this.Char_Offsets = new int[alphabet_size];
this.Char_SA = new SkipListRank<int>[alphabet_size];
var cmp_fun = new Comparison<int> (this.compare_suffixes);
for (int i = 0; i < alphabet_size; ++i) {
this.Char_SA [i] = new SkipListRank<int> (cmp_fun);
}
this.SA_pointers = new SkipList2<SkipListRank<int>.DataRank>.Node[n];
for (int suffixID = this.TXT.Count-1; suffixID >= 0; --suffixID) {
var c = this.TXT [suffixID];
var list = this.Char_SA [c];
//Console.WriteLine ("=== adding: {0} ({1})", c, Convert.ToChar(c));
var p = list.Add (suffixID);
this.SA_pointers [suffixID] = p;
}
this.A = new int[n+1];
this.A[0] = n;
int I = 1;
foreach (var SLR in this.Char_SA) {
foreach (var data in SLR.SKIPLIST.Traverse()) {
this.A[I] = data.Data;
++I;
}
}
this.SA_pointers = null;
var stream = new BitStream32();
this.charT = new List<int>();
stream.Write(true); // $ symbol
this.charT.Add(0);
for (int i = 0; i < alphabet_size; ++i) {
var count = this.Char_SA[i].Count;
if (count > 0) {
stream.Write(true);
stream.Write(false, count-1);
this.charT.Add(i+1);
}
this.Char_SA[i] = null;
}
this.Char_SA = null;
this.newF = BitmapBuilders.GetGGMN_wt(12).Invoke(new FakeBitmap(stream));
}
public long Decode(BitStream32 Buffer, BitStreamCtx ctx)
{
int numbits = unary.Decode (Buffer, ctx);
if (numbits == 0) {
return 1L;
} else {
ulong number;
if (numbits <= 32) {
number = Buffer.Read (numbits, ctx);
} else {
number = Buffer.Read (32, ctx);
number |= Buffer.Read (numbits - 32, ctx) << 32;
}
number = (1UL << numbits) | number;
return (long)number;
}
}
public long Decode(BitStream32 stream, long N, BitStreamCtx ctx)
{
long min = 0;
long max = N - 1;
long mid;
do {
mid = (min >> 1) + (max >> 1);
if (1L == (min & 1L & max)) {
mid++;
}
if (!stream.Read (ctx)) {
max = mid;
} else {
min = mid + 1L;
}
} while (min < max);
return min;
}
public int Decode(BitStream32 stream, BitStreamCtx ctx)
{
HuffmanInner node = this.Huffman.Root;
while (true) {
bool b = stream.Read (ctx);
HuffmanNode next;
if (b) {
next = node.Right;
} else {
next = node.Left;
}
var leaf = next as HuffmanLeaf;
if (leaf != null) {
return leaf.Symbol;
}
node = next as HuffmanInner;
}
}
public int Decode(BitStream32 stream, int N, BitStreamCtx ctx)
{
int min = 0;
int max = N - 1;
int mid;
do {
mid = (min >> 1) + (max >> 1);
if (1 == (min & 1 & max)) {
mid++;
}
if (!stream.Read (ctx)) {
max = mid;
} else {
min = mid + 1;
}
} while (min < max);
return min;
}
public void Encode(BitStream32 stream, long u, long N)
{
long min = 0;
long max = N - 1;
long mid;
do {
mid = (min >> 1) + (max >> 1);
if (1L == (min & 1L & max)) {
mid++;
}
if (u <= mid) {
stream.Write (false);
max = mid;
} else {
stream.Write (true);
min = mid + 1L;
}
} while (min < max);
}
public void Encode(BitStream32 stream, int u, int N)
{
int min = 0;
int max = N - 1;
int mid;
do {
mid = (min >> 1) + (max >> 1);
if (1 == (min & 1 & max)) {
mid++;
}
if (u <= mid) {
stream.Write (false);
max = mid;
} else {
stream.Write (true);
min = mid + 1;
}
} while (min < max);
}
public long Decode(BitStream32 Buffer, BitStreamCtx ctx)
{
int len_code = gammacoder.Decode (Buffer, ctx);
--len_code;
if (len_code == 0) {
return 1L;
} else {
ulong number;
if (len_code <= 32) {
number = Buffer.Read (len_code, ctx);
} else {
number = Buffer.Read (32, ctx);
number |= Buffer.Read (len_code - 32, ctx) << 32;
}
number = (1UL << len_code) | number;
return (long)number;
}
}
public int Decode(BitStream32 stream, BitStreamCtx ctx)
{
int min = 0;
int galloping = 1;
int check;
while (true) {
check = (1 << galloping) - 1;
if (stream.Read (ctx)) {
min = check + 1;
++galloping;
} else {
if (min == 0) {
return this.SecondCoding.Decode(stream, 2, ctx);
} else {
return min + this.SecondCoding.Decode(stream, min, ctx);
}
}
}
}
public long Decode(BitStream32 stream, BitStreamCtx ctx)
{
long min = 0;
long check;
int galloping = 1;
while (true) {
check = (1L << galloping) - 1L;
if (stream.Read (ctx)) {
min = check + 1L;
++galloping;
} else {
if (min == 0L) {
return this.SecondCoding.Decode (stream, 2, ctx);
} else {
return min + this.SecondCoding.Decode (stream, min, ctx);
}
}
}
}
public int Decode(BitStream32 Buffer, BitStreamCtx ctx)
{
// int numbits = unary.Decode (Buffer, ctx);
// the idea is to replace unary coder by explicit inline code such that (hopefully)
// both "code" and "numbits" are stored into registers
var M = (int)Math.Min (32, Buffer.CountBits - ctx.Offset);
var code = (uint)Buffer.Read (M, ctx);
if ((code & 0x1) == 1) {
ctx.Offset -= M - 1;
return 1;
}
if (code == 0) {
throw new ArgumentException ("Integers larger than 31 bits are not supported by EliasGamma32");
}
int numbits = 0;
// Console.WriteLine ("xxxxxxxxxxxxxxxxxxxxxxxxxxxxx ");
// Console.WriteLine ("xxxxx start-read> offset: {0},\t numbits: {1},\t code: {2}", ctx.Offset, numbits, BitAccess.ToAsciiString (code));
while ((code & 0xFF) == 0) {
numbits += 8;
code >>= 8;
}
if ((code & 0xF) == 0) {
numbits += 4;
code >>= 4;
}
while ((code & 0x1) == 0) {
numbits += 1;
code >>= 1;
}
code >>= 1;
// Console.WriteLine ("xxxxx unary-read> offset: {0},\t numbits: {1},\t code: {2}", ctx.Offset, numbits, BitAccess.ToAsciiString (code));
if (numbits >= 16) {
int in_cache = M - 1 - numbits;
code |= ((uint)Buffer.Read (numbits - in_cache, ctx)) << in_cache;
} else {
ctx.Offset -= M - ((numbits << 1) + 1);
code &= (1u << numbits) - 1;
}
// Console.WriteLine ("xxxxx final-read0> offset: {0},\t numbits: {1},\t code: {3},\t number: {2}", ctx.Offset, numbits, code, BitAccess.ToAsciiString (code));
code |= (1u << numbits);
// Console.WriteLine ("xxxxx final-read1> offset: {0},\t numbits: {1},\t code: {3},\t number: {2}", ctx.Offset, numbits, code, BitAccess.ToAsciiString (code));
return (int)code;
}