public void Encode(BitStream32 Buffer, int u)
{
if (u < 0) {
throw new ArgumentOutOfRangeException (String.Format ("Invalid range for UnaryCoding, u: {0}", u));
}
Buffer.Write (false, u);
Buffer.Write (true);
//Buffer.Write (true, u);
//Buffer.Write (false);
}
public void Encode(BitStream32 Buffer, long u)
{
if (u < 1) {
throw new ArgumentOutOfRangeException (String.Format ("Invalid range for elias gamma coding, u: {0}", u));
}
var log2 = BitAccess.Log2 (u);
--log2;
unary.Encode (Buffer, log2);
if (log2 <= 32) {
Buffer.Write ((int)u, log2);
} else {
Buffer.Write ((int)u, 32);
Buffer.Write (u >> 32, log2 - 32);
}
}
public void Build(SAT sat)
{
var n = sat.DB.Count;
this.DB = sat.DB;
var cov = new double[n];
var seq = new int[n];
this.root = sat.root.objID;
int nonzeros = 0;
var visit = new Action<SAT.Node,SAT.Node>((parent, child) => {
seq[child.objID] = (parent == null) ? n : parent.objID;
cov[child.objID] = child.cov;
if (child.cov > 0) ++nonzeros;
});
visit (null, sat.root);
this.Build_IterateSAT (sat.root, visit);
var listibuilder = ListIBuilders.GetArray ();
var permbuilder = PermutationBuilders.GetCyclicPerms (1, listibuilder, listibuilder);
var seqbuilder = SequenceBuilders.GetSeqSinglePerm (permbuilder);
this.SEQ = seqbuilder (seq, n + 1);
this.COV = new List<double> (nonzeros);
var cov_zero = new BitStream32 ();
cov_zero.Write (true, n);
for (int objID = 0; objID < n; ++objID) {
if (cov[objID] > 0) {
this.COV.Add(cov[objID]);
cov_zero[objID] = false;
}
}
this.COV_ZERO = new GGMN ();
this.COV_ZERO.Build (cov_zero, 8);
}
// int alphabet_numbits;
public SA_fss(IList<int> text, int alphabet_size)
{
this.TXT = text;
var n = text.Count;
// this.alphabet_numbits = ListIFS.GetNumBits(alphabet_size);
// this.SA = new int[n];
//this.Char_Offsets = new int[alphabet_size];
this.Char_SA = new SkipListRank<int>[alphabet_size];
var cmp_fun = new Comparison<int> (this.compare_suffixes);
for (int i = 0; i < alphabet_size; ++i) {
this.Char_SA [i] = new SkipListRank<int> (cmp_fun);
}
this.SA_pointers = new SkipList2<SkipListRank<int>.DataRank>.Node[n];
for (int suffixID = this.TXT.Count-1; suffixID >= 0; --suffixID) {
var c = this.TXT [suffixID];
var list = this.Char_SA [c];
//Console.WriteLine ("=== adding: {0} ({1})", c, Convert.ToChar(c));
var p = list.Add (suffixID);
this.SA_pointers [suffixID] = p;
}
this.A = new int[n+1];
this.A[0] = n;
int I = 1;
foreach (var SLR in this.Char_SA) {
foreach (var data in SLR.SKIPLIST.Traverse()) {
this.A[I] = data.Data;
++I;
}
}
this.SA_pointers = null;
var stream = new BitStream32();
this.charT = new List<int>();
stream.Write(true); // $ symbol
this.charT.Add(0);
for (int i = 0; i < alphabet_size; ++i) {
var count = this.Char_SA[i].Count;
if (count > 0) {
stream.Write(true);
stream.Write(false, count-1);
this.charT.Add(i+1);
}
this.Char_SA[i] = null;
}
this.Char_SA = null;
this.newF = BitmapBuilders.GetGGMN_wt(12).Invoke(new FakeBitmap(stream));
}
public void Encode(BitStream32 Buffer, int u)
{
if (u < 1) {
throw new ArgumentOutOfRangeException (String.Format ("Invalid range for elias delta coding, u: {0}", u));
}
var log2 = BitAccess.Log2 (u);
gammacoder.Encode (Buffer, log2);
Buffer.Write (u, log2-1);
}
public void Encode(BitStream32 Buffer, int u)
{
if (u < 0) {
throw new ArgumentOutOfRangeException (String.Format ("Invalid range for BlockCoding, u: {0}", u));
}
int skip = u >> this.Power;
this.SkipCoder.Encode (Buffer, skip);
u &= (1 << this.Power) - 1;
Buffer.Write(u, this.Power);
}
public static BitStream32 CreateBitStream32(IList<int> L, int n = 0)
{
if (n == 0 && L.Count > 0) {
n = L[L.Count - 1] + 1;
}
var b = new BitStream32 ();
b.Write (false, n);
foreach (var p in L) {
b[p] = true;
}
return b;
}
public void Encode(BitStream32 stream, long u)
{
long check;
long min = 0;
int galloping = 1;
while (true) {
check = (1L << galloping) - 1L;
if (u > check) {
stream.Write (true);
min = check + 1L;
++galloping;
} else {
stream.Write (false);
if (min == 0L) {
this.SecondCoding.Encode (stream, u, 2);
} else {
this.SecondCoding.Encode (stream, u - min, min);
}
break;
}
}
}
public void Encode(BitStream32 stream, int u)
{
int min = 0;
int galloping = 1;
int check;
while (true) {
check = (1 << galloping) - 1;
if (u > check) {
stream.Write (true);
min = check + 1;
++galloping;
} else {
stream.Write (false);
if (min == 0) {
this.SecondCoding.Encode (stream, u, 2);
} else {
this.SecondCoding.Encode (stream, u - min, min);
}
break;
}
}
}
public void Encode(BitStream32 stream, int u)
{
int min = 0;
int galloping = 1;
int check;
while (true) {
check = (1 << galloping) - 1;
if (u > check) {
stream.Write (true);
min = check + 1;
++galloping;
} else {
stream.Write (false);
if (galloping == 1) {
stream.Write(u == 1);
return;
} else {
u -= min;
min = 0;
galloping = 1;
}
}
}
}
public void Encode(BitStream32 stream, long u)
{
long min = 0;
long check;
int galloping = 1;
while (true) {
check = (1L << galloping) - 1L;
if (u > check) {
stream.Write (true);
min = check + 1L;
++galloping;
} else {
stream.Write (false);
if (galloping == 1) {
stream.Write (u == 1L);
return;
} else {
u -= min;
min = 0L;
galloping = 1;
}
}
}
}
public void Build(string sa_name, SequenceBuilder seq_builder = null, BitmapFromBitStream bitmap_builder = null)
{
if (seq_builder == null) {
seq_builder = SequenceBuilders.GetSeqXLB_DiffSetRL2_64(16, 63);
}
using (var Input = new BinaryReader (File.OpenRead (sa_name + ".structs"))) {
this.newF = RankSelectGenericIO.Load (Input);
if (bitmap_builder != null) {
var newF_stream = new BitStream32();
for (int i = 0; i < this.newF.Count; ++i) {
newF_stream.Write (this.newF.Access(i));
}
this.newF = bitmap_builder(new FakeBitmap(newF_stream));
}
int len = this.newF.Count1;
this.charT = new int[len];
// Console.WriteLine ("*****>> charT => {0} bytes", this.charT.Length * 4);
PrimitiveIO<int>.ReadFromFile (Input, len, this.charT);
}
using (var Input = new BinaryReader (File.OpenRead (sa_name + ".psi"))) {
int seqlen = this.newF.Count;
var seq = new int[seqlen];
var L = new List<int>(this.N/this.Sigma + 1);
int curr = 0;
for (int i = 1; i <= this.AlphabetSize; i++) {
int next;
if (i == this.AlphabetSize) {
next = this.newF.Count;
} else {
next = this.newF.Select1 (i + 1);
}
int len = next - curr;
L.Clear();
PrimitiveIO<int>.ReadFromFile (Input, len, L);
for (int j = 0; j < len; ++j) {
var x = L[j];
try {
seq[ x ] = i - 1;
} catch (Exception e) {
Console.WriteLine ("== i: {0}, j: {1}, x: {2}, seq-count: {3}, len: {4}",
i, j, x, seq.Length, len);
throw e;
}
}
curr = next;
}
this.SeqPsi = seq_builder(seq, this.AlphabetSize);
}
using (var Input = new BinaryReader (File.OpenRead (sa_name + ".samples"))) {
this.SA_sample_step = Input.ReadInt16 ();
this.SA_marked = RankSelectGenericIO.Load (Input);
var _samples = new ListIFS ();
_samples.Load (Input);
var _invsamples = new ListIFS ();
_invsamples.Load (Input);
this.SA_samples = _samples;
this.SA_invsamples = _invsamples;
}
}
/// <summary>
/// Build the index
/// </summary>
public virtual void Build(MetricDB db, int num_centers, SequenceBuilder seq_builder = null)
{
this.DB = db;
this.CENTERS = RandomSets.GetRandomSubSet (num_centers, this.DB.Count);
Sorting.Sort<int> (this.CENTERS);
BitStream32 IsCenter = new BitStream32 ();
IsCenter.Write (false, db.Count);
var seq = new int[db.Count];
this.COV = new float[num_centers];
for (int i = 0; i < num_centers; i++) {
IsCenter [this.CENTERS [i]] = true;
seq [this.CENTERS [i]] = this.CENTERS.Count;
}
this.BuildInternal (IsCenter, seq, seq_builder);
//this.Save (output_name, invindex);
}
/// <summary>
/// Build the specified seq, sigma and t.
/// </summary>
public void Build(IList<int> seq, int sigma, short t = 16, IIEncoder32 rl2_coder = null, short rl2_block_size = 127)
{
// A counting sort construction of the permutation
var counters = new int[sigma];
foreach (var s in seq) {
if (s + 1 < sigma) {
counters [s + 1]++;
}
}
for (int i = 1; i < sigma; i++) {
counters [i] += counters [i - 1];
}
var n = seq.Count;
var P = new int[n];
for (int i = 0; i < n; i++) {
var sym = seq [i];
var pos = counters [sym];
P [pos] = i;
counters [sym] = pos + 1;
}
// the bitmap to save the lengths
var lens = new BitStream32 ();
int prevc = 0;
foreach (var c in counters) {
var len = c - prevc;
prevc = c;
lens.Write (true);
lens.Write (false, len);
}
// an additional 1 to the end, to simplify source code
lens.Write (true);
var bb_lens = new FakeBitmap (lens);
this.LENS = BitmapBuilder (bb_lens);
this.PERM = new SuccRL2CyclicPerms_MRRR ();
if (rl2_coder == null) {
rl2_coder = new EliasGamma32 ();
}
var build_params = new SuccRL2CyclicPerms_MRRR.BuildParams (rl2_coder, rl2_block_size);
this.PERM.Build (P, t, build_params);
}
public void ArrayAdd(BitStream32 Buffer, int val)
{
Buffer.Write (val, this.NumBits);
}
public void BuildBackend(IList<uint> bitblocks, short Brank, int Bselect, int N)
{
this.BaseIndex = new GGMN ();
this.BaseIndex.BuildBackend (bitblocks, N, Brank);
this.B = Bselect;
int M = this.BaseIndex.Rank1 (N - 1);
// Console.WriteLine("XXXX N: {0}, M: {1}", N, M);
this.PosAbs = new int[(int)Math.Ceiling (M * 1.0 / this.B)];
int m = 0;
int i = 0;
int large_limit = 32 * this.B;
BitStream32 mark_bits = new BitStream32 (this.PosAbs.Count >> 5);
int num_saved_slots = 0;
while (m < M) {
int start_pos = this.BaseIndex.Select1 (m + 1);
int Bmin = Math.Min (this.B, M - m);
int end_pos = this.BaseIndex.Select1 (m + Bmin);
if (end_pos - start_pos >= large_limit) {
mark_bits.Write (true);
num_saved_slots += Bmin;
} else {
mark_bits.Write (false);
// uncomment to make all large blocks (debugging). Comment the previous line to enable to following
// mark_bits.Write (true);
// num_saved_slots += Bmin;
}
this.PosAbs[i] = start_pos;
m += this.B;
i++;
}
this.IsLargeBlock = new GGMN ();
this.IsLargeBlock.Build (mark_bits, 4);
int R = this.IsLargeBlock.Rank1 (this.IsLargeBlock.Count - 1);
this.SavedPos = new int[num_saved_slots];
for (int k = 0; k < R; k++) {
int index = this.IsLargeBlock.Select1 (k + 1);
int rank_base = this.B * index;
int index_base = this.B * k;
int maxB = Math.Min (this.B, M - rank_base);
for (int rank_rel = 0; rank_rel < maxB; rank_rel++) {
var pos = this.BaseIndex.Select1( rank_rel + rank_base + 1);
this.SavedPos[rank_rel + index_base] = pos;
}
}
}
public void Build(IList<int> orderedList, int N, short Brank, int Bselect)
{
BitStream32 b = new BitStream32 ();
PlainSortedList s = new PlainSortedList ();
s.Build (orderedList, (int)N);
for (int i = 0; i < N; i++) {
b.Write (s[i]);
}
this.Build (b, Brank, Bselect);
}
public void Build(IList<long> orderedList, long n, byte numLowerBits, BitmapFromBitStream H_builder)
{
//this.M = orderedList.Count;
int M = orderedList.Count;
this.N = n;
if (M > this.N) {
Console.WriteLine ("XXXXX LastItem: {0}", orderedList [orderedList.Count - 1]);
throw new ArgumentOutOfRangeException (String.Format ("SArray N < M, N: {0}, M: {1}", this.N, M));
}
if (numLowerBits < 1) {
numLowerBits = 1;
}
// this.NumLowerBits = numLowerBits;
this.L = new ListIFS (numLowerBits, new BitStream32 ((numLowerBits / 32) * M));
// Creating bitmaps
// 2^ (log N - log N / M) = 2^ \log N M / N = M.
// 2^ (log N - log N / M) = 2^ \log N M / N = M.
int numpart = (int)Math.Ceiling (Math.Pow (2, (Math.Ceiling (Math.Log (this.N)) - this.GetNumLowerBits ())));
var H_stream = new BitStream32 (M + (numpart / 32 + 1));
long mask = this.get_mask ();
int prevblock = -1;
for (int i = 0; i < M; i++) {
this.L.Add ((int)(orderedList [i] & mask));
int currentblock = (int)(orderedList [i] >> this.GetNumLowerBits ());
if (prevblock != currentblock) {
while (prevblock < currentblock) {
H_stream.Write (false);
prevblock++;
}
}
H_stream.Write (true);
}
//an additional technical zero
H_stream.Write (false, M - prevblock);
H_stream.Write (false);
if (H_builder == null) {
H_builder = BitmapBuilders.GetDArray_wt(16,32);
}
var fb = new FakeBitmap(H_stream);
this.H = H_builder(fb);
}
public void Sort()
{
this._numerical_sort (0, 0, this.SA.Length - 1);
// this._sort(0, 0, this.SA.Length-1);
// creating auxiliar structures and bitmaps
this.charT = new List<int> ();
var B_newF = new BitStream32 ();
this.charT.Add ('$'); // the special lexicographically smaller symbol
B_newF.Write (true);
for (int i = 1; i < this.SA.Length; i++) {
var c = this.TXT[this.SA[i]];
if (i == 1) {
this.charT.Add (c);
B_newF.Write (true);
} else {
if (this.charT[this.charT.Count - 1] != c) {
this.charT.Add (c);
B_newF.Write (true);
} else {
B_newF.Write (false);
}
}
}
this.newF = new GGMN ();
this.newF.Build (B_newF, 8);
}
/// <summary>
/// Read the database from a listing file (one filename per line)
/// </summary>
public void Build(string filename, BitmapFromBitStream len_builder = null)
{
Console.WriteLine ("****** Reading database from list of files");
this.Name = filename;
var NAMES = File.ReadAllLines (filename);
int counter = 0;
var data_stream = new List<byte> ();
var lens_stream = new BitStream32 ();
foreach (var s in NAMES) {
++counter;
if (s.Length == 0) {
continue;
}
if (counter % 1000 == 0) {
Console.WriteLine ("*** Processing docid {0}/{1} (adv: {2:0.000}%): '{3}'",
counter, NAMES.Length, counter*100.0/NAMES.Length, s);
}
var data = (IList<byte>)this.Parse (s, true);
if (data.Count == 0) {
throw new ArgumentException(String.Format("AFP files must not be empty: {0}", s));
}
lens_stream.Write (true);
lens_stream.Write (false, data.Count-1);
data_stream.Capacity += data.Count;
foreach (var b in data) {
data_stream.Add (b);
}
}
lens_stream.Write(true);
if (len_builder == null) {
len_builder = BitmapBuilders.GetGGMN_wt (12);
}
this.LENS = len_builder (new FakeBitmap (lens_stream));
this.DATA = data_stream;
}
public void Build(string out_filename, IList<IList<byte>> data_list, BitmapFromBitStream len_builder = null)
{
this.Name = out_filename;
int counter = 0;
var data_stream = new List<byte> ();
var lens_stream = new BitStream32 ();
foreach (var data in data_list) {
++counter;
if (counter % 1000 == 0) {
Console.WriteLine ("*** Processing docid {0}/{1} (adv: {2:0.000}%)",
counter, data_list.Count, counter*100.0/data_list.Count);
}
lens_stream.Write (true);
lens_stream.Write (false, data.Count-1);
// data_stream.Capacity += data.Count;
foreach (var b in data) {
data_stream.Add (b);
}
}
lens_stream.Write(true);
if (len_builder == null) {
len_builder = BitmapBuilders.GetGGMN_wt (12);
}
this.LENS = len_builder (new FakeBitmap (lens_stream));
this.DATA = data_stream;
}
public void Build(IList<int> seq, int sigma)
{
// NOTE: Please check sigma <=> BlockSize in this method
this.sigma = sigma;
this.n = seq.Count;
var B_stream = new BitStream32 ();
var X_stream = new BitStream32[ sigma ];
for (int i = 0; i < sigma; i++) {
X_stream [i] = new BitStream32 ();
}
var lists = new List<int>[sigma];
for (int i = 0; i < sigma; i++) {
lists [i] = new List<int> ();
}
int num_blocks = (int)Math.Ceiling (this.n * 1.0 / this.sigma);
this.perms = new List<IPermutation> (num_blocks);
for (int i = 0; i < this.n; i+= this.sigma) {
// writing block separators
foreach (var b in X_stream) {
b.Write (false);
}
// clearing perm B
// selecting block size
int s = Math.Min (this.n - i, this.sigma);
this.BuildPermInvIndex (seq, i, s, lists);
var P = new List<int> (s);
for (int j = 0; j < this.sigma; j++) {
var c = lists [j].Count;
B_stream.Write (false);
if (c > 0) {
X_stream [j].Write (true, c);
B_stream.Write (true, c);
foreach (var u in lists[j]) {
P.Add (u);
}
}
}
var _perm = this.PermBuilder (P);
this.perms.Add (_perm);
}
var _X_stream = X_stream [0];
for (int i = 1; i < X_stream.Length; i++) {
var _X_curr = X_stream [i];
for (int j = 0; j < _X_curr.CountBits; j++) {
// esto se podria hace por entero en lugar de bit
_X_stream.Write (_X_curr [j]);
}
}
// If we write a zero at the end of the streams the code is simplified
_X_stream.Write (false);
B_stream.Write (false);
this.B = this.BitmapBuilder (new FakeBitmap (B_stream));
this.X = this.BitmapBuilder (new FakeBitmap (_X_stream));
this.Xacc = new List<int> ();
for (int i = 0; i < this.sigma; i++) {
int acc_rank = this.X.Rank1 (this.X.Select0 (i * num_blocks + 1));
this.Xacc.Add (acc_rank);
}
this.compute_num_blocks ();
}
public void Build(IRankSelect RS, short blockSize)
{
// we need to store n bits, this is not the best solution!!
BitStream32 B = new BitStream32 ();
for (int i = 0, count = RS.Count; i < count; i++) {
B.Write (RS.Access(i));
}
this.Build (B, blockSize);
}
public void Build(IList<int> seq, int sigma, BitmapFromBitStream bitmap_builder, int cyclic_perm_t)
{
// NOTE: Please check sigma <=> BlockSize in this method
this.sigma = sigma;
this.n = seq.Count;
var B_stream = new BitStream32 ();
var X_stream = new BitStream32[ sigma ];
for (int i = 0; i < sigma; i++) {
X_stream [i] = new BitStream32 ();
}
var lists = new List<int>[sigma];
for (int i = 0; i < sigma; i++) {
lists [i] = new List<int> ();
}
int num_blocks = (int)Math.Ceiling (this.n * 1.0 / this.sigma);
//this.perms = new IPermutation[num_blocks];
this.perms = new CyclicPerms_MRRR[num_blocks];
for (int i = 0, I = 0; i < this.n; i+= this.sigma, ++I) {
// writing block separators
foreach (var b in X_stream) {
b.Write (true);
}
// clearing perm B
// selecting block size
int s = Math.Min (this.n - i, this.sigma);
this.BuildPermInvIndex (seq, i, s, lists);
var P = new List<int> (s);
for (int j = 0; j < this.sigma; j++) {
var c = lists [j].Count;
B_stream.Write (false);
if (c > 0) {
X_stream [j].Write (false, c);
B_stream.Write (true, c);
foreach (var u in lists[j]) {
P.Add (u);
}
}
}
//var _perm = perm_builder(P);
//this.perms[I] = _perm;
this.perms [I] = (CyclicPerms_MRRR)PermutationBuilders.GetCyclicPermsListIFS(cyclic_perm_t).Invoke (P);
}
var _X_stream = X_stream [0];
for (int i = 1; i < X_stream.Length; i++) {
var _X_curr = X_stream [i];
for (int j = 0; j < _X_curr.CountBits; j++) {
// esto se podria hace por entero en lugar de bit
_X_stream.Write (_X_curr [j]);
}
}
// If we write a zero at the end of the streams the code is simplified
_X_stream.Write (true);
B_stream.Write (false);
this.B = bitmap_builder (new FakeBitmap (B_stream));
this.X = bitmap_builder (new FakeBitmap (_X_stream));
this.compute_num_blocks ();
}
public void Save_CSA_BWT(string sa_name, int sample_step)
{
Console.WriteLine ("Save_CSA_BWT destroys the SA, if you need the plain SA");
Console.WriteLine ("first save it and reload it after the call");
using (var Output = new BinaryWriter (File.Create (sa_name + ".structs"))) {
RankSelectGenericIO.Save (Output, this.newF);
PrimitiveIO<int>.WriteVector (Output, this.charT);
}
using (var Output = new BinaryWriter (File.Create (sa_name + ".samples"))) {
Output.Write ((short)sample_step);
var B = new BitStream32 ();
int numbits = (int)Math.Ceiling (Math.Log (this.SA.Length, 2));
var SA_samples = new List<int> ();
var SA_invsamples = new List<int> ();
for (int i = 0; i < this.SA.Length; i++) {
var s = this.SA[i];
if ((s + 1 == this.SA.Length) || (s % sample_step == 0)) {
B.Write (true);
SA_samples.Add (s);
SA_invsamples.Add (i);
} else {
B.Write (false);
}
}
GGMN G = new GGMN ();
G.Build (B, 8);
RankSelectGenericIO.Save (Output, G);
{
var _SA_samples = new ListIFS (numbits);
foreach (var u in SA_samples) {
_SA_samples.Add (u);
}
_SA_samples.Save (Output);
}
{
Sorting.Sort<int, int> (SA_samples, SA_invsamples);
var _SA_invsamples = new ListIFS (numbits);
foreach (var u in SA_invsamples) {
_SA_invsamples.Add (u);
}
_SA_invsamples.Save (Output);
SA_samples = null;
SA_invsamples = null;
}
}
// building bwt
using (var Output = new BinaryWriter (File.Create (sa_name + ".bwt"))) {
int alphabet_numbits = (int)Math.Ceiling (Math.Log (this.charT.Count + 1, 2));
var L = new ListIFS (alphabet_numbits);
int bwt_len = this.SA.Length;
for (int i = 0; i < bwt_len; i++) {
var v = this.SA[i];
if (v == 0) {
L.Add (0);
} else {
// Output.Write ("{0} ", (int)this.Text[v - 1]);
var c = this.Text[v - 1];
var u = GenericSearch.FindLast<int> (c, this.charT, 1, this.charT.Count);
L.Add (u);
}
}
L.Save (Output);
// for (int i = 0; i < bwt_len; i++) {
// var v = this.SA[i];
// if (v == 0) {
// Output.Write ("{0} ", -1);
// } else {
// // Output.Write ("{0} ", (int)this.Text[v - 1]);
// var c = this.Text[v - 1];
// var u = GenericSearch.FindLast<int> (c, this.charT);
// Output.Write ("{0} ", u);
// }
// }
// PrimitiveIO<byte>.WriteVector (Output, BWT);
}
// building psi
using (var Output = new BinaryWriter (File.Create (sa_name + ".psi"))) {
var INV = new int[this.SA.Length];
for (int i = 0; i < INV.Length; i++) {
INV[this.SA[i]] = i;
}
var PSI = this.SA;
for (int i = 0; i < PSI.Length; i++) {
var p = (PSI[i] + 1) % PSI.Length;
PSI[i] = INV[p];
}
PrimitiveIO<int>.WriteVector (Output, PSI);
/*Console.Write ("charT => ");
for (int i = 0; i < this.charT.Count; i++) {
Console.Write ("[{0}] ", (char)this.charT[i]);
}
Console.WriteLine ();
Console.Write ("newF => ");
for (int i = 0; i < this.newF.Count1; i++) {
Console.Write ("{0} ", this.newF.Select1(i+1));
}
Console.WriteLine ();
Console.Write ("PSI => ");
for (int i = 0; i < PSI.Length; i++) {
Console.Write ("{0} ", PSI[i]);
}
Console.WriteLine ();
*/
INV = null;
}
this.SA = null;
}
public void Encode(BitStream32 Buffer, int u)
{
if (u < 0) {
var message = String.Format ("Negative numbers are not valid for BinaryCoding, u: {0}", u);
throw new ArgumentOutOfRangeException (message);
}
int mask = 1 << this.NumBits;
if (u >= mask) {
var message = String.Format ("Number too large for {0} bits BinaryCoding, {1} >= {2}",
this.NumBits, u, mask);
throw new ArgumentOutOfRangeException (message);
}
--mask;
u &= mask;
Buffer.Write(u, this.NumBits);
}
public void Encode(BitStream32 stream, long u, long N)
{
long min = 0;
long max = N - 1;
long mid;
do {
mid = (min >> 1) + (max >> 1);
if (1L == (min & 1L & max)) {
mid++;
}
if (u <= mid) {
stream.Write (false);
max = mid;
} else {
stream.Write (true);
min = mid + 1L;
}
} while (min < max);
}
public void Build(IList<int> seq, int sigma, PermutationBuilder perm_builder, BitmapFromBitStream bitmap_builder)
{
// A counting sort construction of the permutation
var counters = new int[sigma];
foreach (var s in seq) {
if (s + 1 < sigma) {
counters [s + 1]++;
}
}
for (int i = 1; i < sigma; i++) {
counters [i] += counters [i - 1];
}
var n = seq.Count;
var P = new int[n];
for (int i = 0; i < n; i++) {
var sym = seq [i];
var pos = counters [sym];
P [pos] = i;
counters [sym] = pos + 1;
}
// the bitmap to save the lengths
var lens = new BitStream32 ();
int prevc = 0;
foreach (var c in counters) {
var len = c - prevc;
prevc = c;
lens.Write (true);
lens.Write (false, len);
}
// an additional 1 to the end, to simplify source code
lens.Write (true);
var bb_lens = new FakeBitmap (lens);
this.LENS = bitmap_builder(bb_lens);
this.PERM = perm_builder(P);
}
public void BuildWebGraph(string filename, SequenceBuilder seqbuilder, BitmapFromBitStream bitmapbuilder = null)
{
if (bitmapbuilder == null) {
bitmapbuilder = BitmapBuilders.GetGGMN_wt (12);
}
var len_stream = new BitStream32 ();
var seq = new List<int> ();
int prev_context = -1;
using (var Input = File.OpenText (filename)) {
string line;
int lineno = 0;
int counterlineno = 0;
while (true) {
{
if (lineno % 10000 == 0) {
if (counterlineno % 10 == 0) {
Console.WriteLine ();
Console.Write ("Processing lines: ");
}
++counterlineno;
Console.Write ("{0}, ", lineno);
}
++lineno;
}
line = Input.ReadLine ();
if (line == null) {
break;
}
if (line.StartsWith ("#")) {
continue;
}
var link = line.Split ('\t', ' ');
var start_node = int.Parse (link [0]);
var end_node = int.Parse (link [1]);
// on webgraph format, starting nodes are already sorted, just advance and count
if (start_node != prev_context) {
for (int diffcount = start_node - prev_context; diffcount > 0; --diffcount) {
len_stream.Write (true);
}
prev_context = start_node;
}
len_stream.Write (false);
seq.Add (end_node);
}
// a simple hack simplifying direct-neighbors's retrieval
len_stream.Write (true);
}
this.SEQ = seqbuilder (seq, prev_context + 1);
this.LENS = bitmapbuilder (new FakeBitmap (len_stream));
}
/// <summary>
/// Builds the index for the sequence
/// </summary>
public void Build(IList<int> sequence, int alphabet_size, int t = 16,
BitmapFromList rowbuilder = null, BitmapFromBitStream lenbuilder = null)
{
if (rowbuilder == null) {
rowbuilder = BitmapBuilders.GetSArray ();
}
if (lenbuilder == null) {
lenbuilder = BitmapBuilders.GetGGMN_wt (12);
}
var invindex = new IList<int>[alphabet_size];
for (int i = 0; i < alphabet_size; i++) {
invindex [i] = new List<int> ();
}
int pos = 0;
foreach (var c in sequence) {
invindex [c].Add (pos);
pos++;
}
pos = 0;
this.N = sequence.Count;
this.InvIndex = new Bitmap[alphabet_size];
var lens = new BitStream32 ();
for (int i = 0; i < alphabet_size; i++) {
if (i % 1000 == 0) {
if (i % 10000 == 0) {
Console.WriteLine ();
Console.Write ("*** InvIndexXLBSeq {0}/{1}", i, alphabet_size);
} else {
Console.Write (", {0}", i);
}
}
this.InvIndex [i] = rowbuilder (invindex [i]);
lens.Write (true);
lens.Write (false, invindex [i].Count);
invindex [i] = null;
}
lens.Write (true);
Console.WriteLine ();
Console.WriteLine ("done, now saving permutation and the Len bitmap");
this.Lens = lenbuilder (new FakeBitmap (lens));
var p = new ListGen_MRRR ();
p.Build (this.GetNotIdxPERM (), t, null);
Console.WriteLine ("done");
this.Perm = p;
}