public void Build(string sa_name, SequenceBuilder seq_builder = null, BitmapFromBitStream bitmap_builder = null)
{
if (seq_builder == null) {
seq_builder = SequenceBuilders.GetSeqXLB_DiffSetRL2_64(16, 63);
}
using (var Input = new BinaryReader (File.OpenRead (sa_name + ".structs"))) {
this.newF = RankSelectGenericIO.Load (Input);
if (bitmap_builder != null) {
var newF_stream = new BitStream32();
for (int i = 0; i < this.newF.Count; ++i) {
newF_stream.Write (this.newF.Access(i));
}
this.newF = bitmap_builder(new FakeBitmap(newF_stream));
}
int len = this.newF.Count1;
this.charT = new int[len];
// Console.WriteLine ("*****>> charT => {0} bytes", this.charT.Length * 4);
PrimitiveIO<int>.ReadFromFile (Input, len, this.charT);
}
using (var Input = new BinaryReader (File.OpenRead (sa_name + ".psi"))) {
int seqlen = this.newF.Count;
var seq = new int[seqlen];
var L = new List<int>(this.N/this.Sigma + 1);
int curr = 0;
for (int i = 1; i <= this.AlphabetSize; i++) {
int next;
if (i == this.AlphabetSize) {
next = this.newF.Count;
} else {
next = this.newF.Select1 (i + 1);
}
int len = next - curr;
L.Clear();
PrimitiveIO<int>.ReadFromFile (Input, len, L);
for (int j = 0; j < len; ++j) {
var x = L[j];
try {
seq[ x ] = i - 1;
} catch (Exception e) {
Console.WriteLine ("== i: {0}, j: {1}, x: {2}, seq-count: {3}, len: {4}",
i, j, x, seq.Length, len);
throw e;
}
}
curr = next;
}
this.SeqPsi = seq_builder(seq, this.AlphabetSize);
}
using (var Input = new BinaryReader (File.OpenRead (sa_name + ".samples"))) {
this.SA_sample_step = Input.ReadInt16 ();
this.SA_marked = RankSelectGenericIO.Load (Input);
var _samples = new ListIFS ();
_samples.Load (Input);
var _invsamples = new ListIFS ();
_invsamples.Load (Input);
this.SA_samples = _samples;
this.SA_invsamples = _invsamples;
}
}
/// <summary>
/// Suffix array built in
/// </summary>
public void Build(IList<int> text, int alphabet_size)
{
this.Text = text;
var SS = new SA_fss (text, alphabet_size);
this.A = SS.A;
this.charT = SS.charT;
this.newF = SS.newF;
}
/// <summary>
/// Suffix array built in
/// </summary>
public void Build(IList<int> text, int alphabet_size)
{
this.Text = text;
var SS = new SuffixSorter (text, alphabet_size);
SS.Sort ();
this.SA = SS.SA;
this.charT = SS.charT;
this.newF = SS.newF;
}
public WaveletInner(WaveletInner parent, bool building)
: base(parent)
{
if (building) {
this.B = new FakeBitmap ();
}
this.Left = null;
this.Right = null;
}
public override void AssertEquality(IRankSelect obj)
{
var other = obj as SArray;
if (this.N != other.N) {
throw new ArgumentException (String.Format ("SArray.N inequality. this.N {0}, other.N: {1}",
this.N, other.N));
}
this.H.AssertEquality (other.H);
Assertions.AssertIList<int> (this.L, other.L, "SArray.L");
}
public void AssertEquality(IRankSelect obj)
{
var other = obj as DArray;
this.BaseIndex.AssertEquality (other.BaseIndex);
this.IsLargeBlock.AssertEquality (other.IsLargeBlock);
Assertions.AssertIList<int> (this.PosAbs, other.PosAbs, "DArray.PosAbs");
Assertions.AssertIList<int> (this.SavedPos, other.SavedPos, "DArray.SavedPos");
if (this.B != other.B) {
throw new ArgumentException ("DArray inequality on B");
}
}
public void Build(string sa_name, BitmapFromList create_bitmap)
{
using (var Input = new BinaryReader (File.OpenRead (sa_name + ".structs"))) {
this.newF = RankSelectGenericIO.Load (Input);
int len = this.newF.Count1;
this.charT = new int[len];
// Console.WriteLine ("*****>> charT => {0} bytes", this.charT.Length * 4);
PrimitiveIO<int>.ReadFromFile (Input, len, this.charT);
}
using (var Input = new BinaryReader (File.OpenRead (sa_name + ".psi"))) {
this.InvIndex = new IRankSelect[this.AlphabetSize];
int curr = 0;
for (int i = 1; i <= this.AlphabetSize; i++) {
int next;
if (i == this.AlphabetSize) {
next = this.newF.Count;
} else {
next = this.newF.Select1 (i + 1);
}
int len = next - curr;
var L = new int[len];
PrimitiveIO<int>.ReadFromFile (Input, len, L);
this.InvIndex [i - 1] = create_bitmap (L);
curr = next;
}
}
using (var Input = new BinaryReader (File.OpenRead (sa_name + ".samples"))) {
this.SA_sample_step = Input.ReadInt16 ();
this.SA_marked = RankSelectGenericIO.Load (Input);
var _samples = new ListIFS ();
_samples.Load (Input);
var _invsamples = new ListIFS ();
_invsamples.Load (Input);
this.SA_samples = _samples;
this.SA_invsamples = _invsamples;
}
}
public void Load(string basename)
{
using (var Input = new BinaryReader (File.OpenRead (basename + ".idx"))) {
this.newF = RankSelectGenericIO.Load (Input);
this.charT = new int[this.newF.Count1];
PrimitiveIO<int>.ReadFromFile (Input, this.charT.Length, this.charT);
}
using (var Input = new BinaryReader (File.OpenRead (basename + ".psi"))) {
this.InvIndex = new IRankSelect[this.AlphabetSize];
for (int i = 0; i < this.AlphabetSize; i++) {
this.InvIndex [i] = RankSelectGenericIO.Load (Input);
}
}
using (var Input = new BinaryReader (File.OpenRead (basename + ".samples"))) {
this.SA_sample_step = Input.ReadInt16 ();
this.SA_marked = RankSelectGenericIO.Load (Input);
var _samples = new ListIFS ();
_samples.Load (Input);
var _invsamples = new ListIFS ();
_invsamples.Load (Input);
this.SA_samples = _samples;
this.SA_invsamples = _invsamples;
}
}
public void AssertEquality(IRankSelect other)
{
throw new System.NotImplementedException ();
}
public void Build(string sa_name)
{
using (var Input = new BinaryReader (File.OpenRead (sa_name + ".structs"))) {
this.newF = RankSelectGenericIO.Load (Input);
int len = this.newF.Count1;
this.charT = new int[len];
PrimitiveIO<int>.ReadFromFile (Input, len, this.charT);
}
using (var Input = new BinaryReader (File.OpenRead (sa_name + ".bwt"))) {
var L = new ListIFS ();
L.Load (Input);
this.seqIndex = this.SeqBuilder (L, this.charT.Length);
}
using (var Input = new BinaryReader (File.OpenRead (sa_name + ".samples"))) {
this.SA_sample_step = Input.ReadInt16 ();
this.SA_marked = RankSelectGenericIO.Load (Input);
var _samples = new ListIFS ();
_samples.Load (Input);
var _invsamples = new ListIFS ();
_invsamples.Load (Input);
this.SA_samples = _samples;
this.SA_invsamples = _invsamples;
}
}
public void Build(string listname, int qsize, int symsize)
{
this.Q = qsize;
this.SymbolSize = symsize;
this.Name = listname;
int linenum = 0;
var lens = new List<int>();
var D = new List<byte>();
lens.Add(0);
foreach (var filename in File.ReadAllLines (listname)) {
linenum++;
Console.WriteLine ("**** Loading line-number: {0}, file: {1}", linenum, filename);
var data = BinQ8HammingSpace.LoadObjectFromFile (filename, false);
//D.Capacity += data.Count;
foreach (var b in data) {
D.Add(b);
}
lens.Add(lens[lens.Count-1]+data.Count);
}
this.LENS = BitmapBuilders.GetSArray().Invoke(lens);
this.Data = D;
}
public abstract void AssertEquality(IRankSelect other);
public void Load(BinaryReader Input)
{
this.PERM = new SuccRL2CyclicPerms_MRRR ();
this.PERM.Load (Input);
this.LENS = RankSelectGenericIO.Load (Input);
}
public void Load(BinaryReader Input)
{
this.n = Input.ReadInt32 ();
this.sigma = Input.ReadInt32 ();
var c = Input.ReadInt32 ();
this.perms = new IPermutation[c];
for (int i = 0; i < c; i++) {
this.perms [i] = PermutationGenericIO.Load (Input);
}
this.B = RankSelectGenericIO.Load (Input);
this.X = RankSelectGenericIO.Load (Input);
/*var len = Input.ReadInt32 ();
this.Xacc = new int[len];
PrimitiveIO<int>.ReadFromFile (Input, len, this.Xacc);*/
this.compute_num_blocks ();
}
public override void AssertEquality(IRankSelect obj)
{
var other = obj as RankSelectBlocks;
if (this.SuperBlockSize != other.SuperBlockSize) {
throw new ArgumentException (String.Format ("RankSelectBlocks.SuperBlockSize inequality"));
}
if (this.BlockSize != other.BlockSize) {
throw new ArgumentException (String.Format ("RankSelectBlocks.SuperBlockSize inequality"));
}
if (this.bits_per_block != other.bits_per_block) {
throw new ArgumentException (String.Format ("RankSelectBlocks.bits_per_block inequality"));
}
if (this.bits_per_super_block != other.bits_per_super_block) {
throw new ArgumentException (String.Format ("RankSelectBlocks.bits_per_super_block inequality"));
}
Assertions.AssertIList<short> (this.Blocks, other.Blocks, "RankSelectBlocks.Blocks");
Assertions.AssertIList<int> (this.SuperBlocks, other.SuperBlocks, "RankSelectBlocks.SuperBlocks");
Assertions.AssertIList<uint> (this.BitBlocks, other.BitBlocks, "RankSelectBlocks.BitBlocks");
if (this.N != other.N) {
throw new ArgumentException ("RankSelectBlocks.N inequality");
}
}
public void Build(IList<int> seq, int sigma)
{
// A counting sort construction of the permutation
var counters = new int[sigma];
foreach (var s in seq) {
if (s + 1 < sigma) {
counters [s + 1]++;
}
}
for (int i = 1; i < sigma; i++) {
counters [i] += counters [i - 1];
}
var n = seq.Count;
var P = new int[n];
for (int i = 0; i < n; i++) {
var sym = seq [i];
var pos = counters [sym];
P [pos] = i;
counters [sym] = pos + 1;
}
// the bitmap to save the lengths
var lens = new BitStream32 ();
int prevc = 0;
foreach (var c in counters) {
var len = c - prevc;
prevc = c;
lens.Write (true);
lens.Write (false, len);
}
// an additional 1 to the end, to simplify source code
lens.Write (true);
var bb_lens = new FakeBitmap (lens);
this.LENS = this.BitmapBuilder (bb_lens);
this.PERM = this.PermBuilder (P);
}
public void Load(BinaryReader Input)
{
this.PERM = PermutationGenericIO.Load (Input);
this.LENS = RankSelectGenericIO.Load (Input);
}
public override void AssertEquality(IRankSelect _other)
{
DiffSet other = _other as DiffSet;
if (this.N != other.N) {
throw new ArgumentException ("DiffSet N difference");
}
if (this.M != other.M) {
throw new ArgumentException ("DiffSet M difference");
}
if (this.B != other.B) {
throw new ArgumentException ("DiffSet B difference");
}
Assertions.AssertIList<int> (this.Samples, other.Samples, "DiffSet Samples difference");
Assertions.AssertIList<long> (this.Offsets, other.Offsets, "DiffSet Offsets difference");
this.Stream.AssertEquality (other.Stream);
}
public void Build(IList<int> seq, int sigma, PermutationBuilder perm_builder, BitmapFromBitStream bitmap_builder)
{
// NOTE: Please check sigma <=> BlockSize in this method
this.sigma = sigma;
this.n = seq.Count;
var B_stream = new BitStream32 ();
var X_stream = new BitStream32[ sigma ];
for (int i = 0; i < sigma; i++) {
X_stream [i] = new BitStream32 ();
}
var lists = new List<int>[sigma];
for (int i = 0; i < sigma; i++) {
lists [i] = new List<int> ();
}
int num_blocks = (int)Math.Ceiling (this.n * 1.0 / this.sigma);
this.perms = new List<IPermutation> (num_blocks);
for (int i = 0; i < this.n; i+= this.sigma) {
// writing block separators
foreach (var b in X_stream) {
b.Write (true);
}
// clearing perm B
// selecting block size
int s = Math.Min (this.n - i, this.sigma);
this.BuildPermInvIndex (seq, i, s, lists);
var P = new List<int> (s);
for (int j = 0; j < this.sigma; j++) {
var c = lists [j].Count;
B_stream.Write (false);
if (c > 0) {
X_stream [j].Write (false, c);
B_stream.Write (true, c);
foreach (var u in lists[j]) {
P.Add (u);
}
}
}
var _perm = perm_builder(P);
this.perms.Add (_perm);
}
var _X_stream = X_stream [0];
for (int i = 1; i < X_stream.Length; i++) {
var _X_curr = X_stream [i];
for (int j = 0; j < _X_curr.CountBits; j++) {
// esto se podria hace por entero en lugar de bit
_X_stream.Write (_X_curr [j]);
}
}
// If we write a zero at the end of the streams the code is simplified
_X_stream.Write (true);
B_stream.Write (false);
this.B = bitmap_builder (new FakeBitmap (B_stream));
this.X = bitmap_builder (new FakeBitmap (_X_stream));
this.compute_num_blocks ();
}
public override void AssertEquality(IRankSelect obj)
{
var other = obj as GGMN;
if (this.N != other.N) {
throw new ArgumentException (String.Format ("GNBitmap.N inequality"));
}
if (this.B != other.B) {
throw new ArgumentException (String.Format ("GNBitmap.B inequality"));
}
Assertions.AssertIList<uint> (this.BitBlocks, other.BitBlocks, "GNBitmap.Bitmap");
Assertions.AssertIList<uint> (this.Abs, other.Abs, "GNBitmap.Abs");
}
/// <summary>
/// Build the specified seq, sigma and t.
/// </summary>
public void Build(IList<int> seq, int sigma, short t = 16, IIEncoder32 rl2_coder = null, short rl2_block_size = 127)
{
// A counting sort construction of the permutation
var counters = new int[sigma];
foreach (var s in seq) {
if (s + 1 < sigma) {
counters [s + 1]++;
}
}
for (int i = 1; i < sigma; i++) {
counters [i] += counters [i - 1];
}
var n = seq.Count;
var P = new int[n];
for (int i = 0; i < n; i++) {
var sym = seq [i];
var pos = counters [sym];
P [pos] = i;
counters [sym] = pos + 1;
}
// the bitmap to save the lengths
var lens = new BitStream32 ();
int prevc = 0;
foreach (var c in counters) {
var len = c - prevc;
prevc = c;
lens.Write (true);
lens.Write (false, len);
}
// an additional 1 to the end, to simplify source code
lens.Write (true);
var bb_lens = new FakeBitmap (lens);
this.LENS = BitmapBuilder (bb_lens);
this.PERM = new SuccRL2CyclicPerms_MRRR ();
if (rl2_coder == null) {
rl2_coder = new EliasGamma32 ();
}
var build_params = new SuccRL2CyclicPerms_MRRR.BuildParams (rl2_coder, rl2_block_size);
this.PERM.Build (P, t, build_params);
}
public override void AssertEquality(IRankSelect other)
{
throw new NotSupportedException ();
}
public void Load(BinaryReader Input)
{
this.LENS = RankSelectGenericIO.Load(Input);
this.SymbolSize = Input.ReadInt32();
this.Q = Input.ReadInt32();
var len = Input.ReadInt32 ();
this.Data = new byte[ len ];
PrimitiveIO<byte>.ReadFromFile(Input, len, this.Data);
this.Name = Input.ReadString();
}
public void Build(IRankSelect RS, short blockSize)
{
// we need to store n bits, this is not the best solution!!
BitStream32 B = new BitStream32 ();
for (int i = 0, count = RS.Count; i < count; i++) {
B.Write (RS.Access(i));
}
this.Build (B, blockSize);
}
/// <summary>
/// Read the database from a listing file (one filename per line)
/// </summary>
public void Build(string filename, BitmapFromBitStream len_builder = null)
{
Console.WriteLine ("****** Reading database from list of files");
this.Name = filename;
var NAMES = File.ReadAllLines (filename);
int counter = 0;
var data_stream = new List<byte> ();
var lens_stream = new BitStream32 ();
foreach (var s in NAMES) {
++counter;
if (s.Length == 0) {
continue;
}
if (counter % 1000 == 0) {
Console.WriteLine ("*** Processing docid {0}/{1} (adv: {2:0.000}%): '{3}'",
counter, NAMES.Length, counter*100.0/NAMES.Length, s);
}
var data = (IList<byte>)this.Parse (s, true);
if (data.Count == 0) {
throw new ArgumentException(String.Format("AFP files must not be empty: {0}", s));
}
lens_stream.Write (true);
lens_stream.Write (false, data.Count-1);
data_stream.Capacity += data.Count;
foreach (var b in data) {
data_stream.Add (b);
}
}
lens_stream.Write(true);
if (len_builder == null) {
len_builder = BitmapBuilders.GetGGMN_wt (12);
}
this.LENS = len_builder (new FakeBitmap (lens_stream));
this.DATA = data_stream;
}
public void Load(string basename)
{
using (var Input = new BinaryReader (File.OpenRead (basename + ".structs"))) {
this.newF = RankSelectGenericIO.Load (Input);
this.charT = new int[this.newF.Count1];
PrimitiveIO<int>.ReadFromFile (Input, this.charT.Length, this.charT);
}
// this.seqIndex = new WaveletTree ();
// this.seqIndex.Load (Input);
using (var Input = new BinaryReader (File.OpenRead (basename + ".bwt-index"))) {
this.seqIndex = RankSelectSeqGenericIO.Load (Input);
}
using (var Input = new BinaryReader (File.OpenRead (basename + ".structs-samples"))) {
this.SA_sample_step = Input.ReadInt16 ();
this.SA_marked = RankSelectGenericIO.Load (Input);
}
using (var Input = new BinaryReader (File.OpenRead (basename + ".samples"))) {
var _samples = new ListIFS ();
_samples.Load (Input);
var _invsamples = new ListIFS ();
_invsamples.Load (Input);
this.SA_samples = _samples;
this.SA_invsamples = _invsamples;
}
}
public override void AssertEquality(IRankSelect _other)
{
RRR other = _other as RRR;
if (other == null) {
throw new ArgumentNullException ("RRR Other should be a RRR object too");
}
if (this.N != other.N) {
throw new ArgumentNullException ("RRR Inequality on N");
}
if (this.BlockSize != other.BlockSize) {
throw new ArgumentException ("RRR Inequality on BlockSize");
}
Assertions.AssertIList<int> (this.Klasses, other.Klasses, "RRR Classes");
Assertions.AssertIList<int> (this.AbsRank, other.AbsRank, "RRR AbsRank");
Assertions.AssertIList<int> (this.AbsOffset, other.AbsOffset, "RRR AbsOffset");
}
public void Build(string out_filename, IList<IList<byte>> data_list, BitmapFromBitStream len_builder = null)
{
this.Name = out_filename;
int counter = 0;
var data_stream = new List<byte> ();
var lens_stream = new BitStream32 ();
foreach (var data in data_list) {
++counter;
if (counter % 1000 == 0) {
Console.WriteLine ("*** Processing docid {0}/{1} (adv: {2:0.000}%)",
counter, data_list.Count, counter*100.0/data_list.Count);
}
lens_stream.Write (true);
lens_stream.Write (false, data.Count-1);
// data_stream.Capacity += data.Count;
foreach (var b in data) {
data_stream.Add (b);
}
}
lens_stream.Write(true);
if (len_builder == null) {
len_builder = BitmapBuilders.GetGGMN_wt (12);
}
this.LENS = len_builder (new FakeBitmap (lens_stream));
this.DATA = data_stream;
}
public override void AssertEquality(IRankSelect _other)
{
var other = (PlainSortedList)_other;
if (this.N != other.N) {
throw new ArgumentException ("Parameter PlainSortedList.N have differences");
}
Assertions.AssertIList<int> (this.sortedList, other.sortedList, "PlainSortedList.sortedList");
}
public sort_pair(IRankSelect invlist, string word)
{
this.invlist = invlist;
this.word = word;
}
