public void Build(string sa_name, SequenceBuilder seq_builder = null, BitmapFromBitStream bitmap_builder = null) { if (seq_builder == null) { seq_builder = SequenceBuilders.GetSeqXLB_DiffSetRL2_64(16, 63); } using (var Input = new BinaryReader (File.OpenRead (sa_name + ".structs"))) { this.newF = RankSelectGenericIO.Load (Input); if (bitmap_builder != null) { var newF_stream = new BitStream32(); for (int i = 0; i < this.newF.Count; ++i) { newF_stream.Write (this.newF.Access(i)); } this.newF = bitmap_builder(new FakeBitmap(newF_stream)); } int len = this.newF.Count1; this.charT = new int[len]; // Console.WriteLine ("*****>> charT => {0} bytes", this.charT.Length * 4); PrimitiveIO<int>.ReadFromFile (Input, len, this.charT); } using (var Input = new BinaryReader (File.OpenRead (sa_name + ".psi"))) { int seqlen = this.newF.Count; var seq = new int[seqlen]; var L = new List<int>(this.N/this.Sigma + 1); int curr = 0; for (int i = 1; i <= this.AlphabetSize; i++) { int next; if (i == this.AlphabetSize) { next = this.newF.Count; } else { next = this.newF.Select1 (i + 1); } int len = next - curr; L.Clear(); PrimitiveIO<int>.ReadFromFile (Input, len, L); for (int j = 0; j < len; ++j) { var x = L[j]; try { seq[ x ] = i - 1; } catch (Exception e) { Console.WriteLine ("== i: {0}, j: {1}, x: {2}, seq-count: {3}, len: {4}", i, j, x, seq.Length, len); throw e; } } curr = next; } this.SeqPsi = seq_builder(seq, this.AlphabetSize); } using (var Input = new BinaryReader (File.OpenRead (sa_name + ".samples"))) { this.SA_sample_step = Input.ReadInt16 (); this.SA_marked = RankSelectGenericIO.Load (Input); var _samples = new ListIFS (); _samples.Load (Input); var _invsamples = new ListIFS (); _invsamples.Load (Input); this.SA_samples = _samples; this.SA_invsamples = _invsamples; } }
/// <summary> /// Suffix array built in /// </summary> public void Build(IList<int> text, int alphabet_size) { this.Text = text; var SS = new SA_fss (text, alphabet_size); this.A = SS.A; this.charT = SS.charT; this.newF = SS.newF; }
/// <summary> /// Suffix array built in /// </summary> public void Build(IList<int> text, int alphabet_size) { this.Text = text; var SS = new SuffixSorter (text, alphabet_size); SS.Sort (); this.SA = SS.SA; this.charT = SS.charT; this.newF = SS.newF; }
public WaveletInner(WaveletInner parent, bool building) : base(parent) { if (building) { this.B = new FakeBitmap (); } this.Left = null; this.Right = null; }
public override void AssertEquality(IRankSelect obj) { var other = obj as SArray; if (this.N != other.N) { throw new ArgumentException (String.Format ("SArray.N inequality. this.N {0}, other.N: {1}", this.N, other.N)); } this.H.AssertEquality (other.H); Assertions.AssertIList<int> (this.L, other.L, "SArray.L"); }
public void AssertEquality(IRankSelect obj) { var other = obj as DArray; this.BaseIndex.AssertEquality (other.BaseIndex); this.IsLargeBlock.AssertEquality (other.IsLargeBlock); Assertions.AssertIList<int> (this.PosAbs, other.PosAbs, "DArray.PosAbs"); Assertions.AssertIList<int> (this.SavedPos, other.SavedPos, "DArray.SavedPos"); if (this.B != other.B) { throw new ArgumentException ("DArray inequality on B"); } }
public void Build(string sa_name, BitmapFromList create_bitmap) { using (var Input = new BinaryReader (File.OpenRead (sa_name + ".structs"))) { this.newF = RankSelectGenericIO.Load (Input); int len = this.newF.Count1; this.charT = new int[len]; // Console.WriteLine ("*****>> charT => {0} bytes", this.charT.Length * 4); PrimitiveIO<int>.ReadFromFile (Input, len, this.charT); } using (var Input = new BinaryReader (File.OpenRead (sa_name + ".psi"))) { this.InvIndex = new IRankSelect[this.AlphabetSize]; int curr = 0; for (int i = 1; i <= this.AlphabetSize; i++) { int next; if (i == this.AlphabetSize) { next = this.newF.Count; } else { next = this.newF.Select1 (i + 1); } int len = next - curr; var L = new int[len]; PrimitiveIO<int>.ReadFromFile (Input, len, L); this.InvIndex [i - 1] = create_bitmap (L); curr = next; } } using (var Input = new BinaryReader (File.OpenRead (sa_name + ".samples"))) { this.SA_sample_step = Input.ReadInt16 (); this.SA_marked = RankSelectGenericIO.Load (Input); var _samples = new ListIFS (); _samples.Load (Input); var _invsamples = new ListIFS (); _invsamples.Load (Input); this.SA_samples = _samples; this.SA_invsamples = _invsamples; } }
public void Load(string basename) { using (var Input = new BinaryReader (File.OpenRead (basename + ".idx"))) { this.newF = RankSelectGenericIO.Load (Input); this.charT = new int[this.newF.Count1]; PrimitiveIO<int>.ReadFromFile (Input, this.charT.Length, this.charT); } using (var Input = new BinaryReader (File.OpenRead (basename + ".psi"))) { this.InvIndex = new IRankSelect[this.AlphabetSize]; for (int i = 0; i < this.AlphabetSize; i++) { this.InvIndex [i] = RankSelectGenericIO.Load (Input); } } using (var Input = new BinaryReader (File.OpenRead (basename + ".samples"))) { this.SA_sample_step = Input.ReadInt16 (); this.SA_marked = RankSelectGenericIO.Load (Input); var _samples = new ListIFS (); _samples.Load (Input); var _invsamples = new ListIFS (); _invsamples.Load (Input); this.SA_samples = _samples; this.SA_invsamples = _invsamples; } }
public void AssertEquality(IRankSelect other) { throw new System.NotImplementedException (); }
public void Build(string sa_name) { using (var Input = new BinaryReader (File.OpenRead (sa_name + ".structs"))) { this.newF = RankSelectGenericIO.Load (Input); int len = this.newF.Count1; this.charT = new int[len]; PrimitiveIO<int>.ReadFromFile (Input, len, this.charT); } using (var Input = new BinaryReader (File.OpenRead (sa_name + ".bwt"))) { var L = new ListIFS (); L.Load (Input); this.seqIndex = this.SeqBuilder (L, this.charT.Length); } using (var Input = new BinaryReader (File.OpenRead (sa_name + ".samples"))) { this.SA_sample_step = Input.ReadInt16 (); this.SA_marked = RankSelectGenericIO.Load (Input); var _samples = new ListIFS (); _samples.Load (Input); var _invsamples = new ListIFS (); _invsamples.Load (Input); this.SA_samples = _samples; this.SA_invsamples = _invsamples; } }
public void Build(string listname, int qsize, int symsize) { this.Q = qsize; this.SymbolSize = symsize; this.Name = listname; int linenum = 0; var lens = new List<int>(); var D = new List<byte>(); lens.Add(0); foreach (var filename in File.ReadAllLines (listname)) { linenum++; Console.WriteLine ("**** Loading line-number: {0}, file: {1}", linenum, filename); var data = BinQ8HammingSpace.LoadObjectFromFile (filename, false); //D.Capacity += data.Count; foreach (var b in data) { D.Add(b); } lens.Add(lens[lens.Count-1]+data.Count); } this.LENS = BitmapBuilders.GetSArray().Invoke(lens); this.Data = D; }
public abstract void AssertEquality(IRankSelect other);
public void Load(BinaryReader Input) { this.PERM = new SuccRL2CyclicPerms_MRRR (); this.PERM.Load (Input); this.LENS = RankSelectGenericIO.Load (Input); }
public void Load(BinaryReader Input) { this.n = Input.ReadInt32 (); this.sigma = Input.ReadInt32 (); var c = Input.ReadInt32 (); this.perms = new IPermutation[c]; for (int i = 0; i < c; i++) { this.perms [i] = PermutationGenericIO.Load (Input); } this.B = RankSelectGenericIO.Load (Input); this.X = RankSelectGenericIO.Load (Input); /*var len = Input.ReadInt32 (); this.Xacc = new int[len]; PrimitiveIO<int>.ReadFromFile (Input, len, this.Xacc);*/ this.compute_num_blocks (); }
public override void AssertEquality(IRankSelect obj) { var other = obj as RankSelectBlocks; if (this.SuperBlockSize != other.SuperBlockSize) { throw new ArgumentException (String.Format ("RankSelectBlocks.SuperBlockSize inequality")); } if (this.BlockSize != other.BlockSize) { throw new ArgumentException (String.Format ("RankSelectBlocks.SuperBlockSize inequality")); } if (this.bits_per_block != other.bits_per_block) { throw new ArgumentException (String.Format ("RankSelectBlocks.bits_per_block inequality")); } if (this.bits_per_super_block != other.bits_per_super_block) { throw new ArgumentException (String.Format ("RankSelectBlocks.bits_per_super_block inequality")); } Assertions.AssertIList<short> (this.Blocks, other.Blocks, "RankSelectBlocks.Blocks"); Assertions.AssertIList<int> (this.SuperBlocks, other.SuperBlocks, "RankSelectBlocks.SuperBlocks"); Assertions.AssertIList<uint> (this.BitBlocks, other.BitBlocks, "RankSelectBlocks.BitBlocks"); if (this.N != other.N) { throw new ArgumentException ("RankSelectBlocks.N inequality"); } }
public void Build(IList<int> seq, int sigma) { // A counting sort construction of the permutation var counters = new int[sigma]; foreach (var s in seq) { if (s + 1 < sigma) { counters [s + 1]++; } } for (int i = 1; i < sigma; i++) { counters [i] += counters [i - 1]; } var n = seq.Count; var P = new int[n]; for (int i = 0; i < n; i++) { var sym = seq [i]; var pos = counters [sym]; P [pos] = i; counters [sym] = pos + 1; } // the bitmap to save the lengths var lens = new BitStream32 (); int prevc = 0; foreach (var c in counters) { var len = c - prevc; prevc = c; lens.Write (true); lens.Write (false, len); } // an additional 1 to the end, to simplify source code lens.Write (true); var bb_lens = new FakeBitmap (lens); this.LENS = this.BitmapBuilder (bb_lens); this.PERM = this.PermBuilder (P); }
public void Load(BinaryReader Input) { this.PERM = PermutationGenericIO.Load (Input); this.LENS = RankSelectGenericIO.Load (Input); }
public override void AssertEquality(IRankSelect _other) { DiffSet other = _other as DiffSet; if (this.N != other.N) { throw new ArgumentException ("DiffSet N difference"); } if (this.M != other.M) { throw new ArgumentException ("DiffSet M difference"); } if (this.B != other.B) { throw new ArgumentException ("DiffSet B difference"); } Assertions.AssertIList<int> (this.Samples, other.Samples, "DiffSet Samples difference"); Assertions.AssertIList<long> (this.Offsets, other.Offsets, "DiffSet Offsets difference"); this.Stream.AssertEquality (other.Stream); }
public void Build(IList<int> seq, int sigma, PermutationBuilder perm_builder, BitmapFromBitStream bitmap_builder) { // NOTE: Please check sigma <=> BlockSize in this method this.sigma = sigma; this.n = seq.Count; var B_stream = new BitStream32 (); var X_stream = new BitStream32[ sigma ]; for (int i = 0; i < sigma; i++) { X_stream [i] = new BitStream32 (); } var lists = new List<int>[sigma]; for (int i = 0; i < sigma; i++) { lists [i] = new List<int> (); } int num_blocks = (int)Math.Ceiling (this.n * 1.0 / this.sigma); this.perms = new List<IPermutation> (num_blocks); for (int i = 0; i < this.n; i+= this.sigma) { // writing block separators foreach (var b in X_stream) { b.Write (true); } // clearing perm B // selecting block size int s = Math.Min (this.n - i, this.sigma); this.BuildPermInvIndex (seq, i, s, lists); var P = new List<int> (s); for (int j = 0; j < this.sigma; j++) { var c = lists [j].Count; B_stream.Write (false); if (c > 0) { X_stream [j].Write (false, c); B_stream.Write (true, c); foreach (var u in lists[j]) { P.Add (u); } } } var _perm = perm_builder(P); this.perms.Add (_perm); } var _X_stream = X_stream [0]; for (int i = 1; i < X_stream.Length; i++) { var _X_curr = X_stream [i]; for (int j = 0; j < _X_curr.CountBits; j++) { // esto se podria hace por entero en lugar de bit _X_stream.Write (_X_curr [j]); } } // If we write a zero at the end of the streams the code is simplified _X_stream.Write (true); B_stream.Write (false); this.B = bitmap_builder (new FakeBitmap (B_stream)); this.X = bitmap_builder (new FakeBitmap (_X_stream)); this.compute_num_blocks (); }
public override void AssertEquality(IRankSelect obj) { var other = obj as GGMN; if (this.N != other.N) { throw new ArgumentException (String.Format ("GNBitmap.N inequality")); } if (this.B != other.B) { throw new ArgumentException (String.Format ("GNBitmap.B inequality")); } Assertions.AssertIList<uint> (this.BitBlocks, other.BitBlocks, "GNBitmap.Bitmap"); Assertions.AssertIList<uint> (this.Abs, other.Abs, "GNBitmap.Abs"); }
/// <summary> /// Build the specified seq, sigma and t. /// </summary> public void Build(IList<int> seq, int sigma, short t = 16, IIEncoder32 rl2_coder = null, short rl2_block_size = 127) { // A counting sort construction of the permutation var counters = new int[sigma]; foreach (var s in seq) { if (s + 1 < sigma) { counters [s + 1]++; } } for (int i = 1; i < sigma; i++) { counters [i] += counters [i - 1]; } var n = seq.Count; var P = new int[n]; for (int i = 0; i < n; i++) { var sym = seq [i]; var pos = counters [sym]; P [pos] = i; counters [sym] = pos + 1; } // the bitmap to save the lengths var lens = new BitStream32 (); int prevc = 0; foreach (var c in counters) { var len = c - prevc; prevc = c; lens.Write (true); lens.Write (false, len); } // an additional 1 to the end, to simplify source code lens.Write (true); var bb_lens = new FakeBitmap (lens); this.LENS = BitmapBuilder (bb_lens); this.PERM = new SuccRL2CyclicPerms_MRRR (); if (rl2_coder == null) { rl2_coder = new EliasGamma32 (); } var build_params = new SuccRL2CyclicPerms_MRRR.BuildParams (rl2_coder, rl2_block_size); this.PERM.Build (P, t, build_params); }
public override void AssertEquality(IRankSelect other) { throw new NotSupportedException (); }
public void Load(BinaryReader Input) { this.LENS = RankSelectGenericIO.Load(Input); this.SymbolSize = Input.ReadInt32(); this.Q = Input.ReadInt32(); var len = Input.ReadInt32 (); this.Data = new byte[ len ]; PrimitiveIO<byte>.ReadFromFile(Input, len, this.Data); this.Name = Input.ReadString(); }
public void Build(IRankSelect RS, short blockSize) { // we need to store n bits, this is not the best solution!! BitStream32 B = new BitStream32 (); for (int i = 0, count = RS.Count; i < count; i++) { B.Write (RS.Access(i)); } this.Build (B, blockSize); }
/// <summary> /// Read the database from a listing file (one filename per line) /// </summary> public void Build(string filename, BitmapFromBitStream len_builder = null) { Console.WriteLine ("****** Reading database from list of files"); this.Name = filename; var NAMES = File.ReadAllLines (filename); int counter = 0; var data_stream = new List<byte> (); var lens_stream = new BitStream32 (); foreach (var s in NAMES) { ++counter; if (s.Length == 0) { continue; } if (counter % 1000 == 0) { Console.WriteLine ("*** Processing docid {0}/{1} (adv: {2:0.000}%): '{3}'", counter, NAMES.Length, counter*100.0/NAMES.Length, s); } var data = (IList<byte>)this.Parse (s, true); if (data.Count == 0) { throw new ArgumentException(String.Format("AFP files must not be empty: {0}", s)); } lens_stream.Write (true); lens_stream.Write (false, data.Count-1); data_stream.Capacity += data.Count; foreach (var b in data) { data_stream.Add (b); } } lens_stream.Write(true); if (len_builder == null) { len_builder = BitmapBuilders.GetGGMN_wt (12); } this.LENS = len_builder (new FakeBitmap (lens_stream)); this.DATA = data_stream; }
public void Load(string basename) { using (var Input = new BinaryReader (File.OpenRead (basename + ".structs"))) { this.newF = RankSelectGenericIO.Load (Input); this.charT = new int[this.newF.Count1]; PrimitiveIO<int>.ReadFromFile (Input, this.charT.Length, this.charT); } // this.seqIndex = new WaveletTree (); // this.seqIndex.Load (Input); using (var Input = new BinaryReader (File.OpenRead (basename + ".bwt-index"))) { this.seqIndex = RankSelectSeqGenericIO.Load (Input); } using (var Input = new BinaryReader (File.OpenRead (basename + ".structs-samples"))) { this.SA_sample_step = Input.ReadInt16 (); this.SA_marked = RankSelectGenericIO.Load (Input); } using (var Input = new BinaryReader (File.OpenRead (basename + ".samples"))) { var _samples = new ListIFS (); _samples.Load (Input); var _invsamples = new ListIFS (); _invsamples.Load (Input); this.SA_samples = _samples; this.SA_invsamples = _invsamples; } }
public override void AssertEquality(IRankSelect _other) { RRR other = _other as RRR; if (other == null) { throw new ArgumentNullException ("RRR Other should be a RRR object too"); } if (this.N != other.N) { throw new ArgumentNullException ("RRR Inequality on N"); } if (this.BlockSize != other.BlockSize) { throw new ArgumentException ("RRR Inequality on BlockSize"); } Assertions.AssertIList<int> (this.Klasses, other.Klasses, "RRR Classes"); Assertions.AssertIList<int> (this.AbsRank, other.AbsRank, "RRR AbsRank"); Assertions.AssertIList<int> (this.AbsOffset, other.AbsOffset, "RRR AbsOffset"); }
public void Build(string out_filename, IList<IList<byte>> data_list, BitmapFromBitStream len_builder = null) { this.Name = out_filename; int counter = 0; var data_stream = new List<byte> (); var lens_stream = new BitStream32 (); foreach (var data in data_list) { ++counter; if (counter % 1000 == 0) { Console.WriteLine ("*** Processing docid {0}/{1} (adv: {2:0.000}%)", counter, data_list.Count, counter*100.0/data_list.Count); } lens_stream.Write (true); lens_stream.Write (false, data.Count-1); // data_stream.Capacity += data.Count; foreach (var b in data) { data_stream.Add (b); } } lens_stream.Write(true); if (len_builder == null) { len_builder = BitmapBuilders.GetGGMN_wt (12); } this.LENS = len_builder (new FakeBitmap (lens_stream)); this.DATA = data_stream; }
public override void AssertEquality(IRankSelect _other) { var other = (PlainSortedList)_other; if (this.N != other.N) { throw new ArgumentException ("Parameter PlainSortedList.N have differences"); } Assertions.AssertIList<int> (this.sortedList, other.sortedList, "PlainSortedList.sortedList"); }
public sort_pair(IRankSelect invlist, string word) { this.invlist = invlist; this.word = word; }