public override SortedDocValues GetSorted(FieldInfo field) { FSTEntry entry = Fsts[field.Number]; FST <long?> instance; lock (this) { if (!FstInstances.TryGetValue(field.Number, out instance)) { Data.Seek(entry.Offset); instance = new FST <long?>(Data, PositiveIntOutputs.Singleton); RamBytesUsed_Renamed.AddAndGet(instance.SizeInBytes()); FstInstances[field.Number] = instance; } } var docToOrd = GetNumeric(field); var fst = instance; // per-thread resources var @in = fst.BytesReader; var firstArc = new FST.Arc <long?>(); var scratchArc = new FST.Arc <long?>(); var scratchInts = new IntsRef(); var fstEnum = new BytesRefFSTEnum <long?>(fst); return(new SortedDocValuesAnonymousInnerClassHelper(entry, docToOrd, fst, @in, firstArc, scratchArc, scratchInts, fstEnum)); }
public override SortedSetDocValues GetSortedSet(FieldInfo field) { FSTEntry entry = Fsts[field.Number]; if (entry.NumOrds == 0) { return(DocValues.EMPTY_SORTED_SET); // empty FST! } FST <long?> instance; lock (this) { if (!FstInstances.TryGetValue(field.Number, out instance)) { Data.Seek(entry.Offset); instance = new FST <long?>((DataInput)Data, Lucene.Net.Util.Fst.PositiveIntOutputs.Singleton); RamBytesUsed_Renamed.AddAndGet(instance.SizeInBytes()); FstInstances[field.Number] = instance; } } BinaryDocValues docToOrds = GetBinary(field); FST <long?> fst = instance; // per-thread resources var @in = fst.BytesReader; var firstArc = new FST.Arc <long?>(); var scratchArc = new FST.Arc <long?>(); var scratchInts = new IntsRef(); var fstEnum = new BytesRefFSTEnum <long?>(fst); var @ref = new BytesRef(); var input = new ByteArrayDataInput(); return(new SortedSetDocValuesAnonymousInnerClassHelper(entry, docToOrds, fst, @in, firstArc, scratchArc, scratchInts, fstEnum, @ref, input)); }
public override SortedDocValues GetSorted(FieldInfo field) { FSTEntry entry = fsts[field.Number]; if (entry.numOrds == 0) { return(DocValues.EMPTY_SORTED); } FST <long?> instance; lock (this) { instance = fstInstances[field.Number]; if (instance == null) { data.Seek(entry.offset); instance = new FST <long?>(data, PositiveIntOutputs.Singleton); ramBytesUsed.AddAndGet(instance.SizeInBytes()); fstInstances[field.Number] = instance; } } var docToOrd = GetNumeric(field); var fst = instance; // per-thread resources var @in = fst.BytesReader; var firstArc = new FST.Arc <long?>(); var scratchArc = new FST.Arc <long?>(); var scratchInts = new IntsRef(); var fstEnum = new BytesRefFSTEnum <long?>(fst); return(new SortedDocValuesAnonymousInnerClassHelper(entry, docToOrd, fst, @in, firstArc, scratchArc, scratchInts, fstEnum)); }
/// <summary> /// Returns byte size of the underlying FST. </summary> public override long SizeInBytes() { if (fst == null) { return(0); } return(fst.SizeInBytes()); }
/// <summary> /// Returns byte size of the underlying FST. </summary> public override long GetSizeInBytes() { return((fst == null) ? 0 : fst.SizeInBytes()); }
public long RamBytesUsed() { return((fst != null) ? fst.SizeInBytes() : 0); }
/// <summary>Returns approximate RAM bytes used</summary> public long RamBytesUsed() { return(Fst == null ? 0 : Fst.SizeInBytes()); }
public override SortedSetDocValues GetSortedSet(FieldInfo field) { FSTEntry entry = Fsts[field.Number]; if (entry.NumOrds == 0) { return DocValues.EMPTY_SORTED_SET; // empty FST! } FST<long> instance; lock (this) { if (!FstInstances.TryGetValue(field.Number, out instance)) { Data.Seek(entry.Offset); instance = new FST<long>((DataInput)Data, Lucene.Net.Util.Fst.PositiveIntOutputs.Singleton); RamBytesUsed_Renamed.AddAndGet(instance.SizeInBytes()); FstInstances[field.Number] = instance; } } BinaryDocValues docToOrds = GetBinary(field); FST<long> fst = instance; // per-thread resources FST<long>.BytesReader @in = fst.BytesReader; FST<long>.Arc<long> firstArc = new FST<long>.Arc<long>(); FST<long>.Arc<long> scratchArc = new FST<long>.Arc<long>(); IntsRef scratchInts = new IntsRef(); BytesRefFSTEnum<long> fstEnum = new BytesRefFSTEnum<long>(fst); BytesRef @ref = new BytesRef(); ByteArrayDataInput input = new ByteArrayDataInput(); return new SortedSetDocValuesAnonymousInnerClassHelper(this, entry, docToOrds, fst, @in, firstArc, scratchArc, scratchInts, fstEnum, @ref, input); }
public override SortedDocValues GetSorted(FieldInfo field) { FSTEntry entry = Fsts[field.Number]; FST<long> instance; lock (this) { if (!FstInstances.TryGetValue(field.Number, out instance)) { Data.Seek(entry.Offset); instance = new FST<long>(Data, PositiveIntOutputs.Singleton); RamBytesUsed_Renamed.AddAndGet(instance.SizeInBytes()); FstInstances[field.Number] = instance; } } NumericDocValues docToOrd = GetNumeric(field); FST<long> fst = instance; // per-thread resources FST<long>.BytesReader @in = fst.BytesReader; FST<long>.Arc<long> firstArc = new FST<long>.Arc<long>(); FST<long>.Arc<long> scratchArc = new FST<long>.Arc<long>(); IntsRef scratchInts = new IntsRef(); BytesRefFSTEnum<long> fstEnum = new BytesRefFSTEnum<long>(fst); return new SortedDocValuesAnonymousInnerClassHelper(this, entry, docToOrd, fst, @in, firstArc, scratchArc, scratchInts, fstEnum); }
public virtual void Test() { int[] ints = new int[7]; IntsRef input = new IntsRef(ints, 0, ints.Length); int seed = Random().Next(); Directory dir = new MMapDirectory(CreateTempDir("2BFST")); for (int doPackIter = 0; doPackIter < 2; doPackIter++) { bool doPack = doPackIter == 1; // Build FST w/ NoOutputs and stop when nodeCount > 2.2B if (!doPack) { Console.WriteLine("\nTEST: 3B nodes; doPack=false output=NO_OUTPUTS"); Outputs <object> outputs = NoOutputs.Singleton; object NO_OUTPUT = outputs.NoOutput; Builder <object> b = new Builder <object>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, int.MaxValue, outputs, null, doPack, PackedInts.COMPACT, true, 15); int count = 0; Random r = new Random(seed); int[] ints2 = new int[200]; IntsRef input2 = new IntsRef(ints2, 0, ints2.Length); while (true) { //System.out.println("add: " + input + " -> " + output); for (int i = 10; i < ints2.Length; i++) { ints2[i] = r.Next(256); } b.Add(input2, NO_OUTPUT); count++; if (count % 100000 == 0) { Console.WriteLine(count + ": " + b.FstSizeInBytes() + " bytes; " + b.TotStateCount + " nodes"); } if (b.TotStateCount > int.MaxValue + 100L * 1024 * 1024) { break; } NextInput(r, ints2); } FST <object> fst = b.Finish(); for (int verify = 0; verify < 2; verify++) { Console.WriteLine("\nTEST: now verify [fst size=" + fst.SizeInBytes() + "; nodeCount=" + fst.NodeCount + "; arcCount=" + fst.ArcCount + "]"); Arrays.Fill(ints2, 0); r = new Random(seed); for (int i = 0; i < count; i++) { if (i % 1000000 == 0) { Console.WriteLine(i + "...: "); } for (int j = 10; j < ints2.Length; j++) { ints2[j] = r.Next(256); } Assert.AreEqual(NO_OUTPUT, Util.Get(fst, input2)); NextInput(r, ints2); } Console.WriteLine("\nTEST: enum all input/outputs"); IntsRefFSTEnum <object> fstEnum = new IntsRefFSTEnum <object>(fst); Arrays.Fill(ints2, 0); r = new Random(seed); int upto = 0; while (true) { IntsRefFSTEnum <object> .InputOutput <object> pair = fstEnum.Next(); if (pair == null) { break; } for (int j = 10; j < ints2.Length; j++) { ints2[j] = r.Next(256); } Assert.AreEqual(input2, pair.Input); Assert.AreEqual(NO_OUTPUT, pair.Output); upto++; NextInput(r, ints2); } Assert.AreEqual(count, upto); if (verify == 0) { Console.WriteLine("\nTEST: save/load FST and re-verify"); IndexOutput @out = dir.CreateOutput("fst", IOContext.DEFAULT); fst.Save(@out); @out.Dispose(); IndexInput @in = dir.OpenInput("fst", IOContext.DEFAULT); fst = new FST <object>(@in, outputs); @in.Dispose(); } else { dir.DeleteFile("fst"); } } } // Build FST w/ ByteSequenceOutputs and stop when FST // size = 3GB { Console.WriteLine("\nTEST: 3 GB size; doPack=" + doPack + " outputs=bytes"); Outputs <BytesRef> outputs = ByteSequenceOutputs.Singleton; Builder <BytesRef> b = new Builder <BytesRef>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, int.MaxValue, outputs, null, doPack, PackedInts.COMPACT, true, 15); var outputBytes = new byte[20]; BytesRef output = new BytesRef(outputBytes); Arrays.Fill(ints, 0); int count = 0; Random r = new Random(seed); while (true) { r.NextBytes(outputBytes); //System.out.println("add: " + input + " -> " + output); b.Add(input, BytesRef.DeepCopyOf(output)); count++; if (count % 1000000 == 0) { Console.WriteLine(count + "...: " + b.FstSizeInBytes() + " bytes"); } if (b.FstSizeInBytes() > LIMIT) { break; } NextInput(r, ints); } FST <BytesRef> fst = b.Finish(); for (int verify = 0; verify < 2; verify++) { Console.WriteLine("\nTEST: now verify [fst size=" + fst.SizeInBytes() + "; nodeCount=" + fst.NodeCount + "; arcCount=" + fst.ArcCount + "]"); r = new Random(seed); Arrays.Fill(ints, 0); for (int i = 0; i < count; i++) { if (i % 1000000 == 0) { Console.WriteLine(i + "...: "); } r.NextBytes((byte[])(Array)outputBytes); Assert.AreEqual(output, Util.Get(fst, input)); NextInput(r, ints); } Console.WriteLine("\nTEST: enum all input/outputs"); IntsRefFSTEnum <BytesRef> fstEnum = new IntsRefFSTEnum <BytesRef>(fst); Arrays.Fill(ints, 0); r = new Random(seed); int upto = 0; while (true) { IntsRefFSTEnum <BytesRef> .InputOutput <BytesRef> pair = fstEnum.Next(); if (pair == null) { break; } Assert.AreEqual(input, pair.Input); r.NextBytes((byte[])(Array)outputBytes); Assert.AreEqual(output, pair.Output); upto++; NextInput(r, ints); } Assert.AreEqual(count, upto); if (verify == 0) { Console.WriteLine("\nTEST: save/load FST and re-verify"); IndexOutput @out = dir.CreateOutput("fst", IOContext.DEFAULT); fst.Save(@out); @out.Dispose(); IndexInput @in = dir.OpenInput("fst", IOContext.DEFAULT); fst = new FST <BytesRef>(@in, outputs); @in.Dispose(); } else { dir.DeleteFile("fst"); } } } // Build FST w/ PositiveIntOutputs and stop when FST // size = 3GB { Console.WriteLine("\nTEST: 3 GB size; doPack=" + doPack + " outputs=long"); Outputs <long?> outputs = PositiveIntOutputs.Singleton; Builder <long?> b = new Builder <long?>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, int.MaxValue, outputs, null, doPack, PackedInts.COMPACT, true, 15); long output = 1; Arrays.Fill(ints, 0); int count = 0; Random r = new Random(seed); while (true) { //System.out.println("add: " + input + " -> " + output); b.Add(input, output); output += 1 + r.Next(10); count++; if (count % 1000000 == 0) { Console.WriteLine(count + "...: " + b.FstSizeInBytes() + " bytes"); } if (b.FstSizeInBytes() > LIMIT) { break; } NextInput(r, ints); } FST <long?> fst = b.Finish(); for (int verify = 0; verify < 2; verify++) { Console.WriteLine("\nTEST: now verify [fst size=" + fst.SizeInBytes() + "; nodeCount=" + fst.NodeCount + "; arcCount=" + fst.ArcCount + "]"); Arrays.Fill(ints, 0); output = 1; r = new Random(seed); for (int i = 0; i < count; i++) { if (i % 1000000 == 0) { Console.WriteLine(i + "...: "); } // forward lookup: Assert.AreEqual(output, (long)Util.Get(fst, input)); // reverse lookup: Assert.AreEqual(input, Util.GetByOutput(fst, output)); output += 1 + r.Next(10); NextInput(r, ints); } Console.WriteLine("\nTEST: enum all input/outputs"); IntsRefFSTEnum <long?> fstEnum = new IntsRefFSTEnum <long?>(fst); Arrays.Fill(ints, 0); r = new Random(seed); int upto = 0; output = 1; while (true) { IntsRefFSTEnum <long?> .InputOutput <long?> pair = fstEnum.Next(); if (pair == null) { break; } Assert.AreEqual(input, pair.Input); Assert.AreEqual(output, pair.Output.Value); output += 1 + r.Next(10); upto++; NextInput(r, ints); } Assert.AreEqual(count, upto); if (verify == 0) { Console.WriteLine("\nTEST: save/load FST and re-verify"); IndexOutput @out = dir.CreateOutput("fst", IOContext.DEFAULT); fst.Save(@out); @out.Dispose(); IndexInput @in = dir.OpenInput("fst", IOContext.DEFAULT); fst = new FST <long?>(@in, outputs); @in.Dispose(); } else { dir.DeleteFile("fst"); } } } } dir.Dispose(); }
public override SortedSetDocValues GetSortedSet(FieldInfo field) { var entry = fsts[field.Number]; if (entry.numOrds == 0) { return DocValues.EMPTY_SORTED_SET; // empty FST! } FST<long?> instance; lock (this) { instance = fstInstances[field.Number]; if (instance == null) { data.Seek(entry.offset); instance = new FST<long?>(data, PositiveIntOutputs.Singleton); ramBytesUsed.AddAndGet(instance.SizeInBytes()); fstInstances[field.Number] = instance; } } var docToOrds = GetBinary(field); var fst = instance; // per-thread resources var @in = fst.BytesReader; var firstArc = new FST.Arc<long?>(); var scratchArc = new FST.Arc<long?>(); var scratchInts = new IntsRef(); var fstEnum = new BytesRefFSTEnum<long?>(fst); var @ref = new BytesRef(); var input = new ByteArrayDataInput(); return new SortedSetDocValuesAnonymousInnerClassHelper(entry, docToOrds, fst, @in, firstArc, scratchArc, scratchInts, fstEnum, @ref, input); }