public override void Before(LuceneTestCase testInstance) { // LUCENENET specific - SOLR setup code removed // if verbose: print some debugging stuff about which codecs are loaded. if (LuceneTestCase.VERBOSE) { ICollection <string> codecs = Codec.AvailableCodecs; foreach (string codec in codecs) { Console.WriteLine("Loaded codec: '" + codec + "': " + Codec.ForName(codec).GetType().Name); } ICollection <string> postingsFormats = PostingsFormat.AvailablePostingsFormats; foreach (string postingsFormat in postingsFormats) { Console.WriteLine("Loaded postingsFormat: '" + postingsFormat + "': " + PostingsFormat.ForName(postingsFormat).GetType().Name); } } savedInfoStream = InfoStream.Default; Random random = LuceneTestCase.Random; bool v = random.NextBoolean(); if (LuceneTestCase.INFOSTREAM) { InfoStream.Default = new ThreadNameFixingPrintStreamInfoStream(Console.Out); } else if (v) { InfoStream.Default = new NullInfoStream(); } Type targetClass = testInstance?.GetType() ?? LuceneTestCase.GetTestClass(); avoidCodecs = new JCG.HashSet <string>(); var suppressCodecsAttribute = targetClass.GetTypeInfo().GetCustomAttribute <LuceneTestCase.SuppressCodecsAttribute>(); if (suppressCodecsAttribute != null) { avoidCodecs.UnionWith(suppressCodecsAttribute.Value); } // set back to default LuceneTestCase.OldFormatImpersonationIsActive = false; savedCodec = Codec.Default; int randomVal = random.Next(10); if ("Lucene3x".Equals(LuceneTestCase.TEST_CODEC, StringComparison.Ordinal) || ("random".Equals(LuceneTestCase.TEST_CODEC, StringComparison.Ordinal) && "random".Equals(LuceneTestCase.TEST_POSTINGSFORMAT, StringComparison.Ordinal) && "random".Equals(LuceneTestCase.TEST_DOCVALUESFORMAT, StringComparison.Ordinal) && randomVal == 3 && !ShouldAvoidCodec("Lucene3x"))) // preflex-only setup { codec = Codec.ForName("Lucene3x"); Debug.Assert((codec is PreFlexRWCodec), "fix your ICodecFactory to scan Lucene.Net.Tests before Lucene.Net.TestFramework"); LuceneTestCase.OldFormatImpersonationIsActive = true; } else if ("Lucene40".Equals(LuceneTestCase.TEST_CODEC, StringComparison.Ordinal) || ("random".Equals(LuceneTestCase.TEST_CODEC, StringComparison.Ordinal) && "random".Equals(LuceneTestCase.TEST_POSTINGSFORMAT, StringComparison.Ordinal) && randomVal == 0 && !ShouldAvoidCodec("Lucene40"))) // 4.0 setup { codec = Codec.ForName("Lucene40"); LuceneTestCase.OldFormatImpersonationIsActive = true; Debug.Assert((codec is Lucene40RWCodec), "fix your ICodecFactory to scan Lucene.Net.Tests before Lucene.Net.TestFramework"); Debug.Assert((PostingsFormat.ForName("Lucene40") is Lucene40RWPostingsFormat), "fix your IPostingsFormatFactory to scan Lucene.Net.Tests before Lucene.Net.TestFramework"); } else if ("Lucene41".Equals(LuceneTestCase.TEST_CODEC, StringComparison.Ordinal) || ("random".Equals(LuceneTestCase.TEST_CODEC, StringComparison.Ordinal) && "random".Equals(LuceneTestCase.TEST_POSTINGSFORMAT, StringComparison.Ordinal) && "random".Equals(LuceneTestCase.TEST_DOCVALUESFORMAT, StringComparison.Ordinal) && randomVal == 1 && !ShouldAvoidCodec("Lucene41"))) { codec = Codec.ForName("Lucene41"); LuceneTestCase.OldFormatImpersonationIsActive = true; Debug.Assert((codec is Lucene41RWCodec), "fix your ICodecFactory to scan Lucene.Net.Tests before Lucene.Net.TestFramework"); } else if ("Lucene42".Equals(LuceneTestCase.TEST_CODEC, StringComparison.Ordinal) || ("random".Equals(LuceneTestCase.TEST_CODEC, StringComparison.Ordinal) && "random".Equals(LuceneTestCase.TEST_POSTINGSFORMAT, StringComparison.Ordinal) && "random".Equals(LuceneTestCase.TEST_DOCVALUESFORMAT, StringComparison.Ordinal) && randomVal == 2 && !ShouldAvoidCodec("Lucene42"))) { codec = Codec.ForName("Lucene42"); LuceneTestCase.OldFormatImpersonationIsActive = true; Debug.Assert((codec is Lucene42RWCodec), "fix your ICodecFactory to scan Lucene.Net.Tests before Lucene.Net.TestFramework"); } else if ("Lucene45".Equals(LuceneTestCase.TEST_CODEC, StringComparison.Ordinal) || ("random".Equals(LuceneTestCase.TEST_CODEC, StringComparison.Ordinal) && "random".Equals(LuceneTestCase.TEST_POSTINGSFORMAT, StringComparison.Ordinal) && "random".Equals(LuceneTestCase.TEST_DOCVALUESFORMAT, StringComparison.Ordinal) && randomVal == 5 && !ShouldAvoidCodec("Lucene45"))) { codec = Codec.ForName("Lucene45"); LuceneTestCase.OldFormatImpersonationIsActive = true; Debug.Assert((codec is Lucene45RWCodec), "fix your ICodecFactory to scan Lucene.Net.Tests before Lucene.Net.TestFramework"); } else if (("random".Equals(LuceneTestCase.TEST_POSTINGSFORMAT, StringComparison.Ordinal) == false) || ("random".Equals(LuceneTestCase.TEST_DOCVALUESFORMAT, StringComparison.Ordinal) == false)) { // the user wired postings or DV: this is messy // refactor into RandomCodec.... PostingsFormat format; if ("random".Equals(LuceneTestCase.TEST_POSTINGSFORMAT, StringComparison.Ordinal)) { format = PostingsFormat.ForName("Lucene41"); } else if ("MockRandom".Equals(LuceneTestCase.TEST_POSTINGSFORMAT, StringComparison.Ordinal)) { format = new MockRandomPostingsFormat(new Random(random.Next())); } else { format = PostingsFormat.ForName(LuceneTestCase.TEST_POSTINGSFORMAT); } DocValuesFormat dvFormat; if ("random".Equals(LuceneTestCase.TEST_DOCVALUESFORMAT, StringComparison.Ordinal)) { dvFormat = DocValuesFormat.ForName("Lucene45"); } else { dvFormat = DocValuesFormat.ForName(LuceneTestCase.TEST_DOCVALUESFORMAT); } codec = new Lucene46CodecAnonymousInnerClassHelper(this, format, dvFormat); } else if ("SimpleText".Equals(LuceneTestCase.TEST_CODEC, StringComparison.Ordinal) || ("random".Equals(LuceneTestCase.TEST_CODEC, StringComparison.Ordinal) && randomVal == 9 && LuceneTestCase.Rarely(random) && !ShouldAvoidCodec("SimpleText"))) { codec = new SimpleTextCodec(); } else if ("CheapBastard".Equals(LuceneTestCase.TEST_CODEC, StringComparison.Ordinal) || ("random".Equals(LuceneTestCase.TEST_CODEC, StringComparison.Ordinal) && randomVal == 8 && !ShouldAvoidCodec("CheapBastard") && !ShouldAvoidCodec("Lucene41"))) { // we also avoid this codec if Lucene41 is avoided, since thats the postings format it uses. codec = new CheapBastardCodec(); } else if ("Asserting".Equals(LuceneTestCase.TEST_CODEC, StringComparison.Ordinal) || ("random".Equals(LuceneTestCase.TEST_CODEC, StringComparison.Ordinal) && randomVal == 6 && !ShouldAvoidCodec("Asserting"))) { codec = new AssertingCodec(); } else if ("Compressing".Equals(LuceneTestCase.TEST_CODEC, StringComparison.Ordinal) || ("random".Equals(LuceneTestCase.TEST_CODEC, StringComparison.Ordinal) && randomVal == 5 && !ShouldAvoidCodec("Compressing"))) { codec = CompressingCodec.RandomInstance(random); } else if (!"random".Equals(LuceneTestCase.TEST_CODEC, StringComparison.Ordinal)) { codec = Codec.ForName(LuceneTestCase.TEST_CODEC); } else if ("random".Equals(LuceneTestCase.TEST_POSTINGSFORMAT, StringComparison.Ordinal)) { codec = new RandomCodec(random, avoidCodecs); } else { Debug.Assert(false); } Codec.Default = codec; // Initialize locale/ timezone. string testLocale = SystemProperties.GetProperty("tests.locale", "random"); string testTimeZone = SystemProperties.GetProperty("tests.timezone", "random"); // Always pick a random one for consistency (whether tests.locale was specified or not). savedLocale = CultureInfo.CurrentCulture; CultureInfo randomLocale = LuceneTestCase.RandomCulture(random); locale = testLocale.Equals("random", StringComparison.Ordinal) ? randomLocale : LuceneTestCase.CultureForName(testLocale); #if NETSTANDARD CultureInfo.CurrentCulture = locale; #else Thread.CurrentThread.CurrentCulture = locale; #endif // TimeZone.getDefault will set user.timezone to the default timezone of the user's locale. // So store the original property value and restore it at end. restoreProperties["user.timezone"] = SystemProperties.GetProperty("user.timezone"); savedTimeZone = TimeZoneInfo.Local; TimeZoneInfo randomTimeZone = LuceneTestCase.RandomTimeZone(random); timeZone = testTimeZone.Equals("random", StringComparison.Ordinal) ? randomTimeZone : TimeZoneInfo.FindSystemTimeZoneById(testTimeZone); //TimeZone.Default = TimeZone; // LUCENENET NOTE: There doesn't seem to be an equivalent to this, but I don't think we need it. similarity = random.NextBoolean() ? (Similarity) new DefaultSimilarity() : new RandomSimilarityProvider(random); // Check codec restrictions once at class level. try { CheckCodecRestrictions(codec); } catch (Exception e) { Console.Error.WriteLine("NOTE: " + e.Message + " Suppressed codecs: " + Arrays.ToString(avoidCodecs.ToArray())); throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details) } }
public override void WriteField(FieldInfo info, IIndexableField field) { fieldsStream.WriteVInt32(info.Number); int bits = 0; BytesRef bytes; string @string; // TODO: maybe a field should serialize itself? // this way we don't bake into indexer all these // specific encodings for different fields? and apps // can customize... // LUCENENET specific - To avoid boxing/unboxing, we don't // call GetNumericValue(). Instead, we check the field.NumericType and then // call the appropriate conversion method. if (field.NumericType != NumericFieldType.NONE) { switch (field.NumericType) { case NumericFieldType.BYTE: case NumericFieldType.INT16: case NumericFieldType.INT32: bits |= Lucene3xStoredFieldsReader.FIELD_IS_NUMERIC_INT; break; case NumericFieldType.INT64: bits |= Lucene3xStoredFieldsReader.FIELD_IS_NUMERIC_LONG; break; case NumericFieldType.SINGLE: bits |= Lucene3xStoredFieldsReader.FIELD_IS_NUMERIC_FLOAT; break; case NumericFieldType.DOUBLE: bits |= Lucene3xStoredFieldsReader.FIELD_IS_NUMERIC_DOUBLE; break; default: throw new System.ArgumentException("cannot store numeric type " + field.NumericType); } @string = null; bytes = null; } else { bytes = field.GetBinaryValue(); if (bytes != null) { bits |= Lucene3xStoredFieldsReader.FIELD_IS_BINARY; @string = null; } else { @string = field.GetStringValue(); if (@string == null) { throw new System.ArgumentException("field " + field.Name + " is stored but does not have binaryValue, stringValue nor numericValue"); } } } fieldsStream.WriteByte((byte)(sbyte)bits); if (bytes != null) { fieldsStream.WriteVInt32(bytes.Length); fieldsStream.WriteBytes(bytes.Bytes, bytes.Offset, bytes.Length); } else if (@string != null) { fieldsStream.WriteString(field.GetStringValue()); } else { switch (field.NumericType) { case NumericFieldType.BYTE: case NumericFieldType.INT16: case NumericFieldType.INT32: fieldsStream.WriteInt32(field.GetInt32Value().Value); break; case NumericFieldType.INT64: fieldsStream.WriteInt64(field.GetInt64Value().Value); break; case NumericFieldType.SINGLE: fieldsStream.WriteInt32(J2N.BitConversion.SingleToInt32Bits(field.GetSingleValue().Value)); break; case NumericFieldType.DOUBLE: fieldsStream.WriteInt64(J2N.BitConversion.DoubleToInt64Bits(field.GetDoubleValue().Value)); break; default: Debug.Assert(false); break; } } }
public override void VisitDocument(int n, StoredFieldVisitor visitor) { Debug.Assert(n >= 0 && n < maxDoc); @in.VisitDocument(n, visitor); }
public override void Write(Directory directory, string segmentName, string segmentSuffix, FieldInfos infos, IOContext context) { string fileName = IndexFileNames.SegmentFileName(segmentName, "", FIELD_INFOS_EXTENSION); IndexOutput output = directory.CreateOutput(fileName, context); bool success = false; try { output.WriteVInt32(FORMAT_PREFLEX_RW); output.WriteVInt32(infos.Count); foreach (FieldInfo fi in infos) { sbyte bits = 0x0; if (fi.HasVectors) { bits |= STORE_TERMVECTOR; } if (fi.OmitsNorms) { bits |= OMIT_NORMS; } if (fi.HasPayloads) { bits |= STORE_PAYLOADS; } if (fi.IsIndexed) { bits |= IS_INDEXED; Debug.Assert(fi.IndexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !fi.HasPayloads); if (fi.IndexOptions == IndexOptions.DOCS_ONLY) { bits |= OMIT_TERM_FREQ_AND_POSITIONS; } else if (fi.IndexOptions == IndexOptions.DOCS_AND_FREQS) { bits |= OMIT_POSITIONS; } } output.WriteString(fi.Name); /* * we need to write the field number since IW tries * to stabelize the field numbers across segments so the * FI ordinal is not necessarily equivalent to the field number */ output.WriteInt32(fi.Number); output.WriteByte((byte)bits); if (fi.IsIndexed && !fi.OmitsNorms) { // to allow null norm types we need to indicate if norms are written // only in RW case output.WriteByte((byte)(sbyte)(fi.NormType == Index.DocValuesType.NONE ? 0 : 1)); } Debug.Assert(fi.Attributes == null); // not used or supported } success = true; } finally { if (success) { output.Dispose(); } else { IOUtils.DisposeWhileHandlingException(output); } } }
public override void Write(Directory directory, string segmentName, string segmentSuffix, FieldInfos infos, IOContext context) { string fileName = IndexFileNames.SegmentFileName(segmentName, "", Lucene40FieldInfosFormat.FIELD_INFOS_EXTENSION); IndexOutput output = directory.CreateOutput(fileName, context); bool success = false; try { CodecUtil.WriteHeader(output, Lucene40FieldInfosFormat.CODEC_NAME, Lucene40FieldInfosFormat.FORMAT_CURRENT); output.WriteVInt32(infos.Count); foreach (FieldInfo fi in infos) { IndexOptions indexOptions = fi.IndexOptions; sbyte bits = 0x0; if (fi.HasVectors) { bits |= Lucene40FieldInfosFormat.STORE_TERMVECTOR; } if (fi.OmitsNorms) { bits |= Lucene40FieldInfosFormat.OMIT_NORMS; } if (fi.HasPayloads) { bits |= Lucene40FieldInfosFormat.STORE_PAYLOADS; } if (fi.IsIndexed) { bits |= Lucene40FieldInfosFormat.IS_INDEXED; Debug.Assert(indexOptions.CompareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !fi.HasPayloads); if (indexOptions == IndexOptions.DOCS_ONLY) { bits |= Lucene40FieldInfosFormat.OMIT_TERM_FREQ_AND_POSITIONS; } else if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) { bits |= Lucene40FieldInfosFormat.STORE_OFFSETS_IN_POSTINGS; } else if (indexOptions == IndexOptions.DOCS_AND_FREQS) { bits |= Lucene40FieldInfosFormat.OMIT_POSITIONS; } } output.WriteString(fi.Name); output.WriteVInt32(fi.Number); output.WriteByte((byte)bits); // pack the DV types in one byte byte dv = DocValuesByte(fi.DocValuesType, fi.GetAttribute(Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY)); byte nrm = DocValuesByte(fi.NormType, fi.GetAttribute(Lucene40FieldInfosReader.LEGACY_NORM_TYPE_KEY)); Debug.Assert((dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0); var val = (byte)(0xff & ((nrm << 4) | (byte)dv)); output.WriteByte(val); output.WriteStringStringMap(fi.Attributes); } success = true; } finally { if (success) { output.Dispose(); } else { IOUtils.DisposeWhileHandlingException(output); } } }
public virtual AttributeSource.State NextState() { Debug.Assert(NextRead < InputTokens.Count); return(InputTokens[NextRead++]); }
public AllDeletedFilterReader(AtomicReader @in) : base(@in) { liveDocs = new Bits.MatchNoBits(@in.MaxDoc); Debug.Assert(MaxDoc == 0 || HasDeletions); }
public override void Run() { // TODO: would be better if this were cross thread, so that we make sure one thread deleting anothers added docs works: IList <string> toDeleteIDs = new List <string>(); IList <SubDocs> toDeleteSubDocs = new List <SubDocs>(); while (Environment.TickCount < stopTime && !outerInstance.m_failed.Get()) { try { // Occasional longish pause if running // nightly if (LuceneTestCase.TEST_NIGHTLY && Random.Next(6) == 3) { if (VERBOSE) { Console.WriteLine(Thread.CurrentThread.Name + ": now long sleep"); } Thread.Sleep(TestUtil.NextInt32(Random, 50, 500)); } // Rate limit ingest rate: if (Random.Next(7) == 5) { Thread.Sleep(TestUtil.NextInt32(Random, 1, 10)); if (VERBOSE) { Console.WriteLine(Thread.CurrentThread.Name + ": done sleep"); } } Document doc = docs.NextDoc(); if (doc == null) { break; } // Maybe add randomly named field string addedField; if (Random.NextBoolean()) { addedField = "extra" + Random.Next(40); doc.Add(NewTextField(addedField, "a random field", Field.Store.YES)); } else { addedField = null; } if (Random.NextBoolean()) { if (Random.NextBoolean()) { // Add/update doc block: string packID; SubDocs delSubDocs; if (toDeleteSubDocs.Count > 0 && Random.NextBoolean()) { delSubDocs = toDeleteSubDocs[Random.Next(toDeleteSubDocs.Count)]; Debug.Assert(!delSubDocs.Deleted); toDeleteSubDocs.Remove(delSubDocs); // Update doc block, replacing prior packID packID = delSubDocs.PackID; } else { delSubDocs = null; // Add doc block, using new packID packID = outerInstance.m_packCount.GetAndIncrement().ToString(CultureInfo.InvariantCulture); } Field packIDField = NewStringField("packID", packID, Field.Store.YES); IList <string> docIDs = new List <string>(); SubDocs subDocs = new SubDocs(packID, docIDs); IList <Document> docsList = new List <Document>(); allSubDocs.Add(subDocs); doc.Add(packIDField); docsList.Add(TestUtil.CloneDocument(doc)); docIDs.Add(doc.Get("docid")); int maxDocCount = TestUtil.NextInt32(Random, 1, 10); while (docsList.Count < maxDocCount) { doc = docs.NextDoc(); if (doc == null) { break; } docsList.Add(TestUtil.CloneDocument(doc)); docIDs.Add(doc.Get("docid")); } outerInstance.m_addCount.AddAndGet(docsList.Count); Term packIDTerm = new Term("packID", packID); if (delSubDocs != null) { delSubDocs.Deleted = true; delIDs.AddAll(delSubDocs.SubIDs); outerInstance.m_delCount.AddAndGet(delSubDocs.SubIDs.Count); if (VERBOSE) { Console.WriteLine(Thread.CurrentThread.Name + ": update pack packID=" + delSubDocs.PackID + " count=" + docsList.Count + " docs=" + Arrays.ToString(docIDs)); } outerInstance.UpdateDocuments(packIDTerm, docsList); } else { if (VERBOSE) { Console.WriteLine(Thread.CurrentThread.Name + ": add pack packID=" + packID + " count=" + docsList.Count + " docs=" + Arrays.ToString(docIDs)); } outerInstance.AddDocuments(packIDTerm, docsList); } doc.RemoveField("packID"); if (Random.Next(5) == 2) { if (VERBOSE) { Console.WriteLine(Thread.CurrentThread.Name + ": buffer del id:" + packID); } toDeleteSubDocs.Add(subDocs); } } else { // Add single doc string docid = doc.Get("docid"); if (VERBOSE) { Console.WriteLine(Thread.CurrentThread.Name + ": add doc docid:" + docid); } outerInstance.AddDocument(new Term("docid", docid), doc); outerInstance.m_addCount.GetAndIncrement(); if (Random.Next(5) == 3) { if (VERBOSE) { Console.WriteLine(Thread.CurrentThread.Name + ": buffer del id:" + doc.Get("docid")); } toDeleteIDs.Add(docid); } } } else { // Update single doc, but we never re-use // and ID so the delete will never // actually happen: if (VERBOSE) { Console.WriteLine(Thread.CurrentThread.Name + ": update doc id:" + doc.Get("docid")); } string docid = doc.Get("docid"); outerInstance.UpdateDocument(new Term("docid", docid), doc); outerInstance.m_addCount.GetAndIncrement(); if (Random.Next(5) == 3) { if (VERBOSE) { Console.WriteLine(Thread.CurrentThread.Name + ": buffer del id:" + doc.Get("docid")); } toDeleteIDs.Add(docid); } } if (Random.Next(30) == 17) { if (VERBOSE) { Console.WriteLine(Thread.CurrentThread.Name + ": apply " + toDeleteIDs.Count + " deletes"); } foreach (string id in toDeleteIDs) { if (VERBOSE) { Console.WriteLine(Thread.CurrentThread.Name + ": del term=id:" + id); } outerInstance.DeleteDocuments(new Term("docid", id)); } int count = outerInstance.m_delCount.AddAndGet(toDeleteIDs.Count); if (VERBOSE) { Console.WriteLine(Thread.CurrentThread.Name + ": tot " + count + " deletes"); } delIDs.AddAll(toDeleteIDs); toDeleteIDs.Clear(); foreach (SubDocs subDocs in toDeleteSubDocs) { Debug.Assert(!subDocs.Deleted); delPackIDs.Add(subDocs.PackID); outerInstance.DeleteDocuments(new Term("packID", subDocs.PackID)); subDocs.Deleted = true; if (VERBOSE) { Console.WriteLine(Thread.CurrentThread.Name + ": del subs: " + subDocs.SubIDs + " packID=" + subDocs.PackID); } delIDs.AddAll(subDocs.SubIDs); outerInstance.m_delCount.AddAndGet(subDocs.SubIDs.Count); } toDeleteSubDocs.Clear(); } if (addedField != null) { doc.RemoveField(addedField); } } catch (Exception t) { Console.WriteLine(Thread.CurrentThread.Name + ": hit exc"); Console.WriteLine(t.ToString()); Console.Write(t.StackTrace); outerInstance.m_failed.Set(true); throw new Exception(t.ToString(), t); } } if (VERBOSE) { Console.WriteLine(Thread.CurrentThread.Name + ": indexing done"); } outerInstance.DoAfterIndexingThreadDone(); }