/// <summary> /// this constructor is used to initialize a <seealso cref="IOContext"/> instance with a new value for the readOnce variable. </summary> /// <param name="ctxt"> <seealso cref="IOContext"/> object whose information is used to create the new instance except the readOnce variable. </param> /// <param name="readOnce"> The new <seealso cref="IOContext"/> object will use this value for readOnce. </param> public IOContext(IOContext ctxt, bool readOnce) { this.Context = ctxt.Context; this.MergeInfo = ctxt.MergeInfo; this.FlushInfo = ctxt.FlushInfo; this.ReadOnce = readOnce; }
/// <summary> /// Creates an IndexInput for the file with the given name. </summary> public override IndexInput OpenInput(string name, IOContext context) { EnsureOpen(); var path = new FileInfo(Path.Combine(Directory.FullName, name)); var raf = new FileStream(path.FullName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); return new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + path.FullName + "\")", raf, context); }
public MockSingleIntIndexInput(Directory dir, string fileName, IOContext context) { @in = dir.OpenInput(fileName, context); CodecUtil.CheckHeader(@in, MockSingleIntIndexOutput.CODEC, MockSingleIntIndexOutput.VERSION_START, MockSingleIntIndexOutput.VERSION_START); }
public IndexInputSlicerAnonymousInnerClassHelper(SimpleFSDirectory outerInstance, IOContext context, FileInfo file, FileStream descriptor) : base(outerInstance) { this.Context = context; this.File = file; this.Descriptor = descriptor; }
public override IndexInputSlicer CreateSlicer(string name, IOContext context) { EnsureOpen(); var path = new FileInfo(Path.Combine(Directory.FullName, name)); var fc = new FileStream(path.FullName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); return new IndexInputSlicerAnonymousInnerClassHelper(this, context, path, fc); }
public override IndexInputSlicer CreateSlicer(string name, IOContext context) { EnsureOpen(); var file = new FileInfo(Path.Combine(Directory.FullName, name)); var descriptor = new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); return new IndexInputSlicerAnonymousInnerClassHelper(this, context, file, descriptor); }
/// <summary> /// Sole constructor. </summary> public CompressingTermVectorsReader(Directory d, SegmentInfo si, string segmentSuffix, FieldInfos fn, IOContext context, string formatName, CompressionMode compressionMode) { this.compressionMode = compressionMode; string segment = si.Name; bool success = false; fieldInfos = fn; numDocs = si.DocCount; ChecksumIndexInput indexStream = null; try { // Load the index into memory string indexStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, CompressingTermVectorsWriter.VECTORS_INDEX_EXTENSION); indexStream = d.OpenChecksumInput(indexStreamFN, context); string codecNameIdx = formatName + CompressingTermVectorsWriter.CODEC_SFX_IDX; version = CodecUtil.CheckHeader(indexStream, codecNameIdx, CompressingTermVectorsWriter.VERSION_START, CompressingTermVectorsWriter.VERSION_CURRENT); Debug.Assert(CodecUtil.HeaderLength(codecNameIdx) == indexStream.FilePointer); indexReader = new CompressingStoredFieldsIndexReader(indexStream, si); if (version >= CompressingTermVectorsWriter.VERSION_CHECKSUM) { indexStream.ReadVLong(); // the end of the data file CodecUtil.CheckFooter(indexStream); } else { CodecUtil.CheckEOF(indexStream); } indexStream.Dispose(); indexStream = null; // Open the data file and read metadata string vectorsStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, CompressingTermVectorsWriter.VECTORS_EXTENSION); vectorsStream = d.OpenInput(vectorsStreamFN, context); string codecNameDat = formatName + CompressingTermVectorsWriter.CODEC_SFX_DAT; int version2 = CodecUtil.CheckHeader(vectorsStream, codecNameDat, CompressingTermVectorsWriter.VERSION_START, CompressingTermVectorsWriter.VERSION_CURRENT); if (version != version2) { throw new Exception("Version mismatch between stored fields index and data: " + version + " != " + version2); } Debug.Assert(CodecUtil.HeaderLength(codecNameDat) == vectorsStream.FilePointer); packedIntsVersion = vectorsStream.ReadVInt(); chunkSize = vectorsStream.ReadVInt(); decompressor = compressionMode.NewDecompressor(); this.reader = new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, CompressingTermVectorsWriter.BLOCK_SIZE, 0); success = true; } finally { if (!success) { IOUtils.CloseWhileHandlingException(this, indexStream); } } }
/// <summary> /// Creates an IndexInput for the file with the given name. </summary> public override IndexInput OpenInput(string name, IOContext context) { EnsureOpen(); //File path = new File(Directory, name); FileInfo path = new FileInfo(Path.Combine(Directory.FullName, name)); //path.Create(); FileStream fc = new FileStream(path.FullName, FileMode.Open, FileAccess.Read, FileShare.Read);//FileChannel.open(path.toPath(), StandardOpenOption.READ); return new NIOFSIndexInput("NIOFSIndexInput(path=\"" + path + "\")", fc, context); //return new NIOFSIndexInput(new FileInfo(Path.Combine(Directory.FullName, name)), context, ReadChunkSize); }
public override IndexOutput CreateOutput(string name, IOContext context) { EnsureOpen(); IndexOutput output = base.CreateOutput(name, context); RateLimiter limiter = GetRateLimiter(context.Context); if (limiter != null) { return new RateLimitedIndexOutput(limiter, output); } return output; }
public MockSingleIntIndexOutput(Directory dir, string fileName, IOContext context) { @out = dir.CreateOutput(fileName, context); bool success = false; try { CodecUtil.WriteHeader(@out, CODEC, VERSION_CURRENT); success = true; } finally { if (!success) { IOUtils.CloseWhileHandlingException(@out); } } }
private int NumBufferedDocs; // docBase + numBufferedDocs == current doc ID /// <summary> /// Sole constructor. </summary> public CompressingStoredFieldsWriter(Directory directory, SegmentInfo si, string segmentSuffix, IOContext context, string formatName, CompressionMode compressionMode, int chunkSize) { Debug.Assert(directory != null); this.Directory = directory; this.Segment = si.Name; this.SegmentSuffix = segmentSuffix; this.CompressionMode = compressionMode; this.Compressor = compressionMode.NewCompressor(); this.ChunkSize = chunkSize; this.DocBase = 0; this.BufferedDocs = new GrowableByteArrayDataOutput(chunkSize); this.NumStoredFields = new int[16]; this.EndOffsets = new int[16]; this.NumBufferedDocs = 0; bool success = false; IndexOutput indexStream = directory.CreateOutput(IndexFileNames.SegmentFileName(Segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_INDEX_EXTENSION), context); try { FieldsStream = directory.CreateOutput(IndexFileNames.SegmentFileName(Segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_EXTENSION), context); string codecNameIdx = formatName + CODEC_SFX_IDX; string codecNameDat = formatName + CODEC_SFX_DAT; CodecUtil.WriteHeader(indexStream, codecNameIdx, VERSION_CURRENT); CodecUtil.WriteHeader(FieldsStream, codecNameDat, VERSION_CURRENT); Debug.Assert(CodecUtil.HeaderLength(codecNameDat) == FieldsStream.FilePointer); Debug.Assert(CodecUtil.HeaderLength(codecNameIdx) == indexStream.FilePointer); IndexWriter = new CompressingStoredFieldsIndexWriter(indexStream); indexStream = null; FieldsStream.WriteVInt(chunkSize); FieldsStream.WriteVInt(PackedInts.VERSION_CURRENT); success = true; } finally { if (!success) { IOUtils.CloseWhileHandlingException(indexStream); Abort(); } } }
// public static boolean DEBUG = false; /// <summary> /// Sole constructor. </summary> public Lucene41PostingsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo segmentInfo, IOContext ioContext, string segmentSuffix) { bool success = false; IndexInput docIn = null; IndexInput posIn = null; IndexInput payIn = null; try { docIn = dir.OpenInput(IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix, Lucene41PostingsFormat.DOC_EXTENSION), ioContext); Version = CodecUtil.CheckHeader(docIn, Lucene41PostingsWriter.DOC_CODEC, Lucene41PostingsWriter.VERSION_START, Lucene41PostingsWriter.VERSION_CURRENT); forUtil = new ForUtil(docIn); if (fieldInfos.HasProx()) { posIn = dir.OpenInput(IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix, Lucene41PostingsFormat.POS_EXTENSION), ioContext); CodecUtil.CheckHeader(posIn, Lucene41PostingsWriter.POS_CODEC, Version, Version); if (fieldInfos.HasPayloads() || fieldInfos.HasOffsets()) { payIn = dir.OpenInput(IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix, Lucene41PostingsFormat.PAY_EXTENSION), ioContext); CodecUtil.CheckHeader(payIn, Lucene41PostingsWriter.PAY_CODEC, Version, Version); } } this.DocIn = docIn; this.PosIn = posIn; this.PayIn = payIn; success = true; } finally { if (!success) { IOUtils.CloseWhileHandlingException(docIn, posIn, payIn); } } }
public override IndexInputSlicer CreateSlicer(string name, IOContext context) { return @in.CreateSlicer(name, context); }
/// <summary> /// Creates an IndexOutput for the file with the given name. </summary> public override IndexOutput CreateOutput(string name, IOContext context) { EnsureOpen(); EnsureCanWrite(name); return new FSIndexOutput(this, name); }
public override void Copy(Directory to, string src, string dest, IOContext context) { @in.Copy(to, src, dest, context); }
public override IndexInputSlicer CreateSlicer(string name, IOContext context) { return GetDirectory(name).CreateSlicer(name, context); }
private RAMDirectory(Directory dir, bool closeDir, IOContext context) : this() { foreach (string file in dir.ListAll()) { dir.Copy(this, file, file, context); } if (closeDir) { dir.Dispose(); } }
public AppendingTermsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo info, PostingsReaderBase postingsReader, IOContext ioContext, String segmentSuffix, int indexDivisor) : base(dir, fieldInfos, info, postingsReader, ioContext, segmentSuffix, indexDivisor) { }
public override IndexInput OpenInput(string name, IOContext context) { return GetDirectory(name).OpenInput(name, context); }
public override IndexInputSlicer CreateSlicer(string name, IOContext context) { lock (this) { EnsureOpen(); if (VERBOSE) { Console.WriteLine("nrtdir.openInput name=" + name); } if (Cache.FileExists(name)) { if (VERBOSE) { Console.WriteLine(" from cache"); } return Cache.CreateSlicer(name, context); } else { return @delegate.CreateSlicer(name, context); } } }
/// <summary> /// Subclass can override this to customize logic; return /// true if this file should be written to the RAMDirectory. /// </summary> protected internal virtual bool DoCacheWrite(string name, IOContext context) { //System.out.println(Thread.currentThread().getName() + ": CACHE check merge=" + merge + " size=" + (merge==null ? 0 : merge.estimatedMergeBytes)); long bytes = 0; if (context.MergeInfo != null) { bytes = context.MergeInfo.EstimatedMergeBytes; } else if (context.FlushInfo != null) { bytes = context.FlushInfo.EstimatedSegmentSize; } return !name.Equals(IndexFileNames.SEGMENTS_GEN) && (bytes <= MaxMergeSizeBytes) && (bytes + Cache.SizeInBytes()) <= MaxCachedBytes; }
/// <summary> /// Creates a new, empty file in the directory with the given name. Returns a stream writing this file. </summary> public override IndexOutput CreateOutput(string name, IOContext context) { EnsureOpen(); RAMFile file = NewRAMFile(); RAMFile existing; FileMap.TryGetValue(name, out existing); if (existing != null) { sizeInBytes.AddAndGet(-existing.SizeInBytes_Renamed); existing.Directory = null; } FileMap[name] = file; return new RAMOutputStream(file); }
public override IndexOutput CreateOutput(string name, IOContext context) { if (VERBOSE) { Console.WriteLine("nrtdir.createOutput name=" + name); } if (DoCacheWrite(name, context)) { if (VERBOSE) { Console.WriteLine(" to cache"); } try { @delegate.DeleteFile(name); } catch (System.IO.IOException ioe) { // this is fine: file may not exist } return Cache.CreateOutput(name, context); } else { try { Cache.DeleteFile(name); } catch (System.IO.IOException ioe) { // this is fine: file may not exist } return @delegate.CreateOutput(name, context); } }
/// <summary> /// Returns a stream reading an existing file. </summary> public override IndexInput OpenInput(string name, IOContext context) { EnsureOpen(); RAMFile file; if (!FileMap.TryGetValue(name, out file)) { throw new FileNotFoundException(name); } return new RAMInputStream(name, file); }
/// <summary> /// Creates a new <code>RAMDirectory</code> instance from a different /// <code>Directory</code> implementation. this can be used to load /// a disk-based index into memory. /// /// <p><b>Warning:</b> this class is not intended to work with huge /// indexes. Everything beyond several hundred megabytes will waste /// resources (GC cycles), because it uses an internal buffer size /// of 1024 bytes, producing millions of {@code byte[1024]} arrays. /// this class is optimized for small memory-resident indexes. /// It also has bad concurrency on multithreaded environments. /// /// <p>For disk-based indexes it is recommended to use /// <seealso cref="MMapDirectory"/>, which is a high-performance directory /// implementation working directly on the file system cache of the /// operating system, so copying data to Java heap space is not useful. /// /// <p>Note that the resulting <code>RAMDirectory</code> instance is fully /// independent from the original <code>Directory</code> (it is a /// complete copy). Any subsequent changes to the /// original <code>Directory</code> will not be visible in the /// <code>RAMDirectory</code> instance. /// </summary> /// <param name="dir"> a <code>Directory</code> value </param> /// <exception cref="System.IO.IOException"> if an error occurs </exception> public RAMDirectory(Directory dir, IOContext context) : this(dir, false, context) { }
public SimpleFSIndexInput(string resourceDesc, FileStream file, IOContext context) : base(resourceDesc, context) { this.File = file; this.Off = 0L; this.End = file.Length; }
/// <summary> /// Creates an IndexInput for the file with the given name. </summary> public override IndexInput OpenInput(string name, IOContext context) { EnsureOpen(); var file = new FileInfo(Path.Combine(Directory.FullName, name)); var c = new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); return new MMapIndexInput(this, "MMapIndexInput(path=\"" + file + "\")", c); }
public override IndexOutput CreateOutput(string name, IOContext context) { return GetDirectory(name).CreateOutput(name, context); }
public override IndexInputSlicer CreateSlicer(string name, IOContext context) { var full = (MMapIndexInput)OpenInput(name, context); return new IndexInputSlicerAnonymousInnerClassHelper(this, full); }
public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, String segmentSuffix, IOContext context) { input = dir.OpenInput( IndexFileNames.SegmentFileName(segment, segmentSuffix, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION), new IOContext(context, true)); this.segment = segment; bool success = false; Debug.Debug.Assert((indexDivisor == -1 || indexDivisor > 0); try { version = readHeader(input); this.indexDivisor = indexDivisor; if (version >= VariableGapTermsIndexWriter.VERSION_CHECKSUM) { CodecUtil.ChecksumEntireFile(input); } SeekDir(in, dirOffset) ; // Read directory int numFields = input.ReadVInt(); if (numFields < 0) { throw new CorruptIndexException("invalid numFields: " + numFields + " (resource=" + input + ")"); } for (int i = 0; i < numFields; i++) { final int field = in. readVInt(); final long indexStart = in. readVLong(); final FieldInfo fieldInfo = fieldInfos.fieldInfo(field); FieldIndexData previous = fields.put(fieldInfo, new FieldIndexData(fieldInfo, indexStart)); if (previous != null) { throw new CorruptIndexException("duplicate field: " + fieldInfo.name + " (resource=" +in + ")" ) ; } } success = true; } finally { if (indexDivisor > 0) { in. close(); in = null; if (success) { indexLoaded = true; } } }