internal virtual System.Collections.ArrayList Files() { System.Collections.ArrayList files = System.Collections.ArrayList.Synchronized(new System.Collections.ArrayList(16)); if (si.GetUseCompoundFile()) { System.String name = segment + ".cfs"; if (Directory().FileExists(name)) { files.Add(name); } } else { for (int i = 0; i < IndexFileNames.INDEX_EXTENSIONS.Length; i++) { System.String name = segment + "." + IndexFileNames.INDEX_EXTENSIONS[i]; if (Directory().FileExists(name)) { files.Add(name); } } } if (si.HasDeletions()) { files.Add(si.GetDelFileName()); } bool addedNrm = false; for (int i = 0; i < fieldInfos.Size(); i++) { System.String name = si.GetNormFileName(i); if (name != null && Directory().FileExists(name)) { if (name.EndsWith("." + IndexFileNames.NORMS_EXTENSION)) { if (addedNrm) { continue; // add .nrm just once } addedNrm = true; } files.Add(name); } } return(files); }
internal void ReWrite(SegmentInfo si) { // NOTE: norms are re-written in regular directory, not cfs System.String oldFileName = si.GetNormFileName(this.number); if (oldFileName != null && !oldFileName.EndsWith("." + IndexFileNames.NORMS_EXTENSION)) { // Mark this file for deletion. Note that we don't // actually try to delete it until the new segments files is // successfully written: Enclosing_Instance.deleter.AddPendingFile(oldFileName); } si.AdvanceNormGen(this.number); IndexOutput out_Renamed = Enclosing_Instance.Directory().CreateOutput(si.GetNormFileName(this.number)); try { out_Renamed.WriteBytes(bytes, Enclosing_Instance.MaxDoc()); } finally { out_Renamed.Close(); } this.dirty = false; }
private void OpenNorms(Directory cfsDir, int readBufferSize) { long nextNormSeek = SegmentMerger.NORMS_HEADER.Length; //skip header (header unused for now) int maxDoc = MaxDoc(); for (int i = 0; i < fieldInfos.Size(); i++) { FieldInfo fi = fieldInfos.FieldInfo(i); if (norms.Contains(fi.name)) { // in case this SegmentReader is being re-opened, we might be able to // reuse some norm instances and skip loading them here continue; } if (fi.isIndexed && !fi.omitNorms) { Directory d = Directory(); System.String fileName = si.GetNormFileName(fi.number); if (!si.HasSeparateNorms(fi.number)) { d = cfsDir; } // singleNormFile means multiple norms share this file bool singleNormFile = fileName.EndsWith("." + IndexFileNames.NORMS_EXTENSION); IndexInput normInput = null; long normSeek; if (singleNormFile) { normSeek = nextNormSeek; if (singleNormStream == null) { singleNormStream = d.OpenInput(fileName, readBufferSize); } // All norms in the .nrm file can share a single IndexInput since // they are only used in a synchronized context. // If this were to change in the future, a clone could be done here. normInput = singleNormStream; } else { normSeek = 0; normInput = d.OpenInput(fileName); } norms[fi.name] = new Norm(this, normInput, singleNormFile, fi.number, normSeek); nextNormSeek += maxDoc; // increment also if some norms are separate } } }
internal virtual SegmentReader ReopenSegment(SegmentInfo si, bool doClone, bool openReadOnly) { lock (this) { bool deletionsUpToDate = (this.si.HasDeletions() == si.HasDeletions()) && (!si.HasDeletions() || this.si.GetDelFileName().Equals(si.GetDelFileName())); bool normsUpToDate = true; bool[] fieldNormsChanged = new bool[core.fieldInfos.Size()]; int fieldCount = core.fieldInfos.Size(); for (int i = 0; i < fieldCount; i++) { if (!this.si.GetNormFileName(i).Equals(si.GetNormFileName(i))) { normsUpToDate = false; fieldNormsChanged[i] = true; } } // if we're cloning we need to run through the reopenSegment logic // also if both old and new readers aren't readonly, we clone to avoid sharing modifications if (normsUpToDate && deletionsUpToDate && !doClone && openReadOnly && readOnly) { return this; } // When cloning, the incoming SegmentInfos should not // have any changes in it: System.Diagnostics.Debug.Assert(!doClone ||(normsUpToDate && deletionsUpToDate)); // clone reader SegmentReader clone; try { if (openReadOnly) clone = (SegmentReader) System.Activator.CreateInstance(READONLY_IMPL); else clone = (SegmentReader) System.Activator.CreateInstance(IMPL); } catch (System.Exception e) { throw new System.SystemException("cannot load SegmentReader class: " + e, e); } bool success = false; try { core.IncRef(); clone.core = core; clone.readOnly = openReadOnly; clone.si = si; clone.readBufferSize = readBufferSize; if (!openReadOnly && hasChanges) { // My pending changes transfer to the new reader clone.pendingDeleteCount = pendingDeleteCount; clone.deletedDocsDirty = deletedDocsDirty; clone.normsDirty = normsDirty; clone.hasChanges = hasChanges; hasChanges = false; } if (doClone) { if (deletedDocs != null) { deletedDocsRef.IncRef(); clone.deletedDocs = deletedDocs; clone.deletedDocsRef = deletedDocsRef; } } else { if (!deletionsUpToDate) { // load deleted docs System.Diagnostics.Debug.Assert(clone.deletedDocs == null); clone.LoadDeletedDocs(); } else if (deletedDocs != null) { deletedDocsRef.IncRef(); clone.deletedDocs = deletedDocs; clone.deletedDocsRef = deletedDocsRef; } } clone.SetDisableFakeNorms(GetDisableFakeNorms()); clone.norms = new System.Collections.Hashtable(); // Clone norms for (int i = 0; i < fieldNormsChanged.Length; i++) { // Clone unchanged norms to the cloned reader if (doClone || !fieldNormsChanged[i]) { System.String curField = core.fieldInfos.FieldInfo(i).name; Norm norm = (Norm) this.norms[curField]; if (norm != null) clone.norms[curField] = norm.Clone(); } } // If we are not cloning, then this will open anew // any norms that have changed: clone.OpenNorms(si.GetUseCompoundFile()?core.GetCFSReader():Directory(), readBufferSize); success = true; } finally { if (!success) { // An exception occured during reopen, we have to decRef the norms // that we incRef'ed already and close singleNormsStream and FieldsReader clone.DecRef(); } } return clone; } }
// Flush all pending changes to the next generation // separate norms file. public void ReWrite(SegmentInfo si) { System.Diagnostics.Debug.Assert(refCount > 0 && (origNorm == null || origNorm.refCount > 0), "refCount=" + refCount + " origNorm=" + origNorm); // NOTE: norms are re-written in regular directory, not cfs si.AdvanceNormGen(this.number); IndexOutput out_Renamed = Enclosing_Instance.Directory().CreateOutput(si.GetNormFileName(this.number)); try { out_Renamed.WriteBytes(bytes, Enclosing_Instance.MaxDoc()); } finally { out_Renamed.Close(); } this.dirty = false; }
internal virtual SegmentReader ReopenSegment(SegmentInfo si) { lock (this) { bool deletionsUpToDate = (this.si.HasDeletions() == si.HasDeletions()) && (!si.HasDeletions() || this.si.GetDelFileName().Equals(si.GetDelFileName())); bool normsUpToDate = true; bool[] fieldNormsChanged = new bool[fieldInfos.Size()]; if (normsUpToDate) { for (int i = 0; i < fieldInfos.Size(); i++) { if (!this.si.GetNormFileName(i).Equals(si.GetNormFileName(i))) { normsUpToDate = false; fieldNormsChanged[i] = true; } } } if (normsUpToDate && deletionsUpToDate) { return(this); } // clone reader SegmentReader clone; if (readOnly) { clone = new ReadOnlySegmentReader(); } else { clone = new SegmentReader(); } bool success = false; try { clone.readOnly = readOnly; clone.directory = directory; clone.si = si; clone.segment = segment; clone.readBufferSize = readBufferSize; clone.cfsReader = cfsReader; clone.storeCFSReader = storeCFSReader; clone.fieldInfos = fieldInfos; clone.tis = tis; clone.freqStream = freqStream; clone.proxStream = proxStream; clone.termVectorsReaderOrig = termVectorsReaderOrig; // we have to open a new FieldsReader, because it is not thread-safe // and can thus not be shared among multiple SegmentReaders // TODO: Change this in case FieldsReader becomes thread-safe in the future System.String fieldsSegment; Directory storeDir = Directory(); if (si.GetDocStoreOffset() != -1) { fieldsSegment = si.GetDocStoreSegment(); if (storeCFSReader != null) { storeDir = storeCFSReader; } } else { fieldsSegment = segment; if (cfsReader != null) { storeDir = cfsReader; } } if (fieldsReader != null) { clone.fieldsReader = new FieldsReader(storeDir, fieldsSegment, fieldInfos, readBufferSize, si.GetDocStoreOffset(), si.docCount); } if (!deletionsUpToDate) { // load deleted docs clone.deletedDocs = null; clone.LoadDeletedDocs(); } else { clone.deletedDocs = this.deletedDocs; } clone.norms = new System.Collections.Hashtable(); if (!normsUpToDate) { // load norms for (int i = 0; i < fieldNormsChanged.Length; i++) { // copy unchanged norms to the cloned reader and incRef those norms if (!fieldNormsChanged[i]) { System.String curField = fieldInfos.FieldInfo(i).name; Norm norm = (Norm)this.norms[curField]; norm.IncRef(); clone.norms[curField] = norm; } } clone.OpenNorms(si.GetUseCompoundFile() ? cfsReader : Directory(), readBufferSize); } else { System.Collections.IEnumerator it = norms.Keys.GetEnumerator(); while (it.MoveNext()) { System.String field = (System.String)it.Current; Norm norm = (Norm)norms[field]; norm.IncRef(); clone.norms[field] = norm; } } if (clone.singleNormStream == null) { for (int i = 0; i < fieldInfos.Size(); i++) { FieldInfo fi = fieldInfos.FieldInfo(i); if (fi.isIndexed && !fi.omitNorms) { Directory d = si.GetUseCompoundFile() ? cfsReader : Directory(); System.String fileName = si.GetNormFileName(fi.number); if (si.HasSeparateNorms(fi.number)) { continue; } if (fileName.EndsWith("." + IndexFileNames.NORMS_EXTENSION)) { clone.singleNormStream = d.OpenInput(fileName, readBufferSize); break; } } } } success = true; } finally { if (this.referencedSegmentReader != null) { // this reader shares resources with another SegmentReader, // so we increment the other readers refCount. We don't // increment the refCount of the norms because we did // that already for the shared norms clone.referencedSegmentReader = this.referencedSegmentReader; referencedSegmentReader.IncRefReaderNotNorms(); } else { // this reader wasn't reopened, so we increment this // readers refCount clone.referencedSegmentReader = this; IncRefReaderNotNorms(); } if (!success) { // An exception occured during reopen, we have to decRef the norms // that we incRef'ed already and close singleNormsStream and FieldsReader clone.DecRef(); } } return(clone); } }
internal void ReWrite(SegmentInfo si) { // NOTE: norms are re-written in regular directory, not cfs si.AdvanceNormGen(this.number); IndexOutput out_Renamed = Enclosing_Instance.Directory().CreateOutput(si.GetNormFileName(this.number)); try { out_Renamed.WriteBytes(bytes, Enclosing_Instance.MaxDoc()); } finally { out_Renamed.Close(); } this.dirty = false; }
// Flush all pending changes to the next generation // separate norms file. public void ReWrite(SegmentInfo si) { System.Diagnostics.Debug.Assert(refCount > 0 && (origNorm == null || origNorm.refCount > 0), "refCount=" + refCount + " origNorm=" + origNorm); // NOTE: norms are re-written in regular directory, not cfs si.AdvanceNormGen(this.number); string normFileName = si.GetNormFileName(this.number); IndexOutput @out = enclosingInstance.Directory().CreateOutput(normFileName); bool success = false; try { try { @out.WriteBytes(bytes, enclosingInstance.MaxDoc()); } finally { @out.Close(); } success = true; } finally { if (!success) { try { enclosingInstance.Directory().DeleteFile(normFileName); } catch (Exception t) { // suppress this so we keep throwing the // original exception } } } this.dirty = false; }
internal virtual SegmentReader ReopenSegment(SegmentInfo si) { lock (this) { bool deletionsUpToDate = (this.si.HasDeletions() == si.HasDeletions()) && (!si.HasDeletions() || this.si.GetDelFileName().Equals(si.GetDelFileName())); bool normsUpToDate = true; bool[] fieldNormsChanged = new bool[fieldInfos.Size()]; if (normsUpToDate) { for (int i = 0; i < fieldInfos.Size(); i++) { if (!this.si.GetNormFileName(i).Equals(si.GetNormFileName(i))) { normsUpToDate = false; fieldNormsChanged[i] = true; } } } if (normsUpToDate && deletionsUpToDate) { return this; } // clone reader SegmentReader clone; if (readOnly) clone = new ReadOnlySegmentReader(); else clone = new SegmentReader(); bool success = false; try { clone.readOnly = readOnly; clone.directory = directory; clone.si = si; clone.segment = segment; clone.readBufferSize = readBufferSize; clone.cfsReader = cfsReader; clone.storeCFSReader = storeCFSReader; clone.fieldInfos = fieldInfos; clone.tis = tis; clone.freqStream = freqStream; clone.proxStream = proxStream; clone.termVectorsReaderOrig = termVectorsReaderOrig; // we have to open a new FieldsReader, because it is not thread-safe // and can thus not be shared among multiple SegmentReaders // TODO: Change this in case FieldsReader becomes thread-safe in the future System.String fieldsSegment; Directory storeDir = Directory(); if (si.GetDocStoreOffset() != - 1) { fieldsSegment = si.GetDocStoreSegment(); if (storeCFSReader != null) { storeDir = storeCFSReader; } } else { fieldsSegment = segment; if (cfsReader != null) { storeDir = cfsReader; } } if (fieldsReader != null) { clone.fieldsReader = new FieldsReader(storeDir, fieldsSegment, fieldInfos, readBufferSize, si.GetDocStoreOffset(), si.docCount); } if (!deletionsUpToDate) { // load deleted docs clone.deletedDocs = null; clone.LoadDeletedDocs(); } else { clone.deletedDocs = this.deletedDocs; } clone.norms = new System.Collections.Hashtable(); if (!normsUpToDate) { // load norms for (int i = 0; i < fieldNormsChanged.Length; i++) { // copy unchanged norms to the cloned reader and incRef those norms if (!fieldNormsChanged[i]) { System.String curField = fieldInfos.FieldInfo(i).name; Norm norm = (Norm) this.norms[curField]; norm.IncRef(); clone.norms[curField] = norm; } } clone.OpenNorms(si.GetUseCompoundFile() ? cfsReader : Directory(), readBufferSize); } else { System.Collections.IEnumerator it = norms.Keys.GetEnumerator(); while (it.MoveNext()) { System.String field = (System.String) it.Current; Norm norm = (Norm) norms[field]; norm.IncRef(); clone.norms[field] = norm; } } if (clone.singleNormStream == null) { for (int i = 0; i < fieldInfos.Size(); i++) { FieldInfo fi = fieldInfos.FieldInfo(i); if (fi.isIndexed && !fi.omitNorms) { Directory d = si.GetUseCompoundFile() ? cfsReader : Directory(); System.String fileName = si.GetNormFileName(fi.number); if (si.HasSeparateNorms(fi.number)) { continue; } if (fileName.EndsWith("." + IndexFileNames.NORMS_EXTENSION)) { clone.singleNormStream = d.OpenInput(fileName, readBufferSize); break; } } } } success = true; } finally { if (this.referencedSegmentReader != null) { // this reader shares resources with another SegmentReader, // so we increment the other readers refCount. We don't // increment the refCount of the norms because we did // that already for the shared norms clone.referencedSegmentReader = this.referencedSegmentReader; referencedSegmentReader.IncRefReaderNotNorms(); } else { // this reader wasn't reopened, so we increment this // readers refCount clone.referencedSegmentReader = this; IncRefReaderNotNorms(); } if (!success) { // An exception occured during reopen, we have to decRef the norms // that we incRef'ed already and close singleNormsStream and FieldsReader clone.DecRef(); } } return clone; } }
/// <summary>Determine index files that are no longer referenced /// and therefore should be deleted. This is called once /// (by the writer), and then subsequently we add onto /// deletable any files that are no longer needed at the /// point that we create the unused file (eg when merging /// segments), and we only remove from deletable when a /// file is successfully deleted. /// </summary> public void FindDeletableFiles() { // Gather all "current" segments: System.Collections.Hashtable current = new System.Collections.Hashtable(); for (int j = 0; j < segmentInfos.Count; j++) { SegmentInfo segmentInfo = (SegmentInfo)segmentInfos[j]; current[segmentInfo.name] = segmentInfo; } // Then go through all files in the Directory that are // Lucene index files, and add to deletable if they are // not referenced by the current segments info: System.String segmentsInfosFileName = segmentInfos.GetCurrentSegmentFileName(); IndexFileNameFilter filter = IndexFileNameFilter.GetFilter(); System.String[] files = directory.List(); for (int i = 0; i < files.Length; i++) { if (filter.Accept(null, files[i]) && !files[i].Equals(segmentsInfosFileName) && !files[i].Equals(IndexFileNames.SEGMENTS_GEN)) { System.String segmentName; System.String extension; // First remove any extension: int loc = files[i].IndexOf((System.Char) '.'); if (loc != -1) { extension = files[i].Substring(1 + loc); segmentName = files[i].Substring(0, (loc) - (0)); } else { extension = null; segmentName = files[i]; } // Then, remove any generation count: loc = segmentName.IndexOf((System.Char) '_', 1); if (loc != -1) { segmentName = segmentName.Substring(0, (loc) - (0)); } // Delete this file if it's not a "current" segment, // or, it is a single index file but there is now a // corresponding compound file: bool doDelete = false; if (!current.ContainsKey(segmentName)) { // Delete if segment is not referenced: doDelete = true; } else { // OK, segment is referenced, but file may still // be orphan'd: SegmentInfo info = (SegmentInfo)current[segmentName]; if (filter.IsCFSFile(files[i]) && info.GetUseCompoundFile()) { // This file is in fact stored in a CFS file for // this segment: doDelete = true; } else { Pattern p = new System.Text.RegularExpressions.Regex("s\\d+"); if ("del".Equals(extension)) { // This is a _segmentName_N.del file: if (!files[i].Equals(info.GetDelFileName())) { // If this is a seperate .del file, but it // doesn't match the current del filename for // this segment, then delete it: doDelete = true; } } else if (extension != null && extension.StartsWith("s") && p.Match(extension).Success) { int field = System.Int32.Parse(extension.Substring(1)); // This is a _segmentName_N.sX file: if (!files[i].Equals(info.GetNormFileName(field))) { // This is an orphan'd separate norms file: doDelete = true; } } else if ("cfs".Equals(extension) && !info.GetUseCompoundFile()) { // This is a partially written // _segmentName.cfs: doDelete = true; } } } if (doDelete) { AddDeletableFile(files[i]); if (infoStream != null) { infoStream.WriteLine("IndexFileDeleter: file \"" + files[i] + "\" is unreferenced in index and will be deleted on next commit"); } } } } }