Add() public méthode

Appends the provided SegmentCommitInfo.
public Add ( Lucene.Net.Index.SegmentCommitInfo si ) : void
si Lucene.Net.Index.SegmentCommitInfo
Résultat void
Exemple #1
0
        /// <summary> Adds a document to this index, using the provided analyzer instead of the
        /// value of {@link #GetAnalyzer()}.  If the document contains more than
        /// {@link #SetMaxFieldLength(int)} terms for a given field, the remainder are
        /// discarded.
        /// </summary>
        public virtual void  AddDocument(Document doc, Analyzer analyzer)
        {
            DocumentWriter dw = new DocumentWriter(ramDirectory, analyzer, this);

            dw.SetInfoStream(infoStream);
            System.String segmentName = NewSegmentName();
            dw.AddDocument(segmentName, doc);
            lock (this)
            {
                segmentInfos.Add(new SegmentInfo(segmentName, 1, ramDirectory));
                MaybeMergeSegments();
            }
        }
Exemple #2
0
        public virtual void Split(DirectoryInfo destDir, string[] segs)
        {
            destDir.Create();
            FSDirectory  destFSDir = FSDirectory.Open(destDir);
            SegmentInfos destInfos = new SegmentInfos();

            destInfos.Counter = Infos.Counter;
            foreach (string n in segs)
            {
                SegmentCommitInfo infoPerCommit = GetInfo(n);
                SegmentInfo       info          = infoPerCommit.Info;
                // Same info just changing the dir:
                SegmentInfo newInfo = new SegmentInfo(destFSDir, info.Version, info.Name, info.DocCount, info.UseCompoundFile, info.Codec, info.Diagnostics);
                destInfos.Add(new SegmentCommitInfo(newInfo, infoPerCommit.DelCount, infoPerCommit.DelGen, infoPerCommit.FieldInfosGen));
                // now copy files over
                ICollection <string> files = infoPerCommit.GetFiles();
                foreach (string srcName in files)
                {
                    FileInfo srcFile  = new FileInfo(Path.Combine(dir.FullName, srcName));
                    FileInfo destFile = new FileInfo(Path.Combine(destDir.FullName, srcName));
                    CopyFile(srcFile, destFile);
                }
            }
            destInfos.Changed();
            destInfos.Commit(destFSDir);
            // Console.WriteLine("destDir:"+destDir.getAbsolutePath());
        }
Exemple #3
0
        /// <summary> Returns a copy of this instance, also copying each
        /// SegmentInfo.
        /// </summary>

        public override System.Object Clone()
        {
            SegmentInfos si = new SegmentInfos();

            for (int i = 0; i < base.Count; i++)
            {
                si.Add(((SegmentInfo) base[i]).Clone());
            }
            si.generation     = this.generation;
            si.lastGeneration = this.lastGeneration;
            return(si);
        }
        /// <summary> Returns a copy of this instance, also copying each
        /// SegmentInfo.
        /// </summary>

        public override System.Object Clone()
        {
            SegmentInfos sis = new SegmentInfos();

            // Copy Fields. const and static fields are ignored
            sis.counter        = this.counter;
            sis.version        = this.version;
            sis.generation     = this.generation;
            sis.lastGeneration = this.lastGeneration;

            for (int i = 0; i < this.Count; i++)
            {
                sis.Add(((SegmentInfo)this[i]).Clone());
            }
            return(sis);
        }
Exemple #5
0
        /// <summary> Returns a copy of this instance, also copying each
        /// SegmentInfo.
        /// </summary>

        public override System.Object Clone()
        {
            SegmentInfos sis = new SegmentInfos();

            for (int i = 0; i < this.Count; i++)
            {
                sis.Add(((SegmentInfo)this[i]).Clone());
            }
            sis.counter        = this.counter;
            sis.generation     = this.generation;
            sis.lastGeneration = this.lastGeneration;
            // sis.pendingSegnOutput = this.pendingSegnOutput; // {{Aroush-2.9}} needed?
            sis.userData = new System.Collections.Generic.Dictionary <string, string>(userData);
            sis.version  = this.version;
            return(sis);
        }
Exemple #6
0
        /// <summary> Returns a copy of this instance, also copying each
        /// SegmentInfo.
        /// </summary>

        public SegmentInfos Clone()
        {
            SegmentInfos sis = new SegmentInfos();

            for (int i = 0; i < this.Count; i++)
            {
                sis.Add(this[i].Clone());
            }
            sis.counter        = this.counter;
            sis.generation     = this.generation;
            sis.lastGeneration = this.lastGeneration;
            // sis.pendingSegnOutput = this.pendingSegnOutput; // {{Aroush-2.9}} needed?
            sis.userData = new Dictionary <string, string>(userData);
            sis.version  = this.version;
            return(sis);
        }
 /// <summary> Returns a copy of this instance, also copying each
 /// SegmentInfo.
 /// </summary>
 public override object Clone()
 {
     SegmentInfos si = new SegmentInfos();
     for (int i = 0; i < base.Count; i++)
     {
         si.Add(((SegmentInfo)base[i]).Clone());
     }
     si.counter = this.counter;
     si.version = this.version;
     si.generation = this.generation;
     si.lastGeneration = this.lastGeneration;
     return si;
 }
Exemple #8
0
		/// <summary> Returns a copy of this instance, also copying each
		/// SegmentInfo.
		/// </summary>
		
		public override System.Object Clone()
		{
            SegmentInfos sis = new SegmentInfos();
            for (int i = 0; i < this.Count; i++)
            {
                sis.Add(((SegmentInfo) this[i]).Clone());
            }
            sis.counter = this.counter;
            sis.generation = this.generation;
            sis.lastGeneration = this.lastGeneration;
            // sis.pendingSegnOutput = this.pendingSegnOutput; // {{Aroush-2.9}} needed?
            sis.userData = new System.Collections.Generic.Dictionary<string, string>(userData);
            sis.version = this.version;
            return sis;
		}
Exemple #9
0
        // For infoStream output
        internal virtual SegmentInfos ToLiveInfos(SegmentInfos sis)
        {
            lock (this)
            {
                SegmentInfos newSIS = new SegmentInfos();
                IDictionary<SegmentCommitInfo, SegmentCommitInfo> liveSIS = new Dictionary<SegmentCommitInfo, SegmentCommitInfo>();
                foreach (SegmentCommitInfo info in segmentInfos.Segments)
                {
                    liveSIS[info] = info;
                }
                foreach (SegmentCommitInfo info in sis.Segments)
                {
                    SegmentCommitInfo infoMod = info;
                    SegmentCommitInfo liveInfo;
                    if (liveSIS.TryGetValue(info, out liveInfo))
                    {
                        infoMod = liveInfo;
                    }
                    newSIS.Add(infoMod);
                }

                return newSIS;
            }
        }
Exemple #10
0
        /// <summary>Returns true if index is clean, else false.</summary>
        public static bool Check(Directory dir, bool doFix)
        {
            System.Globalization.NumberFormatInfo nf = System.Globalization.CultureInfo.CurrentCulture.NumberFormat;
            SegmentInfos sis = new SegmentInfos();

            try
            {
                sis.Read(dir);
            }
            catch (System.Exception t)
            {
                out_Renamed.WriteLine("ERROR: could not read any segments file in directory");
                out_Renamed.Write(t.StackTrace);
                out_Renamed.Flush();
                return(false);
            }

            int numSegments = sis.Count;

            System.String segmentsFileName = sis.GetCurrentSegmentFileName();
            IndexInput    input            = null;

            try
            {
                input = dir.OpenInput(segmentsFileName);
            }
            catch (System.Exception t)
            {
                out_Renamed.WriteLine("ERROR: could not open segments file in directory");
                out_Renamed.Write(t.StackTrace);
                out_Renamed.Flush();
                return(false);
            }
            int format = 0;

            try
            {
                format = input.ReadInt();
            }
            catch (System.Exception t)
            {
                out_Renamed.WriteLine("ERROR: could not read segment file version in directory");
                out_Renamed.Write(t.StackTrace);
                out_Renamed.Flush();
                return(false);
            }
            finally
            {
                if (input != null)
                {
                    input.Close();
                }
            }

            System.String sFormat = "";
            bool          skip    = false;

            if (format == SegmentInfos.FORMAT)
            {
                sFormat = "FORMAT [Lucene Pre-2.1]";
            }
            if (format == SegmentInfos.FORMAT_LOCKLESS)
            {
                sFormat = "FORMAT_LOCKLESS [Lucene 2.1]";
            }
            else if (format == SegmentInfos.FORMAT_SINGLE_NORM_FILE)
            {
                sFormat = "FORMAT_SINGLE_NORM_FILE [Lucene 2.2]";
            }
            else if (format == SegmentInfos.FORMAT_SHARED_DOC_STORE)
            {
                sFormat = "FORMAT_SHARED_DOC_STORE [Lucene 2.3]";
            }
            else if (format < SegmentInfos.FORMAT_SHARED_DOC_STORE)
            {
                sFormat = "int=" + format + " [newer version of Lucene than this tool]";
                skip    = true;
            }
            else
            {
                sFormat = format + " [Lucene 1.3 or prior]";
            }

            out_Renamed.WriteLine("Segments file=" + segmentsFileName + " numSegments=" + numSegments + " version=" + sFormat);

            if (skip)
            {
                out_Renamed.WriteLine("\nERROR: this index appears to be created by a newer version of Lucene than this tool was compiled on; please re-compile this tool on the matching version of Lucene; exiting");
                return(false);
            }

            SegmentInfos newSIS = (SegmentInfos)sis.Clone();

            newSIS.Clear();
            bool changed         = false;
            int  totLoseDocCount = 0;
            int  numBadSegments  = 0;

            for (int i = 0; i < numSegments; i++)
            {
                SegmentInfo info = sis.Info(i);
                out_Renamed.WriteLine("  " + (1 + i) + " of " + numSegments + ": name=" + info.name + " docCount=" + info.docCount);
                int toLoseDocCount = info.docCount;

                SegmentReader reader = null;

                try
                {
                    out_Renamed.WriteLine("    compound=" + info.GetUseCompoundFile());
                    out_Renamed.WriteLine("    numFiles=" + info.Files().Count);
                    out_Renamed.WriteLine(String.Format(nf, "    size (MB)={0:f}", new Object[] { (info.SizeInBytes() / (1024.0 * 1024.0)) }));
                    int docStoreOffset = info.GetDocStoreOffset();
                    if (docStoreOffset != -1)
                    {
                        out_Renamed.WriteLine("    docStoreOffset=" + docStoreOffset);
                        out_Renamed.WriteLine("    docStoreSegment=" + info.GetDocStoreSegment());
                        out_Renamed.WriteLine("    docStoreIsCompoundFile=" + info.GetDocStoreIsCompoundFile());
                    }
                    System.String delFileName = info.GetDelFileName();
                    if (delFileName == null)
                    {
                        out_Renamed.WriteLine("    no deletions");
                    }
                    else
                    {
                        out_Renamed.WriteLine("    has deletions [delFileName=" + delFileName + "]");
                    }
                    out_Renamed.Write("    test: open reader.........");
                    reader = SegmentReader.Get(info);
                    int numDocs = reader.NumDocs();
                    toLoseDocCount = numDocs;
                    if (reader.HasDeletions())
                    {
                        out_Renamed.WriteLine("OK [" + (info.docCount - numDocs) + " deleted docs]");
                    }
                    else
                    {
                        out_Renamed.WriteLine("OK");
                    }

                    out_Renamed.Write("    test: fields, norms.......");
                    System.Collections.IDictionary fieldNames = (System.Collections.IDictionary)reader.GetFieldNames(IndexReader.FieldOption.ALL);
                    System.Collections.IEnumerator it         = fieldNames.Keys.GetEnumerator();
                    while (it.MoveNext())
                    {
                        System.String fieldName = (System.String)it.Current;
                        byte[]        b         = reader.Norms(fieldName);
                        if (b.Length != info.docCount)
                        {
                            throw new System.SystemException("norms for field \"" + fieldName + "\" is length " + b.Length + " != maxDoc " + info.docCount);
                        }
                    }
                    out_Renamed.WriteLine("OK [" + fieldNames.Count + " fields]");

                    out_Renamed.Write("    test: terms, freq, prox...");
                    TermEnum      termEnum      = reader.Terms();
                    TermPositions termPositions = reader.TermPositions();

                    // Used only to count up # deleted docs for this
                    // term
                    MySegmentTermDocs myTermDocs = new MySegmentTermDocs(reader);

                    long termCount = 0;
                    long totFreq   = 0;
                    long totPos    = 0;
                    while (termEnum.Next())
                    {
                        termCount++;
                        Term term    = termEnum.Term();
                        int  docFreq = termEnum.DocFreq();
                        termPositions.Seek(term);
                        int lastDoc = -1;
                        int freq0   = 0;
                        totFreq += docFreq;
                        while (termPositions.Next())
                        {
                            freq0++;
                            int doc  = termPositions.Doc();
                            int freq = termPositions.Freq();
                            if (doc <= lastDoc)
                            {
                                throw new System.SystemException("term " + term + ": doc " + doc + " < lastDoc " + lastDoc);
                            }
                            lastDoc = doc;
                            if (freq <= 0)
                            {
                                throw new System.SystemException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds");
                            }

                            int lastPos = -1;
                            totPos += freq;
                            for (int j = 0; j < freq; j++)
                            {
                                int pos = termPositions.NextPosition();
                                if (pos < 0)
                                {
                                    throw new System.SystemException("term " + term + ": doc " + doc + ": pos " + pos + " is out of bounds");
                                }
                                if (pos <= lastPos)
                                {
                                    throw new System.SystemException("term " + term + ": doc " + doc + ": pos " + pos + " < lastPos " + lastPos);
                                }
                            }
                        }

                        // Now count how many deleted docs occurred in
                        // this term:
                        int delCount;
                        if (reader.HasDeletions())
                        {
                            myTermDocs.Seek(term);
                            while (myTermDocs.Next())
                            {
                            }
                            delCount = myTermDocs.delCount;
                        }
                        else
                        {
                            delCount = 0;
                        }

                        if (freq0 + delCount != docFreq)
                        {
                            throw new System.SystemException("term " + term + " docFreq=" + docFreq + " != num docs seen " + freq0 + " + num docs deleted " + delCount);
                        }
                    }

                    out_Renamed.WriteLine("OK [" + termCount + " terms; " + totFreq + " terms/docs pairs; " + totPos + " tokens]");

                    out_Renamed.Write("    test: stored fields.......");
                    int  docCount  = 0;
                    long totFields = 0;
                    for (int j = 0; j < info.docCount; j++)
                    {
                        if (!reader.IsDeleted(j))
                        {
                            docCount++;
                            Document doc = reader.Document(j);
                            totFields += doc.GetFields().Count;
                        }
                    }

                    if (docCount != reader.NumDocs())
                    {
                        throw new System.SystemException("docCount=" + docCount + " but saw " + docCount + " undeleted docs");
                    }

                    out_Renamed.WriteLine(String.Format(nf, "OK [{0:d} total field count; avg {1:f} fields per doc]", new Object[] { totFields, (((float)totFields) / docCount) }));

                    out_Renamed.Write("    test: term vectors........");
                    int totVectors = 0;
                    for (int j = 0; j < info.docCount; j++)
                    {
                        if (!reader.IsDeleted(j))
                        {
                            TermFreqVector[] tfv = reader.GetTermFreqVectors(j);
                            if (tfv != null)
                            {
                                totVectors += tfv.Length;
                            }
                        }
                    }

                    out_Renamed.WriteLine(String.Format(nf, "OK [{0:d} total vector count; avg {1:f} term/freq vector fields per doc]", new Object[] { totVectors, (((float)totVectors) / docCount) }));
                    out_Renamed.WriteLine("");
                }
                catch (System.Exception t)
                {
                    out_Renamed.WriteLine("FAILED");
                    System.String comment;
                    if (doFix)
                    {
                        comment = "will remove reference to this segment (-fix is specified)";
                    }
                    else
                    {
                        comment = "would remove reference to this segment (-fix was not specified)";
                    }
                    out_Renamed.WriteLine("    WARNING: " + comment + "; full exception:");
                    out_Renamed.Write(t.StackTrace);
                    out_Renamed.Flush();
                    out_Renamed.WriteLine("");
                    totLoseDocCount += toLoseDocCount;
                    numBadSegments++;
                    changed = true;
                    continue;
                }
                finally
                {
                    if (reader != null)
                    {
                        reader.Close();
                    }
                }

                // Keeper
                newSIS.Add(info.Clone());
            }

            if (!changed)
            {
                out_Renamed.WriteLine("No problems were detected with this index.\n");
                return(true);
            }
            else
            {
                out_Renamed.WriteLine("WARNING: " + numBadSegments + " broken segments detected");
                if (doFix)
                {
                    out_Renamed.WriteLine("WARNING: " + totLoseDocCount + " documents will be lost");
                }
                else
                {
                    out_Renamed.WriteLine("WARNING: " + totLoseDocCount + " documents would be lost if -fix were specified");
                }
                out_Renamed.WriteLine();
            }

            if (doFix)
            {
                out_Renamed.WriteLine("NOTE: will write new segments file in 5 seconds; this will remove " + totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!");
                for (int i = 0; i < 5; i++)
                {
                    try
                    {
                        System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 1000));
                    }
                    catch (System.Threading.ThreadInterruptedException)
                    {
                        SupportClass.ThreadClass.Current().Interrupt();
                        i--;
                        continue;
                    }

                    out_Renamed.WriteLine("  " + (5 - i) + "...");
                }
                out_Renamed.Write("Writing...");
                try
                {
                    newSIS.Write(dir);
                }
                catch (System.Exception t)
                {
                    out_Renamed.WriteLine("FAILED; exiting");
                    out_Renamed.Write(t.StackTrace);
                    out_Renamed.Flush();
                    return(false);
                }
                out_Renamed.WriteLine("OK");
                out_Renamed.WriteLine("Wrote new segments file \"" + newSIS.GetCurrentSegmentFileName() + "\"");
            }
            else
            {
                out_Renamed.WriteLine("NOTE: would write new segments file [-fix was not specified]");
            }
            out_Renamed.WriteLine("");

            return(false);
        }
Exemple #11
0
        /// <summary>Checks if any merges are now necessary and returns a
        /// <see cref="MergePolicy.MergeSpecification" /> if so.  A merge
        /// is necessary when there are more than <see cref="MergeFactor" />
        /// segments at a given level.  When
        /// multiple levels have too many segments, this method
        /// will return multiple merges, allowing the <see cref="MergeScheduler" />
        /// to use concurrency.
        /// </summary>
        public override MergeSpecification FindMerges(SegmentInfos infos, IState state)
        {
            int numSegments = infos.Count;

            if (Verbose())
            {
                Message("findMerges: " + numSegments + " segments");
            }

            // Compute levels, which is just log (base mergeFactor)
            // of the size of each segment
            float[] levels = new float[numSegments];
            float   norm   = (float)System.Math.Log(mergeFactor);

            for (int i = 0; i < numSegments; i++)
            {
                SegmentInfo info = infos.Info(i);
                long        size = Size(info, state);

                // Floor tiny segments
                if (size < 1)
                {
                    size = 1;
                }
                levels[i] = (float)System.Math.Log(size) / norm;
            }

            float levelFloor;

            if (minMergeSize <= 0)
            {
                levelFloor = (float)0.0;
            }
            else
            {
                levelFloor = (float)(System.Math.Log(minMergeSize) / norm);
            }

            // Now, we quantize the log values into levels.  The
            // first level is any segment whose log size is within
            // LEVEL_LOG_SPAN of the max size, or, who has such as
            // segment "to the right".  Then, we find the max of all
            // other segments and use that to define the next level
            // segment, etc.

            MergeSpecification spec = null;

            int start = 0;

            while (start < numSegments)
            {
                // Find max level of all segments not already
                // quantized.
                float maxLevel = levels[start];
                for (int i = 1 + start; i < numSegments; i++)
                {
                    float level = levels[i];
                    if (level > maxLevel)
                    {
                        maxLevel = level;
                    }
                }

                // Now search backwards for the rightmost segment that
                // falls into this level:
                float levelBottom;
                if (maxLevel < levelFloor)
                {
                    // All remaining segments fall into the min level
                    levelBottom = -1.0F;
                }
                else
                {
                    levelBottom = (float)(maxLevel - LEVEL_LOG_SPAN);

                    // Force a boundary at the level floor
                    if (levelBottom < levelFloor && maxLevel >= levelFloor)
                    {
                        levelBottom = levelFloor;
                    }
                }

                int upto = numSegments - 1;
                while (upto >= start)
                {
                    if (levels[upto] >= levelBottom)
                    {
                        break;
                    }
                    upto--;
                }
                if (Verbose())
                {
                    Message("  level " + levelBottom + " to " + maxLevel + ": " + (1 + upto - start) + " segments");
                }

                // Finally, record all merges that are viable at this level:
                int end = start + mergeFactor;
                while (end <= 1 + upto)
                {
                    SegmentInfos segmentsToMerge    = new SegmentInfos();
                    long         totalSize          = 0;
                    var          processed          = 0;
                    var          largeSegmentsCount = 0;

                    // we have a limitation: non-contiguous segment merges aren't allowed
                    for (int i = start; i < end; i++)
                    {
                        SegmentInfo info        = infos.Info(i);
                        var         segmentSize = Size(info, state);
                        if (segmentSize >= maxMergeSize || SizeDocs(info, state) >= maxMergeDocs)
                        {
                            // a single segment is larger then the maximum allowed
                            break;
                        }

                        var newTotalSize = totalSize + segmentSize;
                        if (newTotalSize > maxMergeSize)
                        {
                            // the total size of all segments will exceed the maximum allowed size
                            break;
                        }

                        if (segmentSize > largeSegmentSize && ++largeSegmentsCount > numberOfLargeSegmentsToMergeInSingleBatch)
                        {
                            // limit the number of merged large segments
                            break;
                        }

                        totalSize = newTotalSize;
                        processed++;
                        segmentsToMerge.Add(info);
                    }

                    if (segmentsToMerge.Count > 1)
                    {
                        if (spec == null)
                        {
                            spec = new MergeSpecification();
                        }
                        if (Verbose())
                        {
                            Message("    " + segmentsToMerge.Count + " segments were added to this merge, total size: " + totalSize + " bytes");
                        }
                        spec.Add(MakeOneMerge(infos, segmentsToMerge, state));
                    }
                    else if (Verbose())
                    {
                        Message("    " + start + " to " + end + ": doesn't have enough segments to execute a merge");
                    }

                    if (processed == 0)
                    {
                        // no segments to merge, we skip one and try to recalculate using the rest ones
                        processed = 1;
                    }

                    start += processed;

                    if (start == end)
                    {
                        end += mergeFactor;
                    }
                }

                start = 1 + upto;
            }

            return(spec);
        }
 public virtual void Split(DirectoryInfo destDir, string[] segs)
 {
     destDir.Create();
     FSDirectory destFSDir = FSDirectory.Open(destDir);
     SegmentInfos destInfos = new SegmentInfos();
     destInfos.Counter = infos.Counter;
     foreach (string n in segs)
     {
         SegmentCommitInfo infoPerCommit = GetInfo(n);
         SegmentInfo info = infoPerCommit.Info;
         // Same info just changing the dir:
         SegmentInfo newInfo = new SegmentInfo(destFSDir, info.Version, info.Name, info.DocCount, info.UseCompoundFile, info.Codec, info.Diagnostics);
         destInfos.Add(new SegmentCommitInfo(newInfo, infoPerCommit.DelCount, infoPerCommit.DelGen, infoPerCommit.FieldInfosGen));
         // now copy files over
         ICollection<string> files = infoPerCommit.Files();
         foreach (string srcName in files)
         {
             FileInfo srcFile = new FileInfo(Path.Combine(dir.FullName, srcName));
             FileInfo destFile = new FileInfo(Path.Combine(destDir.FullName, srcName));
             CopyFile(srcFile, destFile);
         }
     }
     destInfos.Changed();
     destInfos.Commit(destFSDir);
     // Console.WriteLine("destDir:"+destDir.getAbsolutePath());
 }
Exemple #13
0
		/// <summary> Returns a copy of this instance, also copying each
		/// SegmentInfo.
		/// </summary>
		
		public override System.Object Clone()
		{
			SegmentInfos sis = new SegmentInfos();

			// Copy Fields. const and static fields are ignored
			sis.counter = this.counter;
			sis.version = this.version;
			sis.generation = this.generation;
			sis.lastGeneration = this.lastGeneration;

			for (int i = 0; i < this.Count; i++)
			{
				sis.Add(((SegmentInfo)this[i]).Clone());
			}
			return sis;
		}