Command-line tool that enables listing segments in an index, copying specific segments to another index, and deleting segments from an index.

This tool does file-level copying of segments files. This means it's unable to split apart a single segment into multiple segments. For example if your index is a single segment, this tool won't help. Also, it does basic file-level copying (using simple FileInfo) so it will not work with non FSDirectory Directory impls.

@lucene.experimental You can easily accidentally remove segments from your index so be careful!
Example #1
0
        public static void Main(string[] args)
        {
            if (args.Length < 2)
            {
                // LUCENENET specific - our wrapper console shows the correct usage
                throw new ArgumentException();
                //Console.Error.WriteLine("Usage: IndexSplitter <srcDir> -l (list the segments and their sizes)");
                //Console.Error.WriteLine("IndexSplitter <srcDir> <destDir> <segments>+");
                //Console.Error.WriteLine("IndexSplitter <srcDir> -d (delete the following segments)");
                //return;
            }
            DirectoryInfo srcDir = new DirectoryInfo(args[0]);
            IndexSplitter @is    = new IndexSplitter(srcDir);

            if (!srcDir.Exists)
            {
                throw new Exception("srcdir:" + srcDir.FullName + " doesn't exist");
            }
            if (args[1].Equals("-l", StringComparison.Ordinal))
            {
                @is.ListSegments();
            }
            else if (args[1].Equals("-d", StringComparison.Ordinal))
            {
                IList <string> segs = new List <string>();
                for (int x = 2; x < args.Length; x++)
                {
                    segs.Add(args[x]);
                }
                @is.Remove(segs);
            }
            else
            {
                DirectoryInfo  targetDir = new DirectoryInfo(args[1]);
                IList <string> segs      = new List <string>();
                for (int x = 2; x < args.Length; x++)
                {
                    segs.Add(args[x]);
                }
                @is.Split(targetDir, segs);
            }
        }
Example #2
0
        public static void Main(string[] args)
        {
            if (args.Length < 2)
            {
                Console.Error.WriteLine("Usage: IndexSplitter <srcDir> -l (list the segments and their sizes)");
                Console.Error.WriteLine("IndexSplitter <srcDir> <destDir> <segments>+");
                Console.Error.WriteLine("IndexSplitter <srcDir> -d (delete the following segments)");
                return;
            }
            DirectoryInfo srcDir = new DirectoryInfo(args[0]);
            IndexSplitter @is    = new IndexSplitter(srcDir);

            if (!srcDir.Exists)
            {
                throw new Exception("srcdir:" + srcDir.FullName + " doesn't exist");
            }
            if (args[1].Equals("-l"))
            {
                @is.ListSegments();
            }
            else if (args[1].Equals("-d"))
            {
                IList <string> segs = new List <string>();
                for (int x = 2; x < args.Length; x++)
                {
                    segs.Add(args[x]);
                }
                @is.Remove(segs.ToArray());
            }
            else
            {
                DirectoryInfo  targetDir = new DirectoryInfo(args[1]);
                IList <string> segs      = new List <string>();
                for (int x = 2; x < args.Length; x++)
                {
                    segs.Add(args[x]);
                }
                @is.Split(targetDir, segs.ToArray());
            }
        }
 public static void Main(string[] args)
 {
     if (args.Length < 2)
     {
         Console.Error.WriteLine("Usage: IndexSplitter <srcDir> -l (list the segments and their sizes)");
         Console.Error.WriteLine("IndexSplitter <srcDir> <destDir> <segments>+");
         Console.Error.WriteLine("IndexSplitter <srcDir> -d (delete the following segments)");
         return;
     }
     DirectoryInfo srcDir = new DirectoryInfo(args[0]);
     IndexSplitter @is = new IndexSplitter(srcDir);
     if (!srcDir.Exists)
     {
         throw new Exception("srcdir:" + srcDir.FullName + " doesn't exist");
     }
     if (args[1].Equals("-l"))
     {
         @is.ListSegments();
     }
     else if (args[1].Equals("-d"))
     {
         IList<string> segs = new List<string>();
         for (int x = 2; x < args.Length; x++)
         {
             segs.Add(args[x]);
         }
         @is.Remove(segs.ToArray());
     }
     else
     {
         DirectoryInfo targetDir = new DirectoryInfo(args[1]);
         IList<string> segs = new List<string>();
         for (int x = 2; x < args.Length; x++)
         {
             segs.Add(args[x]);
         }
         @is.Split(targetDir, segs.ToArray());
     }
 }
Example #4
0
        public void Test()
        {
            DirectoryInfo dir     = CreateTempDir(GetType().Name);
            DirectoryInfo destDir = CreateTempDir(GetType().Name);

            Store.Directory fsDir = NewFSDirectory(dir);
            // IndexSplitter.split makes its own commit directly with SIPC/SegmentInfos,
            // so the unreferenced files are expected.
            if (fsDir is MockDirectoryWrapper)
            {
                ((MockDirectoryWrapper)fsDir).AssertNoUnrefencedFilesOnClose = (false);
            }

            MergePolicy mergePolicy = new LogByteSizeMergePolicy();

            mergePolicy.NoCFSRatio          = 1.0;
            mergePolicy.MaxCFSSegmentSizeMB = double.PositiveInfinity;
            IndexWriter iw = new IndexWriter(
                fsDir,
                new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).
                SetOpenMode(OpenMode.CREATE).
                SetMergePolicy(mergePolicy)
                );

            for (int x = 0; x < 100; x++)
            {
                Document doc = DocHelper.CreateDocument(x, "index", 5);
                iw.AddDocument(doc);
            }
            iw.Commit();
            for (int x = 100; x < 150; x++)
            {
                Document doc = DocHelper.CreateDocument(x, "index2", 5);
                iw.AddDocument(doc);
            }
            iw.Commit();
            for (int x = 150; x < 200; x++)
            {
                Document doc = DocHelper.CreateDocument(x, "index3", 5);
                iw.AddDocument(doc);
            }
            iw.Commit();
            DirectoryReader iwReader = iw.GetReader();

            assertEquals(3, iwReader.Leaves.Count);
            iwReader.Dispose();
            iw.Dispose();
            // we should have 2 segments now
            IndexSplitter @is          = new IndexSplitter(dir);
            string        splitSegName = @is.Infos.Info(1).Info.Name;

            @is.Split(destDir, new string[] { splitSegName });
            Store.Directory fsDirDest = NewFSDirectory(destDir);
            DirectoryReader r         = DirectoryReader.Open(fsDirDest);

            assertEquals(50, r.MaxDoc);
            r.Dispose();
            fsDirDest.Dispose();

            // now test cmdline
            DirectoryInfo destDir2 = CreateTempDir(GetType().Name);

            IndexSplitter.Main(new String[] { dir.FullName, destDir2.FullName, splitSegName });
            assertEquals(5, destDir2.GetFiles().Length);
            Store.Directory fsDirDest2 = NewFSDirectory(destDir2);
            r = DirectoryReader.Open(fsDirDest2);
            assertEquals(50, r.MaxDoc);
            r.Dispose();
            fsDirDest2.Dispose();

            // now remove the copied segment from src
            IndexSplitter.Main(new String[] { dir.FullName, "-d", splitSegName });
            r = DirectoryReader.Open(fsDir);
            assertEquals(2, r.Leaves.size());
            r.Dispose();
            fsDir.Dispose();
        }
        public void Test()
        {
            DirectoryInfo dir = CreateTempDir(GetType().Name);
            DirectoryInfo destDir = CreateTempDir(GetType().Name);
            Store.Directory fsDir = NewFSDirectory(dir);
            // IndexSplitter.split makes its own commit directly with SIPC/SegmentInfos,
            // so the unreferenced files are expected.
            if (fsDir is MockDirectoryWrapper)
            {
                ((MockDirectoryWrapper)fsDir).AssertNoUnrefencedFilesOnClose = (false);
            }

            MergePolicy mergePolicy = new LogByteSizeMergePolicy();
            mergePolicy.NoCFSRatio = 1.0;
            mergePolicy.MaxCFSSegmentSizeMB = double.PositiveInfinity;
            IndexWriter iw = new IndexWriter(
                fsDir,
                new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).
                    SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE).
                    SetMergePolicy(mergePolicy)
            );
            for (int x = 0; x < 100; x++)
            {
                Document doc = DocHelper.CreateDocument(x, "index", 5);
                iw.AddDocument(doc);
            }
            iw.Commit();
            for (int x = 100; x < 150; x++)
            {
                Document doc = DocHelper.CreateDocument(x, "index2", 5);
                iw.AddDocument(doc);
            }
            iw.Commit();
            for (int x = 150; x < 200; x++)
            {
                Document doc = DocHelper.CreateDocument(x, "index3", 5);
                iw.AddDocument(doc);
            }
            iw.Commit();
            DirectoryReader iwReader = iw.Reader;
            assertEquals(3, iwReader.Leaves.Count);
            iwReader.Dispose();
            iw.Dispose();
            // we should have 2 segments now
            IndexSplitter @is = new IndexSplitter(dir);
            string splitSegName = @is.infos.Info(1).Info.Name;
            @is.Split(destDir, new string[] { splitSegName });
            Store.Directory fsDirDest = NewFSDirectory(destDir);
            DirectoryReader r = DirectoryReader.Open(fsDirDest);
            assertEquals(50, r.MaxDoc);
            r.Dispose();
            fsDirDest.Dispose();

            // now test cmdline
            DirectoryInfo destDir2 = CreateTempDir(GetType().Name);
            IndexSplitter.Main(new String[] { dir.FullName, destDir2.FullName, splitSegName });
            assertEquals(5, destDir2.GetFiles().Length);
            Store.Directory fsDirDest2 = NewFSDirectory(destDir2);
            r = DirectoryReader.Open(fsDirDest2);
            assertEquals(50, r.MaxDoc);
            r.Dispose();
            fsDirDest2.Dispose();

            // now remove the copied segment from src
            IndexSplitter.Main(new String[] { dir.FullName, "-d", splitSegName });
            r = DirectoryReader.Open(fsDir);
            assertEquals(2, r.Leaves.size());
            r.Dispose();
            fsDir.Dispose();
        }