public MockSingleIntIndexInput(Directory dir, string fileName, IOContext context)
{
@in = dir.OpenInput(fileName, context);
CodecUtil.CheckHeader(@in, MockSingleIntIndexOutput.CODEC,
MockSingleIntIndexOutput.VERSION_START,
MockSingleIntIndexOutput.VERSION_START);
}
public void TestSetup()
{
UmbracoExamineSearcher.DisableInitializationCheck = true;
BaseUmbracoIndexer.DisableInitializationCheck = true;
//we'll copy over the pdf files first
var svc = new TestDataService();
var path = svc.MapPath("/App_Data/Converting_file_to_PDF.pdf");
var f = new FileInfo(path);
var dir = f.Directory;
//ensure the folder is there
System.IO.Directory.CreateDirectory(dir.FullName);
var pdfs = new[] { TestFiles.Converting_file_to_PDF, TestFiles.PDFStandards, TestFiles.SurviorFlipCup, TestFiles.windows_vista };
var names = new[] { "Converting_file_to_PDF.pdf", "PDFStandards.pdf", "SurviorFlipCup.pdf", "windows_vista.pdf" };
for (int index = 0; index < pdfs.Length; index++)
{
var p = pdfs[index];
using (var writer = File.Create(Path.Combine(dir.FullName, names[index])))
{
writer.Write(p, 0, p.Length);
}
}
_luceneDir = new RAMDirectory();
_indexer = IndexInitializer.GetPdfIndexer(_luceneDir);
_indexer.RebuildIndex();
_searcher = IndexInitializer.GetUmbracoSearcher(_luceneDir);
}
private void CheckSplitting(Directory dir, Term splitTerm, int leftCount, int rightCount)
{
using (Directory dir1 = NewDirectory())
{
using (Directory dir2 = NewDirectory())
{
PKIndexSplitter splitter = new PKIndexSplitter(dir, dir1, dir2, splitTerm,
NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())),
NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
splitter.Split();
using (IndexReader ir1 = DirectoryReader.Open(dir1))
{
using (IndexReader ir2 = DirectoryReader.Open(dir2))
{
assertEquals(leftCount, ir1.NumDocs);
assertEquals(rightCount, ir2.NumDocs);
CheckContents(ir1, "1");
CheckContents(ir2, "2");
}
}
}
}
}
public int CloseStaleReaders(Directory dir, TimeSpan ts)
{
lock (_locker)
{
var now = DateTime.Now;
var readersForDir = _oldReaders.Where(x => x.Item1.Directory().GetLockID() == dir.GetLockID()).ToList();
var newest = readersForDir.OrderByDescending(x => x.Item2).FirstOrDefault();
readersForDir.Remove(newest);
var stale = readersForDir.Where(x => now - x.Item2 >= ts).ToArray();
foreach (var reader in stale)
{
//close reader and remove from list
try
{
reader.Item1.Close();
}
catch (AlreadyClosedException)
{
//if this happens, more than one instance has decreased referenced, this could occur if this
//somehow gets called in conjuction with the shutdown code or manually, etc...
}
_oldReaders.Remove(reader);
}
return stale.Length;
}
}
public override void SetUp()
{
base.SetUp();
dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, true), Similarity, TimeZone);
for (int i = 900; i < 1112; i++)
{
Document doc = new Document();
string num = Regex.Replace(Regex.Replace(English.IntToEnglish(i), "[-]", " "), "[,]", "");
doc.Add(NewTextField("numbers", num, Field.Store.NO));
writer.AddDocument(doc);
}
{
Document doc = new Document();
doc.Add(NewTextField("numbers", "thou hast sand betwixt thy toes", Field.Store.NO));
writer.AddDocument(doc);
}
{
Document doc = new Document();
doc.Add(NewTextField("numbers", "hundredeight eightyeight yeight", Field.Store.NO));
writer.AddDocument(doc);
}
{
Document doc = new Document();
doc.Add(NewTextField("numbers", "tres y cinco", Field.Store.NO));
writer.AddDocument(doc);
}
writer.Commit();
writer.Dispose();
}
public override void SetUp()
{
base.SetUp();
dir = NewDirectory();
var iw = new RandomIndexWriter(Random(), dir, Similarity, TimeZone);
int numDocs = TestUtil.NextInt(Random(), 2049, 4000);
for (int i = 0; i < numDocs; i++)
{
var document = new Document
{
NewTextField("english", English.IntToEnglish(i), Field.Store.NO),
NewTextField("oddeven", (i%2 == 0) ? "even" : "odd", Field.Store.NO
),
NewStringField("byte", string.Empty + (unchecked((byte) Random().Next
())), Field.Store.NO),
NewStringField("short", string.Empty + ((short) Random().Next()), Field.Store
.NO),
new IntField("int", Random().Next(), Field.Store.NO),
new LongField("long", Random().NextLong(), Field.Store.NO),
new FloatField("float", Random().NextFloat(), Field.Store.NO),
new DoubleField("double", Random().NextDouble(), Field.Store.NO),
new NumericDocValuesField("intdocvalues", Random().Next()),
new FloatDocValuesField("floatdocvalues", Random().NextFloat())
};
iw.AddDocument(document);
}
reader = iw.Reader;
iw.Dispose();
searcher = NewSearcher(reader);
}
public override void SetUp()
{
base.SetUp();
_dir = NewDirectory();
_indexWriter = new RandomIndexWriter(Random(), _dir, new MockAnalyzer(Random()), Similarity, TimeZone);
FieldType ft = new FieldType(TextField.TYPE_STORED);
ft.StoreTermVectors = true;
ft.StoreTermVectorOffsets = true;
ft.StoreTermVectorPositions = true;
Analyzer analyzer = new MockAnalyzer(Random());
Document doc;
for (int i = 0; i < 100; i++)
{
doc = new Document();
doc.Add(new Field(_idFieldName, Random().toString(), ft));
doc.Add(new Field(_textFieldName, new StringBuilder(Random().toString()).append(Random().toString()).append(
Random().toString()).toString(), ft));
doc.Add(new Field(_classFieldName, Random().toString(), ft));
_indexWriter.AddDocument(doc, analyzer);
}
_indexWriter.Commit();
_originalIndex = SlowCompositeReaderWrapper.Wrap(_indexWriter.Reader);
}
public MapReduceIndex(Directory directory, int id, IndexDefinition indexDefinition,
AbstractViewGenerator viewGenerator, WorkContext context)
: base(directory, id, indexDefinition, viewGenerator, context)
{
jsonSerializer = JsonExtensions.CreateDefaultJsonSerializer();
jsonSerializer.Converters = MapReduceConverters;
}
public NAppIndexUpdater(Configuration.ConfigManager config, Database currentDb)
{
indexFullPath = config.GetSetting(SettingKeys.Index_Directory, System.IO.Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Index"));
indexFullPath = System.IO.Path.GetFullPath(indexFullPath);
directory = FSDirectory.Open(new System.IO.DirectoryInfo(indexFullPath));
this.currentDb = currentDb;
}
public override void SetUp()
{
base.SetUp();
dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer
(Random()));
iwc.SetMergePolicy(NewLogMergePolicy());
var iw = new RandomIndexWriter(Random(), dir, iwc);
var doc = new Document
{
NewStringField("id", "1", Field.Store.YES),
NewTextField("body", "some contents and more contents", Field.Store.NO),
new NumericDocValuesField("popularity", 5)
};
iw.AddDocument(doc);
doc = new Document
{
NewStringField("id", "2", Field.Store.YES),
NewTextField("body", "another document with different contents", Field.Store
.NO),
new NumericDocValuesField("popularity", 20)
};
iw.AddDocument(doc);
doc = new Document
{
NewStringField("id", "3", Field.Store.YES),
NewTextField("body", "crappy contents", Field.Store.NO),
new NumericDocValuesField("popularity", 2)
};
iw.AddDocument(doc);
iw.ForceMerge(1);
reader = iw.Reader;
iw.Dispose();
}
public static void Finish()
{
s.IndexReader.Dispose();
s = null;
Dir.Dispose();
Dir = null;
}
public MultiIndexLockFactory(Lucene.Net.Store.Directory master, Lucene.Net.Store.Directory child)
{
if (master == null) throw new ArgumentNullException("master");
if (child == null) throw new ArgumentNullException("child");
_master = master;
_child = child;
}
public override void SetUp()
{
base.SetUp();
store = NewDirectory();
IndexWriter writer = new IndexWriter(store, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, false)));
Document doc;
doc = new Document();
doc.Add(NewTextField("aaa", "foo", Field.Store.YES));
writer.AddDocument(doc);
doc = new Document();
doc.Add(NewTextField("aaa", "foo", Field.Store.YES));
writer.AddDocument(doc);
doc = new Document();
doc.Add(NewTextField("contents", "Tom", Field.Store.YES));
writer.AddDocument(doc);
doc = new Document();
doc.Add(NewTextField("contents", "Jerry", Field.Store.YES));
writer.AddDocument(doc);
doc = new Document();
doc.Add(NewTextField("zzz", "bar", Field.Store.YES));
writer.AddDocument(doc);
writer.ForceMerge(1);
writer.Dispose();
}
public AuditLog(int DaysToKeepLog, int DaysToKeepRecords)
{
string dir = GetDataDirectory();
_logDays = DaysToKeepLog;
_recordDays = DaysToKeepRecords;
Task.Run(() =>
{
while (IsFileLocked(new FileInfo($"{dir}/write.lock")))
{
Thread.Sleep(500);
}
if (File.Exists($"{dir}/write.lock"))
File.Delete($"{dir}/write.lock");
_dir = FSDirectory.Open(new DirectoryInfo(GetDataDirectory()));
var reader = GetSearcher();
var terms = reader.IndexReader.Terms();
while (terms.Next())
{
if (terms.Term.Field == "content")
trie[terms.Term.Text] = terms.Term.Text;
if (terms.Term.Field == "user")
users.Add(terms.Term.Text);
}
ValidateBackup();
});
}
public void TestSetup()
{
_luceneDir = new RAMDirectory();
_indexer = IndexInitializer.GetUmbracoIndexer(_luceneDir);
_indexer.RebuildIndex();
_searcher = IndexInitializer.GetUmbracoSearcher(_luceneDir);
}
public void Dispose()
{
if (!_disposed)
{
if (_index != null)
{
_index.Dispose();
_index = null;
}
if (_analyzer != null)
{
_analyzer.Dispose();
_analyzer = null;
}
if (_directory != null)
{
_directory.Dispose();
_directory = null;
}
_disposed = true;
}
}
public MockRAMDirectory(Directory dir) : base(dir)
{
if (openFiles == null)
{
openFiles = new System.Collections.Hashtable();
}
}
public void SearchFiltered(IndexWriter writer, Directory directory, Filter filter, bool optimize)
{
try
{
for (int i = 0; i < 60; i++)
{//Simple docs
Document doc = new Document();
doc.Add(new Field(FIELD, i.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
writer.AddDocument(doc);
}
if (optimize)
writer.Optimize();
writer.Close();
BooleanQuery booleanQuery = new BooleanQuery();
booleanQuery.Add(new TermQuery(new Term(FIELD, "36")), Occur.SHOULD);
IndexSearcher indexSearcher = new IndexSearcher(directory);
ScoreDoc[] hits = indexSearcher.Search(booleanQuery, filter, 1000).ScoreDocs;
Assert.AreEqual(1, hits.Length, "Number of matched documents");
}
catch (System.IO.IOException e)
{
Assert.Fail(e.Message);
}
}
private void CreateRandomIndexes(int maxSegments)
{
dir = NewDirectory();
numDocs = AtLeast(150);
int numTerms = TestUtil.NextInt(Random(), 1, numDocs / 5);
ISet<string> randomTerms = new HashSet<string>();
while (randomTerms.size() < numTerms)
{
randomTerms.add(TestUtil.RandomSimpleString(Random()));
}
terms = new List<string>(randomTerms);
int seed = Random().Next();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(new Random(seed)));
iwc.SetMergePolicy(TestSortingMergePolicy.NewSortingMergePolicy(sort));
iw = new RandomIndexWriter(new Random(seed), dir, iwc);
for (int i = 0; i < numDocs; ++i)
{
Document doc = RandomDocument();
iw.AddDocument(doc);
if (i == numDocs / 2 || (i != numDocs - 1 && Random().nextInt(8) == 0))
{
iw.Commit();
}
if (Random().nextInt(15) == 0)
{
string term = RandomInts.RandomFrom(Random(), terms);
iw.DeleteDocuments(new Term("s", term));
}
}
reader = iw.Reader;
}
private RAMDirectory(Directory dir, bool closeDir)
{
System.String[] files = dir.List();
byte[] buf = new byte[BufferedIndexOutput.BUFFER_SIZE];
for (int i = 0; i < files.Length; i++)
{
// make place on ram disk
IndexOutput os = CreateOutput(System.IO.Path.GetFileName(files[i]));
// read current file
IndexInput is_Renamed = dir.OpenInput(files[i]);
// and copy to ram disk
long len = (int) is_Renamed.Length();
long readCount = 0;
while (readCount < len)
{
int toRead = readCount + BufferedIndexOutput.BUFFER_SIZE > len ? (int) (len - readCount) : BufferedIndexOutput.BUFFER_SIZE;
is_Renamed.ReadBytes(buf, 0, toRead);
os.WriteBytes(buf, toRead);
readCount += toRead;
}
// graceful cleanup
is_Renamed.Close();
os.Close();
}
if (closeDir)
dir.Close();
}
public SuggestionQueryIndexExtension(
WorkContext workContext,
string key,
StringDistance distanceType,
bool isRunInMemory,
string field,
float accuracy)
{
this.workContext = workContext;
this.key = key;
this.field = field;
if (isRunInMemory)
{
directory = new RAMDirectory();
}
else
{
directory = FSDirectory.Open(new DirectoryInfo(key));
}
this.spellChecker = new SpellChecker.Net.Search.Spell.SpellChecker(directory, null);
this.spellChecker.SetAccuracy(accuracy);
this.spellChecker.setStringDistance(distanceType);
}
public override void SetUp()
{
base.SetUp();
DirectoryInfo file = CreateTempDir("testIndex");
// use a simple FSDir here, to be sure to have SimpleFSInputs
Dir = new SimpleFSDirectory(file, null);
}
public override void SetUp()
{
base.SetUp();
dir = NewDirectory();
var iw = new RandomIndexWriter(Random(), dir, Similarity, TimeZone);
var doc = new Document
{
NewStringField("id", "1", Field.Store.YES),
NewTextField("body", "some contents and more contents", Field.Store.NO),
new NumericDocValuesField("popularity", 5)
};
iw.AddDocument(doc);
doc = new Document
{
NewStringField("id", "2", Field.Store.YES),
NewTextField("body", "another document with different contents", Field.Store
.NO),
new NumericDocValuesField("popularity", 20)
};
iw.AddDocument(doc);
doc = new Document
{
NewStringField("id", "3", Field.Store.YES),
NewTextField("body", "crappy contents", Field.Store.NO),
new NumericDocValuesField("popularity", 2)
};
iw.AddDocument(doc);
reader = iw.Reader;
searcher = new IndexSearcher(reader);
iw.Dispose();
}
public void Initialize()
{
_cwsDir = new RAMDirectory();
_pdfDir = new RAMDirectory();
_simpleDir = new RAMDirectory();
_conventionDir = new RAMDirectory();
//get all of the indexers and rebuild them all first
var indexers = new IIndexer[]
{
IndexInitializer.GetUmbracoIndexer(_cwsDir),
IndexInitializer.GetSimpleIndexer(_simpleDir),
IndexInitializer.GetUmbracoIndexer(_conventionDir)
};
foreach (var i in indexers)
{
try
{
i.RebuildIndex();
}
finally
{
var d = i as IDisposable;
if (d != null) d.Dispose();
}
}
//now get the multi index searcher for all indexes
_searcher = IndexInitializer.GetMultiSearcher(_pdfDir, _simpleDir, _conventionDir, _cwsDir);
}
public static void Main(string[] args)
{
if (args.Length < 3)
{
Console.Error.WriteLine("Usage: IndexMergeTool <mergedIndex> <index1> <index2> [index3] ...");
Environment.Exit(1);
}
FSDirectory mergedIndex = FSDirectory.Open(new System.IO.DirectoryInfo(args[0]));
#pragma warning disable 612, 618
using (IndexWriter writer = new IndexWriter(mergedIndex, new IndexWriterConfig(LuceneVersion.LUCENE_CURRENT, null)
.SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE)))
#pragma warning restore 612, 618
{
Directory[] indexes = new Directory[args.Length - 1];
for (int i = 1; i < args.Length; i++)
{
indexes[i - 1] = FSDirectory.Open(new System.IO.DirectoryInfo(args[i]));
}
Console.WriteLine("Merging...");
writer.AddIndexes(indexes);
Console.WriteLine("Full merge...");
writer.ForceMerge(1);
}
Console.WriteLine("Done.");
}
public SearchAutoComplete(Directory autoCompleteDir, int maxResults = 8)
{
this.m_directory = autoCompleteDir;
MaxResults = maxResults;
ReplaceSearcher();
}
public override void TearDown()
{
base.TearDown();
dir.Dispose();
dir = null;
anlzr = null;
}
/// <summary>
/// ctor for unit tests
/// </summary>
internal LocalPackageIndex(LuceneDirectory directory, IPackageSearchEngine engine, IReflectorFactory reflectorFactory, ILog logger)
{
_directory = directory;
_engine = engine;
_reflectorFactory = reflectorFactory;
Logger = logger;
}
public FileSwitchDirectory(System.Collections.Hashtable primaryExtensions, Directory primaryDir, Directory secondaryDir, bool doClose)
{
this.primaryExtensions = primaryExtensions;
this.primaryDir = primaryDir;
this.secondaryDir = secondaryDir;
this.doClose = doClose;
this.lockFactory = primaryDir.GetLockFactory();
}
public LuceneIndexingService(Func <EntitiesContext> contextThunk, bool indexContainsAllVersions)
{
if (contextThunk == null)
{
throw new ArgumentNullException("contextThunk");
}
_contextThunk = contextThunk;
_indexContainsAllVersions = indexContainsAllVersions;
_directory = new LuceneFileSystem(LuceneCommon.IndexDirectory);
_getShouldAutoUpdate = () => true;
}
public override void TestSetup()
{
_luceneDir = new RAMDirectory();
_indexer = IndexInitializer.GetSimpleIndexer(_luceneDir);
_reIndexDateTime = DateTime.Now;
Thread.Sleep(1000);
_indexer.RebuildIndex();
_searcher = IndexInitializer.GetLuceneSearcher(_luceneDir);
}
public void TestConsistencyOnExceptions()
{
// so the handler's index isn't empty
replicator.Publish(CreateRevision(1));
client.UpdateNow();
client.Dispose();
callback.Dispose();
// Replicator violates write-once policy. It may be that the
// handler copies files to the index dir, then fails to copy a
// file and reverts the copy operation. On the next attempt, it
// will copy the same file again. There is nothing wrong with this
// in a real system, but it does violate write-once, and MDW
// doesn't like it. Disabling it means that we won't catch cases
// where the handler overwrites an existing index file, but
// there's nothing currently we can do about it, unless we don't
// use MDW.
handlerIndexDir.PreventDoubleWrite = (false);
handlerTaxoDir.PreventDoubleWrite = (false);
// wrap sourceDirFactory to return a MockDirWrapper so we can simulate errors
ISourceDirectoryFactory @in = sourceDirFactory;
AtomicInt32 failures = new AtomicInt32(AtLeast(10));
sourceDirFactory = new SourceDirectoryFactoryAnonymousClass(this, @in, failures);
handler = new IndexAndTaxonomyReplicationHandler(handlerIndexDir, handlerTaxoDir, () =>
{
if (Random.NextDouble() < 0.2 && failures > 0)
{
throw RuntimeException.Create("random exception from callback");
}
});
client = new ReplicationClientAnonymousClass(this, replicator, handler, @in, failures);
client.StartUpdateThread(10, "indexAndTaxo");
Directory baseHandlerIndexDir = handlerIndexDir.Delegate;
int numRevisions = AtLeast(20) + 2;
for (int i = 2; i < numRevisions; i++)
{
replicator.Publish(CreateRevision(i));
AssertHandlerRevision(i, baseHandlerIndexDir);
}
// disable errors -- maybe randomness didn't exhaust all allowed failures,
// and we don't want e.g. CheckIndex to hit false errors.
handlerIndexDir.MaxSizeInBytes = (0);
handlerIndexDir.RandomIOExceptionRate = (0.0);
handlerIndexDir.RandomIOExceptionRateOnOpen = (0.0);
handlerTaxoDir.MaxSizeInBytes = (0);
handlerTaxoDir.RandomIOExceptionRate = (0.0);
handlerTaxoDir.RandomIOExceptionRateOnOpen = (0.0);
}
public int getLemmaIDbyLemmaName(string lemmaName)
{
string directory = System.IO.Directory.GetCurrentDirectory();
string[] splitDir = directory.Split('\\');
if (splitDir[splitDir.Length - 1] == "Debug")
{
System.IO.Directory.SetCurrentDirectory("..\\..\\");
}
string[] splitLemmaName = lemmaName.Split('(');
string results = null;
int id = -1;
string indexDir = "Index";
using (Lucene.Net.Store.Directory dir = FSDirectory.Open(indexDir))
using (IndexSearcher searcher = new IndexSearcher(dir))
{
string[] splited;
Term term;
WildcardQuery q = null;
BooleanQuery bq = new BooleanQuery();
if (splitLemmaName[0].Split(' ').Length > 1)
{
splited = lemmaName.Split(' ');
for (int i = 0; i < splited.Length; i++)
{
term = new Term("title", "*" + splited[i].ToLower() + "*");
q = new WildcardQuery(term);
bq.Add(q, Occur.SHOULD);
}
}
else
{
term = new Term("title", "*" + splitLemmaName[0].ToLower() + "*");
q = new WildcardQuery(term);
bq.Add(q, Occur.SHOULD);
}
TopDocs hits = searcher.Search(bq, 1);
foreach (ScoreDoc d in hits.ScoreDocs)
{
Lucene.Net.Documents.Document doc = searcher.Doc(d.Doc);
results = doc.Get("LID").ToString();
}
}
Int32.TryParse(results, out id);
return(id);
}
/// <summary>
/// Copies the provided list of files from the <paramref name="source"/> <see cref="Directory"/> to the
/// <paramref name="target"/> <see cref="Directory"/>, if they are not the same.
/// </summary>
/// <exception cref="IOException"></exception>
public static void CopyFiles(Directory source, Directory target, IList <string> files)
{
if (source.Equals(target))
{
return;
}
foreach (string file in files)
{
source.Copy(target, file, file, IOContext.READ_ONCE);
}
}
public static UmbracoContentIndexer GetUmbracoIndexer(Lucene.Net.Store.Directory luceneDir)
{
var i = new UmbracoContentIndexer(CreateCriteria(),
luceneDir, //custom lucene directory
new TestDataService(),
new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29),
false);
i.IndexingError += IndexingError;
return(i);
}
public void TestOpen()
{
Directory indexDir = NewDirectory();
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.IndexDeletionPolicy = new SnapshotDeletionPolicy(conf.IndexDeletionPolicy);
IndexWriter indexWriter = new IndexWriter(indexDir, conf);
Directory taxoDir = NewDirectory();
IndexAndTaxonomyRevision.SnapshotDirectoryTaxonomyWriter taxoWriter = new IndexAndTaxonomyRevision.SnapshotDirectoryTaxonomyWriter(taxoDir);
try
{
indexWriter.AddDocument(NewDocument(taxoWriter));
indexWriter.Commit();
taxoWriter.Commit();
IRevision rev = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
foreach (var e in rev.SourceFiles)
{
string source = e.Key;
Directory dir = source.Equals(IndexAndTaxonomyRevision.INDEX_SOURCE, StringComparison.Ordinal) ? indexDir : taxoDir;
foreach (RevisionFile file in e.Value)
{
using (IndexInput src = dir.OpenInput(file.FileName, IOContext.READ_ONCE))
using (System.IO.Stream @in = rev.Open(source, file.FileName))
{
assertEquals(src.Length, @in.Length);
byte[] srcBytes = new byte[(int)src.Length];
byte[] inBytes = new byte[(int)src.Length];
int offset = 0;
if (Random.nextBoolean())
{
int skip = Random.Next(10);
if (skip >= src.Length)
{
skip = 0;
}
@in.Seek(skip, SeekOrigin.Current);
src.Seek(skip);
offset = skip;
}
src.ReadBytes(srcBytes, offset, srcBytes.Length - offset);
@in.Read(inBytes, offset, inBytes.Length - offset);
assertArrayEquals(srcBytes, inBytes);
}
}
}
}
finally
{
IOUtils.Dispose(indexWriter, taxoWriter, taxoDir, indexDir);
}
}
public TermVectorsWriter(Directory directory, System.String segment, FieldInfos fieldInfos, IState state)
{
// Open files for TermVector storage
tvx = directory.CreateOutput(segment + "." + IndexFileNames.VECTORS_INDEX_EXTENSION, state);
tvx.WriteInt(TermVectorsReader.FORMAT_CURRENT);
tvd = directory.CreateOutput(segment + "." + IndexFileNames.VECTORS_DOCUMENTS_EXTENSION, state);
tvd.WriteInt(TermVectorsReader.FORMAT_CURRENT);
tvf = directory.CreateOutput(segment + "." + IndexFileNames.VECTORS_FIELDS_EXTENSION, state);
tvf.WriteInt(TermVectorsReader.FORMAT_CURRENT);
this.fieldInfos = fieldInfos;
}
/// <summary>
/// Creates the index at a given path
/// </summary>
/// <param name="indexPath">The pathname to create the index</param>
public void CreateIndex(string indexPath)
{
luceneIndexDirectory = Lucene.Net.Store.FSDirectory.Open(indexPath);
IndexWriter.MaxFieldLength mfl = new IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH);
writer = new Lucene.Net.Index.IndexWriter(luceneIndexDirectory, analyzer, true, mfl);
writer.SetSimilarity(new NewSimilarity());
//writer.SetSimilarity(newSimilarity); // for similarity measure
}
/// <summary>Initializes a <see cref="NuGetSearcherManager"/> instance.</summary>
/// <param name="directory">
/// Optionally, the Lucene directory to read the index from. If <c>null</c> is provided, the directory
/// implementation is determined based off of the configuration (<see cref="config"/>).
/// </param>
/// <param name="loader">
/// Optionally, the loader used to read the JSON data files. If <c>null</c> is provided, the loader
/// implementation is determined based off of the configuration (<see cref="config"/>).
/// </param>
/// <param name="config">
/// The configuration to read which primarily determines whether the resulting instance will read from the local
/// disk or from blob storage.
/// </param>
/// <param name="loggerFactory">
/// Optionally, the logger factory defined by the consuming application.
/// </param>
/// <returns>The resulting <see cref="NuGetSearcherManager"/> instance.</returns>
public static NuGetSearcherManager Create(IndexingConfiguration config,
ILoggerFactory loggerFactory,
Directory directory = null,
ILoader loader = null)
{
if (loggerFactory == null)
{
throw new ArgumentNullException(nameof(loggerFactory));
}
LoggerFactory = loggerFactory;
var logger = loggerFactory.CreateLogger <NuGetSearcherManager>();
// If a local Lucene directory has been specified, create a directory and loader for the specified directory.
var luceneDirectory = config.LocalLuceneDirectory;
if (!string.IsNullOrEmpty(luceneDirectory))
{
directory = directory ?? new SimpleFSDirectory(new DirectoryInfo(luceneDirectory));
loader = loader ?? new FileLoader(config.LocalDataDirectory);
}
IIndexDirectoryProvider indexProvider;
if (directory == null)
{
// If no directory has been provided, create a CloudIndexDirectoryProvider from the configuration.
indexProvider = new CloudIndexDirectoryProvider(config, logger);
}
else
{
// Use the specified directory to create a FixedIndexDirectoryProvider.
var indexContainerName = luceneDirectory ?? config.IndexContainer;
indexProvider = new LocalIndexDirectoryProvider(directory, indexContainerName);
}
// If a loader has been specified, use it.
// Otherwise, create a StorageLoader from the configuration.
loader = loader ?? new StorageLoader(config, logger);
var searcherManager = new NuGetSearcherManager(logger, indexProvider, loader, config.AuxiliaryDataRefreshRateSec, config.IndexReloadRateSec);
var registrationBaseAddress = config.RegistrationBaseAddress;
var semVer2RegistrationBaseAddress = config.SemVer2RegistrationBaseAddress;
searcherManager.RegistrationBaseAddresses.LegacyHttp = MakeRegistrationBaseAddress("http", registrationBaseAddress);
searcherManager.RegistrationBaseAddresses.LegacyHttps = MakeRegistrationBaseAddress("https", registrationBaseAddress);
searcherManager.RegistrationBaseAddresses.SemVer2Http = MakeRegistrationBaseAddress("http", semVer2RegistrationBaseAddress);
searcherManager.RegistrationBaseAddresses.SemVer2Https = MakeRegistrationBaseAddress("https", semVer2RegistrationBaseAddress);
return(searcherManager);
}
public void Ini()
{
if (UseAzureDirectory)
{
var azureBlobConnectionString = config.GetValue <string>("AzureDirectoryConnectionString");
var azureDirectoryCachePath = ReplacePathTokens(config.GetValue <string>("AzureDirectoryCachePath"));
var azureDirectoryContainerName = config.GetValue <string>("AzureDirectoryContainerName");
//if empty string, ensure value is null
if (string.IsNullOrEmpty(azureDirectoryContainerName))
{
azureDirectoryContainerName = null;
}
var cloudStorageAccount = CloudStorageAccount.Parse(azureBlobConnectionString);
Directory = new AzureDirectory(cloudStorageAccount, azureDirectoryCachePath, containerName: azureDirectoryContainerName);
}
else
{
if (!System.IO.Directory.Exists(INDEXPATH))
{
System.IO.Directory.CreateDirectory(INDEXPATH);
}
Directory = FSDirectory.Open(INDEXPATH);
}
bool create = !DirectoryReader.IndexExists(Directory);
lock (write_lock)
{
try
{
SetWriter(create);
if (Writer != null && !UseAzureDirectory)
{
CloseWriter();
}
//Writer.Optimize();
}
catch (Lucene.Net.Store.LockObtainFailedException ex)
{
logger.Log(ex);
}
catch (Exception ex)
{
throw;
//logger.Log(ex);
}
finally
{
//CloseWriter();
}
}
SetSearcher();
}
private void OpenIndexOnStartup()
{
analyzer = new LowerCaseKeywordAnalyzer();
snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
bool resetTried = false;
bool recoveryTried = false;
while (true)
{
LuceneDirectory luceneDirectory = null;
try
{
luceneDirectory = OpenOrCreateLuceneDirectory(indexDirectory);
// Skip sanity test if we are running in memory. Index will not exist anyways.
if (!configuration.RunInMemory && !IsIndexStateValid(luceneDirectory))
{
throw new InvalidOperationException("Sanity check on the index failed.");
}
directory = luceneDirectory;
writer = new IndexWriter(directory, analyzer, snapshotter, IndexWriter.MaxFieldLength.UNLIMITED);
writer.SetMergeScheduler(new ErrorLoggingConcurrentMergeScheduler());
currentIndexSearcherHolder.SetIndexSearcher(new IndexSearcher(directory, true));
break;
}
catch (Exception e)
{
if (resetTried)
{
throw new InvalidOperationException("Could not open / create index for file system '" + name + "', reset already tried", e);
}
if (recoveryTried == false && luceneDirectory != null)
{
recoveryTried = true;
StartupLog.WarnException("Could not open index for file system '" + name + "'. Trying to recover index", e);
StartupLog.Info("Recover functionality is still not implemented. Skipping.");
}
else
{
resetTried = true;
StartupLog.WarnException("Could not open index for file system '" + name + "'. Recovery operation failed, forcibly resetting index", e);
TryResettingIndex();
}
}
}
}
private void button2_Click(object sender, EventArgs e)
{
if (string.IsNullOrEmpty(this.txtSearch.Text))
{
MessageBox.Show("请输入搜索的文本");
}
StringBuilder sb = new StringBuilder();
Stopwatch sw = new Stopwatch();
sw.Start();
//索引库目录
Lucene.Net.Store.Directory dir = FSDirectory.Open(new System.IO.DirectoryInfo("IndexDir"), new NoLockFactory());
IndexReader reader = IndexReader.Open(dir, true);
IndexSearcher search = null;
try
{
search = new IndexSearcher(reader);
QueryParser parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, "body", new PanGuAnalyzer());
Query query = parser.Parse(LuceneHelper.GetKeyWordSplid(this.txtSearch.Text));
//执行搜索，获取查询结果集对象
TopDocs ts = search.Search(query, null, 1000);
///获取命中的文档信息对象
ScoreDoc[] docs = ts.ScoreDocs;
sw.Stop();
this.listBox1.Items.Clear();
for (int i = 0; i < docs.Length; i++)
{
int docId = docs[i].Doc;
Document doc = search.Doc(docId);
this.listBox1.Items.Add(doc.Get("number") + "\r\n");
this.listBox1.Items.Add(doc.Get("body") + "\r\n");
this.listBox1.Items.Add("------------------------\r\n");
}
}
catch (Exception ex)
{
throw;
}
finally
{
if (search != null)
{
search.Dispose();
}
if (dir != null)
{
dir.Dispose();
}
}
this.label1.Text = "搜索用时:" + sw.ElapsedMilliseconds + "毫秒";
}
/// <summary>
/// Remove facet documents from the existing index.
/// </summary>
/// <param name="directoryIndexInfo">The directory infomation where the index files are located.</param>
/// <param name="directoryFacetInfo">The directory infomation where the facet files are to be placed.</param>
/// <param name="textNames">The array of names for text data.</param>
/// <param name="filePaths">The array of full paths (without root 'C:\'. e.g. 'temp/http/index.html') for file documents.</param>
public void RemoveMultiFacetDocuments(DirectoryInfo directoryIndexInfo, DirectoryInfo directoryFacetInfo, string[] textNames, string[] filePaths)
{
Lucene.Net.Index.IndexWriter writer = null;
DirectoryTaxonomyWriter facetWriter = null;
Lucene.Net.Store.Directory directory = null;
Lucene.Net.Store.Directory directoryFacet = null;
try
{
// Create the analyzer.
SimpleAnalyzer simpleAnalyzer = new Analyzer.SimpleAnalyzer();
StandardAnalyzer standardAnalyzer = new Analyzer.StandardAnalyzer(simpleAnalyzer);
// Create the index writer.
directory = FSDirectory.Open(directoryIndexInfo);
IndexWriterConfig indexConfig = new IndexWriterConfig(Lucene.Net.Util.LuceneVersion.LUCENE_48, standardAnalyzer);
indexConfig.SetOpenMode(IndexWriterConfig.OpenMode_e.APPEND);
// Open existing or create new.
writer = new IndexWriter(directory, indexConfig);
// Create the facet writer.
directoryFacet = FSDirectory.Open(directoryFacetInfo);
facetWriter = new DirectoryTaxonomyWriter(directoryFacet, IndexWriterConfig.OpenMode_e.APPEND);
// Create the delet query.
FacetFilter filter = new FacetFilter();
Query[] queries = filter.RemoveDocuments(textNames, filePaths);
writer.DeleteDocuments(queries);
// Commit the index.
writer.Commit();
facetWriter.Commit();
}
catch (Exception)
{
throw;
}
finally
{
if (writer != null)
{
writer.Dispose();
}
if (directory != null)
{
directory.Dispose();
}
}
}
/// <summary>. </summary>
public void SyncFile(Lucene.Net.Store.Directory directory, string fileName, bool CompressBlobs)
{
Trace.WriteLine($"INFO Syncing file {fileName} for {_rootFolderName}");
// then we will get it fresh into local deflatedName
// StreamOutput deflatedStream = new StreamOutput(CacheDirectory.CreateOutput(deflatedName));
// seek back to begininng
if (ShouldCompressFile(fileName, CompressBlobs))
{
using (var deflatedStream = new MemoryStream())
{
#if FULLDEBUG
Trace.WriteLine($"GET {fileName} RETREIVED {deflatedStream.Length} bytes");
#endif
// get the deflated blob
blob.DownloadTo(deflatedStream);
deflatedStream.Seek(0, SeekOrigin.Begin);
// open output file for uncompressed contents
using (var fileStream = new StreamOutput(directory.CreateOutput(fileName)))
using (var decompressor = new DeflateStream(deflatedStream, CompressionMode.Decompress))
{
var bytes = new byte[65535];
var nRead = 0;
do
{
nRead = decompressor.Read(bytes, 0, 65535);
if (nRead > 0)
{
fileStream.Write(bytes, 0, nRead);
}
} while (nRead == 65535);
}
}
}
else
{
using (var fileStream = new StreamOutput(directory.CreateOutput(fileName)))
{
// get the blob
blob.DownloadTo(fileStream);
fileStream.Flush();
#if FULLDEBUG
Trace.WriteLine($"GET {fileName} RETREIVED {fileStream.Length} bytes");
#endif
}
}
}
public void Write(Directory d, System.String name, IState state)
{
IndexOutput output = d.CreateOutput(name, state);
try
{
Write(output);
}
finally
{
output.Close();
}
}
protected virtual IndexWriter CreateWriter(Directory d, Analyzer a)
{
try
{
return(CreateWriterNoTry(d, a));
}
catch (Lucene.Net.Store.LockObtainFailedException)
{
logger.Debug("Failed to obtain lock, deleting it and retrying.");
ClearLock();
return(CreateWriterNoTry(d, a));
}
}
public virtual Directory GetDirectory()
{
lock (this)
{
if (!System.IO.Directory.Exists(indexPath))
{
System.IO.Directory.CreateDirectory(indexPath);
}
var d = directory ?? (directory = new SimpleFSDirectory(new DirectoryInfo(indexPath), new SimpleFSLockFactory()));
return(d);
}
}
private void Initialize(Directory directory, System.String segment, FieldInfos fis, int interval, bool isi, IState state)
{
indexInterval = interval;
fieldInfos = fis;
isIndex = isi;
output = directory.CreateOutput(segment + (isIndex?".tii":".tis"), state);
output.WriteInt(FORMAT_CURRENT); // write format
output.WriteLong(0); // leave space for size
output.WriteInt(indexInterval); // write indexInterval
output.WriteInt(skipInterval); // write skipInterval
output.WriteInt(maxSkipLevels); // write maxSkipLevels
System.Diagnostics.Debug.Assert(InitUTF16Results());
}
public LocalPackageIndex(ILog logger = null)
{
Logger = logger;
_location = Environment.ExpandEnvironmentVariables(DefaultIndexPath);
if (!System.IO.Directory.Exists(_location))
{
System.IO.Directory.CreateDirectory(_location);
}
_directory = FSDirectory.Open(_location);
_engine = new PackageSearchEngine(IndexDirectory, Analyzer, Logger);
}
protected virtual Lucene.Net.Store.Directory GetLuceneDirectory()
{
if (_nrtWriter != null)
{
return(_nrtWriter.GetDirectory());
}
if (_luceneDirectory == null)
{
_luceneDirectory = new SimpleFSDirectory(LuceneIndexFolder);
}
return(_luceneDirectory);
}
// Activity 7
public void OpenIndex(string indexPath)
{
/* Make sure to pass a new directory that does not exist */
if (System.IO.Directory.Exists(indexPath))
{
Console.WriteLine("This directory already exists - Choose a directory that does not exist");
Console.Write("Hit any key to exit");
Console.ReadKey();
Environment.Exit(0);
}
luceneIndexDirectory = FSDirectory.Open(indexPath);
}
public RemoteSyncDirectory(IRemoteDirectory remoteDirectory,
Directory cacheDirectory,
bool compressBlobs = false)
{
CacheDirectory = cacheDirectory;
LoggingService = new TraceLoggingService();
RemoteDirectory = remoteDirectory;
_lockFactory = GetLockFactory();
_remoteIndexOutputFactory = GetAzureIndexOutputFactory();
_remoteDirectoryIndexInputFactory = GetAzureIndexInputFactory();
GuardCacheDirectory(CacheDirectory);
CompressBlobs = compressBlobs;
}
public IndexAndTaxonomyReadyCallback(MockDirectoryWrapper indexDir, MockDirectoryWrapper taxoDir)
{
this.indexDir = indexDir;
this.taxoDir = taxoDir;
config = new FacetsConfig();
config.SetHierarchical("A", true);
if (DirectoryReader.IndexExists(indexDir))
{
indexReader = DirectoryReader.Open(indexDir);
lastIndexGeneration = indexReader.IndexCommit.Generation;
taxoReader = new DirectoryTaxonomyReader(taxoDir);
}
}
protected virtual Lucene.Net.Store.Directory GetLuceneDirectory()
{
if (_nrtWriter != null)
{
return(_nrtWriter.GetDirectory());
}
if (_luceneDirectory == null)
{
_luceneDirectory = DirectoryTracker.Current.GetDirectory(LuceneIndexFolder);
}
return(_luceneDirectory);
}
public void Dispose()
{
LogManager.GetCurrentClassLogger().Info("Stopping Lucene indexing services.");
if (PackageFileSystemWatcher != null)
{
PackageFileSystemWatcher.Dispose();
}
PackageIndexer.Dispose();
Provider.Dispose();
LuceneDirectory.Dispose();
}
public virtual String SegString(Directory dir, IState state)
{
var b = new System.Text.StringBuilder();
b.Append("MergeSpec:\n");
int count = merges.Count;
for (int i = 0; i < count; i++)
{
b.Append(" ").Append(1 + i).Append(": ").Append(merges[i].SegString(dir, state));
}
return(b.ToString());
}
public MultiIndexLockFactory(Lucene.Net.Store.Directory master, Lucene.Net.Store.Directory child)
{
if (master == null)
{
throw new ArgumentNullException("master");
}
if (child == null)
{
throw new ArgumentNullException("child");
}
_master = master;
_child = child;
}
public LuceneIndexingService(
IEntityRepository <Package> packageSource,
IEntityRepository <CuratedPackage> curatedPackageSource,
Lucene.Net.Store.Directory directory,
IDiagnosticsService diagnostics,
IAppConfiguration config)
{
_packageRepository = packageSource;
_curatedPackageRepository = curatedPackageSource;
_directory = directory;
_getShouldAutoUpdate = config == null ? new Func <bool>(() => true) : new Func <bool>(() => config.AutoUpdateSearchIndex);
Trace = diagnostics.SafeGetSource("LuceneIndexingService");
}
public void Dispose()
{
lock (this)
{
if (writer != null)
{
writer.Close(waitForMerges: true);
}
writer = null;
searcher = null;
directory = null;
}
}
public SqloogleMiaSearcher(string indexPath, int resultsLimit = 50)
{
_resultsLimit = resultsLimit;
var fields = new[] {
"server",
"database",
"schema",
"name",
"equality",
"inequality",
"included"
};
var boosts = new Dictionary<string, float> {
{ "server", .4F },
{ "database", .3F },
{ "schema", .2F },
{ "name", .1F },
{ "equality", .0F },
{ "inequality", .0F },
{ "included", .0F }
};
_directory = FSDirectory.Open(new DirectoryInfo(indexPath));
_searcher = new IndexSearcher(_directory, true);
_analyzer = new StandardAnalyzer(Version.LUCENE_30);
_parser = new MultiFieldQueryParser(Version.LUCENE_30, fields, _analyzer, boosts) {
DefaultOperator = QueryParser.Operator.AND
};
_logger.Trace("Searcher is ready.");
}
