public static void CreateIndex() { try { var cloudAccount = Azure.GetStorageAccount(); using (var cacheDirectory = new RAMDirectory()) { using (var azureDirectory = new AzureDirectory(cloudAccount, Azure.StorageContainerName, cacheDirectory)) { using (Analyzer analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30)) { using (var indexWriter = new IndexWriter(azureDirectory, analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED)) { AddDocuments(indexWriter); indexWriter.Commit(); } } } } } catch (StorageException ex) { Trace.TraceError(ex.Message); } }
private void GenerateLuceneIndex(List<Entry> entries) { Microsoft.WindowsAzure.Storage.CloudStorageAccount storageAccount = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=diningsearchstorage;AccountKey=xeYMzXThFxrU7SsAMGbSWLdy9psLFRMk5NI8x0bx24xtEg9MPIstf/xwPdjvDm6HpHZaCPxxVFCv/7DDd5wymA=="); AzureDirectory azureDirectory = new AzureDirectory(storageAccount, "diningsearchindex"); Analyzer analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30); IndexWriter indexWriter = new IndexWriter(azureDirectory, analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED); //// int c = 0; foreach (var entry in entries) { c++; var item = new Document(); item.Add(new Field("Id", c.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED)); item.Add(new Field("Dish", entry.DishName, Field.Store.YES, Field.Index.ANALYZED)); item.Add(new Field("Cafe/Restaurant", string.Format("{0}/{1}", entry.CafeName, entry.RestaurantName), Field.Store.YES, Field.Index.ANALYZED)); item.Add(new Field("URL", entry.CafeUrl, Field.Store.YES, Field.Index.NOT_ANALYZED)); item.Add(new Field("Description", entry.Description ?? string.Empty, Field.Store.YES, Field.Index.ANALYZED)); item.Add(new Field("Price", entry.Price, Field.Store.YES, Field.Index.NOT_ANALYZED)); indexWriter.AddDocument(item); } //// indexWriter.Dispose(); azureDirectory.Dispose(); }
public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob) : base(blob.Name) { _name = blob.Uri.Segments[blob.Uri.Segments.Length - 1]; _fileMutex = BlobMutexManager.GrabMutex(_name); _fileMutex.WaitOne(); try { _azureDirectory = azuredirectory; _blobContainer = azuredirectory.BlobContainer; _blob = blob; string fileName = _name; StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName); // get the blob _blob.DownloadToStream(fileStream); fileStream.Flush(); Debug.WriteLine("GET {0} RETREIVED {1} bytes", _name, fileStream.Length); fileStream.Close(); // and open it as an input _indexInput = CacheDirectory.openInput(fileName, IOContext.DEFAULT); } finally { _fileMutex.ReleaseMutex(); } }
public void TestReadAndWrite() { var cloudStorageAccount = CloudStorageAccount.Parse(connectionString); const string containerName = "testcatalog2"; var azureDirectory = new Lucene.Net.Store.Azure.AzureDirectory(cloudStorageAccount, "temp", containerName: containerName); var indexWriterConfig = new IndexWriterConfig( Lucene.Net.Util.LuceneVersion.LUCENE_48, new StandardAnalyzer(Lucene.Net.Util.LuceneVersion.LUCENE_48)); int dog = 0, cat = 0, car = 0; using (var indexWriter = new IndexWriter(azureDirectory, indexWriterConfig)) { for (var iDoc = 0; iDoc < 10000; iDoc++) { var bodyText = GeneratePhrase(40); var doc = new Document { new TextField("id", DateTime.Now.ToFileTimeUtc() + "-" + iDoc, Field.Store.YES), new TextField("Title", GeneratePhrase(10), Field.Store.YES), new TextField("Body", bodyText, Field.Store.YES) }; dog += bodyText.Contains(" dog ") ? 1 : 0; cat += bodyText.Contains(" cat ") ? 1 : 0; car += bodyText.Contains(" car ") ? 1 : 0; indexWriter.AddDocument(doc); } Console.WriteLine("Total docs is {0}, {1} dog, {2} cat, {3} car", indexWriter.NumDocs, dog, cat, car); } try { var ireader = DirectoryReader.Open(azureDirectory); for (var i = 0; i < 100; i++) { var searcher = new IndexSearcher(ireader); var searchForPhrase = SearchForPhrase(searcher, "dog"); Assert.AreEqual(dog, searchForPhrase); searchForPhrase = SearchForPhrase(searcher, "cat"); Assert.AreEqual(cat, searchForPhrase); searchForPhrase = SearchForPhrase(searcher, "car"); Assert.AreEqual(car, searchForPhrase); } Console.WriteLine("Tests passsed"); } catch (Exception x) { Console.WriteLine("Tests failed:\n{0}", x); } finally { // check the container exists, and delete it var blobClient = cloudStorageAccount.CreateCloudBlobClient(); var container = blobClient.GetContainerReference(containerName); Assert.IsTrue(container.Exists()); // check the container exists container.Delete(); } }
private static void UnidirectionalSync(AzureDirectory sourceDirectory, Directory destinationDirectory) { var sourceFiles = sourceDirectory.ListAll(); var fileNameFilter = IndexFileNameFilter.Filter; byte[] buffer = new byte[16384]; foreach (string sourceFile in sourceFiles) { // only copy file if it is accepted by Lucene's default filter // and it does not already exist (except for segment map files, we always want those) if (fileNameFilter.Accept(null, sourceFile) && (!destinationDirectory.FileExists(sourceFile) || sourceFile.StartsWith("segment"))) { IndexOutput indexOutput = null; IndexInput indexInput = null; try { indexOutput = destinationDirectory.CreateOutput(sourceFile); indexInput = sourceDirectory.OpenInput(sourceFile); long length = indexInput.Length(); long position = 0; while (position < length) { int bytesToRead = position + 16384L > length ? (int)(length - position) : 16384; indexInput.ReadBytes(buffer, 0, bytesToRead); indexOutput.WriteBytes(buffer, bytesToRead); position += bytesToRead; } } finally { try { indexOutput?.Close(); } finally { indexInput?.Close(); } } } } // we'll remove old files from both AzureDirectory's cache directory, as well as our destination directory // (only when older than 45 minutes - old files may still have active searches on them so we need a margin) var referenceTimestamp = LuceneTimestampFromDateTime(DateTime.UtcNow.AddMinutes(-45)); // remove old files from AzureDirectory cache directory RemoveOldFiles(sourceDirectory.CacheDirectory, sourceFiles, referenceTimestamp); // remove old files from destination directory RemoveOldFiles(destinationDirectory, sourceFiles, referenceTimestamp); }
static void Main(string[] args) { // default AzureDirectory stores cache in local temp folder var azureDirectory = new AzureDirectory(CloudStorageAccount.Parse(ConfigurationManager.AppSettings["blobStorage"]), "TestCatalog6"); var findexExists = IndexReader.IndexExists(azureDirectory); IndexWriter indexWriter = null; while (indexWriter == null) { try { indexWriter = new IndexWriter(azureDirectory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_CURRENT), !IndexReader.IndexExists(azureDirectory), new Lucene.Net.Index.IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH)); } catch (LockObtainFailedException) { Console.WriteLine("Lock is taken, Hit 'Y' to clear the lock, or anything else to try again"); if (Console.ReadLine().ToLower().Trim() == "y" ) azureDirectory.ClearLock("write.lock"); } }; Console.WriteLine("IndexWriter lock obtained, this process has exclusive write access to index"); indexWriter.SetRAMBufferSizeMB(10.0); //indexWriter.SetUseCompoundFile(false); //indexWriter.SetMaxMergeDocs(10000); //indexWriter.SetMergeFactor(100); for (int iDoc = 0; iDoc < 10000; iDoc++) { if (iDoc % 10 == 0) Console.WriteLine(iDoc); var doc = new Document(); doc.Add(new Field("id", DateTime.Now.ToFileTimeUtc().ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); doc.Add(new Field("Title", GeneratePhrase(10), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); doc.Add(new Field("Body", GeneratePhrase(40), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); indexWriter.AddDocument(doc); } Console.WriteLine("Total docs is {0}", indexWriter.NumDocs()); indexWriter.Dispose(); IndexSearcher searcher; using (new AutoStopWatch("Creating searcher")) { searcher = new IndexSearcher(azureDirectory); } SearchForPhrase(searcher, "dog"); SearchForPhrase(searcher, _random.Next(32768).ToString()); SearchForPhrase(searcher, _random.Next(32768).ToString()); Console.Read(); }
public IActionResult Get(string searchTerm) { if (string.IsNullOrEmpty(searchTerm)) { return new JsonResult(null); } if (searchTerm.StartsWith("*") || searchTerm.StartsWith("?")) { return new JsonResult(null); } CloudStorageAccount cloudStorageAccount = CloudStorageAccount.DevelopmentStorageAccount; CloudStorageAccount.TryParse(_options.ConnectionString, out cloudStorageAccount); var azureDirectory = new AzureDirectory(cloudStorageAccount, "TestCatalog"); var searcher = new IndexSearcher(azureDirectory); var analyzer = new NuGet.Indexing.IdentifierAnalyzer(); var queryParser = new MultiFieldQueryParser(Lucene.Net.Util.Version.LUCENE_30, new[] { "Type", "ReturnType" }, analyzer); Query query = queryParser.Parse(searchTerm); //execute the query var hits = searcher.Search(query, 50); var packages = new List<SearchResult>(); foreach (var hit in hits.ScoreDocs) { var doc = searcher.Doc(hit.Doc); var result = new SearchResult { FullTypeName = doc.GetField("Type").StringValue, PackageName = doc.GetField("Package").StringValue }; if (doc.GetField("Signature") != null) { result.Signature = doc.GetField("Signature").StringValue; } if (doc.GetField("ReturnType") != null) { result.ReturnType = doc.GetField("ReturnType").StringValue; } packages.Add(result); } return new JsonResult(packages); }
public AzureDirectorySynchronizer(AzureDirectory sourceDirectory, Directory destinationDirectory) { if (sourceDirectory == null) { throw new ArgumentNullException(nameof(sourceDirectory)); } if (destinationDirectory == null) { throw new ArgumentNullException(nameof(destinationDirectory)); } SourceDirectory = sourceDirectory; DestinationDirectory = destinationDirectory; }
public ActionResult Index(string searchText) { if (string.IsNullOrWhiteSpace(searchText)) { return View((object)null); } var searchResults = new List<SearchResult>(); try { var cloudAccount = Azure.GetStorageAccount(); using (var cacheDirectory = new RAMDirectory()) { using ( var azureDirectory = new AzureDirectory( cloudAccount, Azure.StorageContainerName, cacheDirectory)) { using (var analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30)) { using (var searcher = new IndexSearcher(azureDirectory)) { var parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, "Body", analyzer); var searchQuery = parser.Parse(searchText); var hits = searcher.Search(searchQuery, 200); var results = hits.TotalHits; for (var i = 0; i < results; i++) { var doc = searcher.Doc(hits.ScoreDocs[i].Doc); searchResults.Add(new SearchResult(doc)); } } } } } } catch (StorageException ex) { Trace.TraceError(ex.Message); return this.View("SearchUnavailable"); } return this.View(searchResults); }
public override bool OnStart() { var readPath = RoleEnvironment.GetLocalResource("ReadStorage").RootPath; var writePath = RoleEnvironment.GetLocalResource("WriteStorage").RootPath; var subscription = "lucene" ; var account = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("Blob")); var hub = new Hub(RoleEnvironment.GetConfigurationSettingValue("ServiceBus")); var criticalWait = Task.Run(() => { string writeFolder = Path.Combine(writePath, "lucene"); if (!System.IO.Directory.Exists(writeFolder)) { System.IO.Directory.CreateDirectory(writeFolder); } string combine = Path.Combine(readPath, subscription); if (!System.IO.Directory.Exists(combine)) { System.IO.Directory.CreateDirectory(combine); } var localDirectory = new Lucene.Net.Store.SimpleFSDirectory(new DirectoryInfo(combine)); var masterDirectory = new AzureDirectory(account, subscription); var irs = new IntermediateReaderService(masterDirectory, hub, localDirectory, true); masterDirectory.Dispose(); return new IndexInfo {IRS = irs as IDisposable, Directory = localDirectory}; }) .ContinueWith(t => _localIndexes = new[] {t.Result}); _luceneWriter = new SearchWriterService(new AzureDirectory(account, "lucene", new Lucene.Net.Store.SimpleFSDirectory(new DirectoryInfo(writePath))), true); var subscriptionName = string.Format("{0:yyyyMMddHHmmss}_{1}", DateTime.UtcNow, Guid.NewGuid().ToString().Replace("-", string.Empty)); hub.CreateSubscription("lucene", subscriptionName) .ContinueWith(t => { _luceneTempInstanceSubscription = t.Result; return hub.SubscribeWithRoutingKey("lucene", subscriptionName, OnSearchMessage) .ContinueWith(t2 => _luceneSearchSubscription = t.Result); }) .Unwrap(); return base.OnStart(); }
public AzureIndexOutput(AzureDirectory azureDirectory, ICloudBlob blob) { _fileMutex = BlobMutexManager.GrabMutex(_name); _fileMutex.WaitOne(); try { _azureDirectory = azureDirectory; _blob = blob; _name = blob.Uri.Segments[blob.Uri.Segments.Length - 1]; // create the local cache one we will operate against... _indexOutput = CacheDirectory.createOutput(_name, IOContext.DEFAULT); } finally { _fileMutex.ReleaseMutex(); } }
public AzureIndexOutput(AzureDirectory azureDirectory, CloudBlob blob) { _fileMutex = new Mutex(false, _name); _fileMutex.WaitOne(); try { _azureDirectory = azureDirectory; _blobContainer = _azureDirectory.BlobContainer; _blob = blob; _name = blob.Uri.Segments[blob.Uri.Segments.Length - 1]; // create the local cache one we will operate against... _indexOutput = CacheDirectory.CreateOutput(_name); } finally { _fileMutex.ReleaseMutex(); } }
/// <summary> /// Initializes the Storage Provider. /// </summary> /// <param name="host">The Host of the Component.</param> /// <param name="config">The Configuration data, if any.</param> /// <param name="wiki">The wiki.</param> /// <exception cref="ArgumentNullException">If <paramref name="host"/> or <paramref name="config"/> are <c>null</c>.</exception> /// <exception cref="InvalidConfigurationException">If <paramref name="config"/> is not valid or is incorrect.</exception> public void Init(IHostV40 host, string config, string wiki) { if (host == null) { throw new ArgumentNullException("host"); } if (config == null) { throw new ArgumentNullException("config"); } if (config == "") { config = Config.GetConnectionString(); } _host = host; _wiki = string.IsNullOrEmpty(wiki) ? "root" : wiki; _directory = new AzureDirectory(TableStorage.StorageAccount(config), wiki + "-searchindex"); }
public void TestReadAndWrite() { var connectionString = Environment.GetEnvironmentVariable("DataConnectionString") ?? "UseDevelopmentStorage=true"; var cloudStorageAccount = CloudStorageAccount.Parse(connectionString); // default AzureDirectory stores cache in local temp folder var azureDirectory = new AzureDirectory(cloudStorageAccount, "testcatalog"); using (var indexWriter = new IndexWriter(azureDirectory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30), !IndexReader.IndexExists(azureDirectory), new Lucene.Net.Index.IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH))) { indexWriter.SetRAMBufferSizeMB(10.0); for (int iDoc = 0; iDoc < 10000; iDoc++) { var doc = new Document(); doc.Add(new Field("id", DateTime.Now.ToFileTimeUtc().ToString() + "-" + iDoc.ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); doc.Add(new Field("Title", GeneratePhrase(10), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); doc.Add(new Field("Body", GeneratePhrase(40), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); indexWriter.AddDocument(doc); } Console.WriteLine("Total docs is {0}", indexWriter.NumDocs()); } using (var searcher = new IndexSearcher(azureDirectory)) { Assert.AreNotEqual(0, SearchForPhrase(searcher, "dog")); Assert.AreNotEqual(0, SearchForPhrase(searcher, "cat")); Assert.AreNotEqual(0, SearchForPhrase(searcher, "car")); } // check the container exists, and delete it var blobClient = cloudStorageAccount.CreateCloudBlobClient(); var container = blobClient.GetContainerReference("testcatalog"); Assert.IsTrue(container.Exists()); // check the container exists container.Delete(); }
public AzureIndexInput(AzureIndexInput cloneInput) : base(cloneInput._blob.Name) { _fileMutex = BlobMutexManager.GrabMutex(cloneInput._name); _fileMutex.WaitOne(); try { _azureDirectory = cloneInput._azureDirectory; _blobContainer = cloneInput._blobContainer; _blob = cloneInput._blob; _indexInput = cloneInput._indexInput.clone(); } catch (Exception) { // sometimes we get access denied on the 2nd stream...but not always. I haven't tracked it down yet // but this covers our tail until I do Debug.WriteLine(String.Format("Dagnabbit, falling back to memory clone for {0}", cloneInput._name)); } finally { _fileMutex.ReleaseMutex(); } }
private List<SearchResult> GetResults(string searchText) { Microsoft.WindowsAzure.Storage.CloudStorageAccount storageAccount = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=diningsearchstorage;AccountKey=xeYMzXThFxrU7SsAMGbSWLdy9psLFRMk5NI8x0bx24xtEg9MPIstf/xwPdjvDm6HpHZaCPxxVFCv/7DDd5wymA=="); AzureDirectory azureDirectory = new AzureDirectory(storageAccount, "diningsearchindex"); IndexReader indexReader = IndexReader.Open(azureDirectory, true); Searcher indexSearch = new IndexSearcher(indexReader); Analyzer analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30); var queryParser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, "Dish", analyzer); var query = queryParser.Parse(searchText); Console.WriteLine("Searching for: " + query.ToString()); TopDocs resultDocs = indexSearch.Search(query, 100); Console.WriteLine("Results Found: " + resultDocs.TotalHits); List<SearchResult> searchResults = new List<SearchResult>(); var hits = resultDocs.ScoreDocs; foreach (var hit in hits) { Document documentFromSearcher = indexSearch.Doc(hit.Doc); SearchResult result = new SearchResult { CafeName = documentFromSearcher.Get("Cafe/Restaurant"), CafeUrl = documentFromSearcher.Get("URL"), DishName = documentFromSearcher.Get("Dish"), Price = documentFromSearcher.Get("Price"), Description = documentFromSearcher.Get("Description") }; searchResults.Add(result); } return searchResults; }
public invTypeIndexRepository() { cloudStorageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=http;AccountName=eveapi;AccountKey=KrpLpd9DQkbKFC718eVzkkUdt5fafelr6W+Pyzu+kT+CrHSWEFBhlB4OmXMgVQ2iC+vYPTTNRbmbk8guUrKmDw=="); azureDirectory = new AzureDirectory(cloudStorageAccount, "invTypesIndex"); searcher = new IndexSearcher(azureDirectory); }
public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob) { _name = blob.Uri.Segments[blob.Uri.Segments.Length - 1]; #if FULLDEBUG Debug.WriteLine(String.Format("opening {0} ", _name)); #endif _fileMutex = BlobMutexManager.GrabMutex(_name); _fileMutex.WaitOne(); try { _azureDirectory = azuredirectory; _blobContainer = azuredirectory.BlobContainer; _blob = blob; var fileName = _name; var fFileNeeded = false; if (!CacheDirectory.FileExists(fileName)) { fFileNeeded = true; } else { long cachedLength = CacheDirectory.FileLength(fileName); string blobLengthMetadata; bool hasMetadataValue = blob.Metadata.TryGetValue("CachedLength", out blobLengthMetadata); long blobLength = blob.Properties.Length; if (hasMetadataValue) long.TryParse(blobLengthMetadata, out blobLength); string blobLastModifiedMetadata; long longLastModified = 0; DateTime blobLastModifiedUTC = blob.Properties.LastModified.Value.UtcDateTime; if (blob.Metadata.TryGetValue("CachedLastModified", out blobLastModifiedMetadata)) { if (long.TryParse(blobLastModifiedMetadata, out longLastModified)) blobLastModifiedUTC = new DateTime(longLastModified).ToUniversalTime(); } if (cachedLength != blobLength) fFileNeeded = true; else { // there seems to be an error of 1 tick which happens every once in a while // for now we will say that if they are within 1 tick of each other and same length var cachedLastModifiedUTC = new DateTime(CacheDirectory.FileModified(fileName), DateTimeKind.Local).ToUniversalTime(); if (cachedLastModifiedUTC != blobLastModifiedUTC) { var timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC); if (timeSpan.TotalSeconds > 1) fFileNeeded = true; else { #if FULLDEBUG Debug.WriteLine(timeSpan.TotalSeconds); #endif // file not needed } } } } // if the file does not exist // or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage) if (fFileNeeded) { if (_azureDirectory.ShouldCompressFile(_name)) { InflateStream(fileName); } else { using (var fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName)) { // get the blob _blob.DownloadToStream(fileStream); fileStream.Flush(); Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length)); } } // and open it as an input _indexInput = CacheDirectory.OpenInput(fileName); } else { #if FULLDEBUG Debug.WriteLine(String.Format("Using cached file for {0}", _name)); #endif // open the file in read only mode _indexInput = CacheDirectory.OpenInput(fileName); } } finally { _fileMutex.ReleaseMutex(); } }
static void Main(string[] args) { // get settings from azure settings or app.config CloudStorageAccount.SetConfigurationSettingPublisher((configName, configSetter) => { try { configSetter(RoleEnvironment.GetConfigurationSettingValue(configName)); } catch (Exception) { // for a console app, reading from App.config configSetter(System.Configuration.ConfigurationManager.AppSettings[configName]); } }); // default AzureDirectory stores cache in local temp folder AzureDirectory azureDirectory = new AzureDirectory(CloudStorageAccount.FromConfigurationSetting("blobStorage"), "TestCatalog6"); bool findexExists = IndexReader.IndexExists(azureDirectory); IndexWriter indexWriter = null; while (indexWriter == null) { try { indexWriter = new IndexWriter(azureDirectory, new StandardAnalyzer(), !IndexReader.IndexExists(azureDirectory)); } catch (LockObtainFailedException) { Console.WriteLine("Lock is taken, Hit 'Y' to clear the lock, or anything else to try again"); if (Console.ReadLine().ToLower().Trim() == "y" ) azureDirectory.ClearLock("write.lock"); } }; Console.WriteLine("IndexWriter lock obtained, this process has exclusive write access to index"); indexWriter.SetRAMBufferSizeMB(10.0); indexWriter.SetUseCompoundFile(false); indexWriter.SetMaxMergeDocs(10000); indexWriter.SetMergeFactor(100); for (int iDoc = 0; iDoc < 10000; iDoc++) { if (iDoc % 10 == 0) Console.WriteLine(iDoc); Document doc = new Document(); doc.Add(new Field("id", DateTime.Now.ToFileTimeUtc().ToString(), Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.NO)); doc.Add(new Field("Title", GeneratePhrase(10), Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.NO)); doc.Add(new Field("Body", GeneratePhrase(40), Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.NO)); indexWriter.AddDocument(doc); } Console.WriteLine("Total docs is {0}", indexWriter.DocCount()); indexWriter.Close(); IndexSearcher searcher; using (new AutoStopWatch("Creating searcher")) { searcher = new IndexSearcher(azureDirectory); } SearchForPhrase(searcher, "dog"); SearchForPhrase(searcher, _random.Next(32768).ToString()); SearchForPhrase(searcher, _random.Next(32768).ToString()); }
public AzureLockFactory(AzureDirectory directory, Directory cacheDirectory, string rootFolder) { _directory = directory; _cacheDirectory = cacheDirectory; _rootFolder = rootFolder; }
public void ProcessRequest(HttpContext context) { Int64 expirationSeconds = 1; HttpCacheability hc = HttpCacheability.NoCache; try { string retVal = string.Empty; XmlDocument xdoc = new XmlDocument(); xdoc.LoadXml(new Utility().getConfigXML()); XmlNode xNode = xdoc.SelectSingleNode("//blobData[@name='default']"); string azureAccount = xNode.SelectSingleNode("Setting[@name='account']").Attributes["value"].Value; string azureEndpoint = xNode.SelectSingleNode("Setting[@name='endpoint']").Attributes["value"].Value; string azureSharedKey = xNode.SelectSingleNode("Setting[@name='accountSharedKey']").Attributes["value"].Value; string blobStorage = azureEndpoint; xNode = xdoc.SelectSingleNode(string.Format("//fragmentData/Setting[@name='luceneFragments']")); string fragmentLocation = xNode.Attributes["value"].Value ; string queryName = string.Empty; SetCompression(context); try { AzureBlobStorage abs = new AzureBlobStorage(azureAccount, blobStorage, azureSharedKey, "SharedKey"); azureResults ar = new azureResults(); // Get the page name and replace the .q extension with .xml queryName = context.Request.Path; queryName = queryName.Substring(queryName.LastIndexOf("/") + 1); queryName = queryName.Substring(0, queryName.LastIndexOf(".")) + ".xml"; byte[] xmlFragment = abs.GetBlob(fragmentLocation, queryName, "", ref ar, ""); if (!ar.Succeeded) { context.Response.StatusCode = (int)ar.StatusCode; } else { xdoc = new XmlDocument(); System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding(); xdoc.LoadXml(enc.GetString(xmlFragment)); /* * http://azure-architect.com/portals/16/MOOPData.xsd */ string azureContainerName = string.Empty; azureContainerName = xdoc.SelectSingleNode("//MOOPData/luceneSearch/azureContainer[1]").InnerText; Microsoft.WindowsAzure.StorageCredentialsAccountAndKey scaak = new Microsoft.WindowsAzure.StorageCredentialsAccountAndKey(azureAccount, azureSharedKey); Microsoft.WindowsAzure.CloudStorageAccount csa = new Microsoft.WindowsAzure.CloudStorageAccount(scaak, false); AzureDirectory azDir = new AzureDirectory(csa, azureContainerName, new RAMDirectory()); // IndexSearcher searcher = new IndexSearcher("azure-lucene-search"); try { var q = new BooleanQuery(); XmlNode xn = xdoc.SelectSingleNode("//luceneSearch[1]"); XmlNodeList xnl = xn.SelectNodes("//query"); Query[] qArray = new Query[xnl.Count]; bool hasReplaceableSearchValues = false; bool hasReplacedAtLeastOneValue = false; bool requirementsMet = true; for (int i = 0; i < xnl.Count; i++) { XmlNode node = xnl[i]; string term = string.Empty; term = node.Attributes["term"].Value; string termValue = node.Attributes["termValue"].Value; string variableValue = node.Attributes["variableValue"].Value; string parmValue = string.Empty; string requiredField = node.Attributes["required"].Value; if (variableValue == "true") // See if there is a replacement attempt { if (requiredField == "true" && !context.Request.Params.AllKeys.Contains(termValue)) requirementsMet = false; hasReplaceableSearchValues = true; // Set the flag to say we are attempting to replace the value if (context.Request.Params.AllKeys.Contains(termValue)) // The parameter exists { hasReplacedAtLeastOneValue = true; parmValue = context.Request.Params[termValue].Replace("+", " ").Replace("%20", " "); } } if (node.Attributes["useAnalyzer"].Value == "true") // Should we use the analyzer { if (variableValue == "true" && context.Request.Params.AllKeys.Contains(termValue)) // Did we actually have a replaceable value or a static value qArray[i] = new QueryParser(Lucene.Net.Util.Version.LUCENE_29, term, new SnowballAnalyzer("English")).Parse(parmValue); if (variableValue != "true") qArray[i] = new QueryParser(Lucene.Net.Util.Version.LUCENE_29, term, new SnowballAnalyzer("English")).Parse(termValue); } else // Using a Boolean { if (variableValue == "true" && context.Request.Params.AllKeys.Contains(termValue)) // Did we actually have a replaceable value or a static value qArray[i] = new TermQuery(new Term(term, parmValue)); if (variableValue != "true") qArray[i] = new TermQuery(new Term(term, termValue)); } if (qArray[i] != null) { switch (node.Attributes["booleanClauseOccurs"].Value.ToLower()) { case "must": q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.MUST)); break; case "must_not": q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.MUST_NOT)); break; case "should": q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.SHOULD)); break; default: q.Add(new BooleanClause(qArray[i], BooleanClause.Occur.MUST)); break; } } } IndexSearcher searcher2 = new IndexSearcher(azDir, true); TopScoreDocCollector collector = TopScoreDocCollector.Create(1000, true); // If we have either no replaceable values or have replaceable values and at least one was provided if (!hasReplaceableSearchValues || (hasReplaceableSearchValues && hasReplacedAtLeastOneValue)) if (requirementsMet) searcher2.Search(q, collector); int indexID = 1; ScoreDoc[] hits = collector.TopDocs().ScoreDocs; StringBuilder xmlOutput = new StringBuilder(); xmlOutput.AppendFormat("<?xml version=\"1.0\"?><root>"); for (int i = 0; i < hits.Length; ++i) { xmlOutput.AppendFormat("<hit>"); int docId = hits[i].Doc; Document d = searcher2.Doc(docId); xmlOutput.AppendFormat("<score>{0}</score><docID>{1}</docID>", hits[i].Score, indexID ++); foreach (Field f in d.GetFields()) { if (f.StringValue() == null) xmlOutput.AppendFormat("<{0} />", f.Name()); else xmlOutput.AppendFormat("<{0}>{1}</{0}>", f.Name(), System.Security.SecurityElement.Escape(f.StringValue())); } xmlOutput.AppendFormat("</hit>"); } xmlOutput.AppendFormat("</root>"); retVal = xmlOutput.ToString(); } catch (Exception ex) { retVal = "<root />"; } MessageFormatOptions defaultOptions = new MessageFormatOptions(); string preferredContentType = string.Empty; if (context.Request.HttpMethod == "GET" || context.Request.HttpMethod == "DELETE") { preferredContentType = (context.Request.AcceptTypes ?? new string[0]) .Select(m => m.Split(';')[0]) .FirstOrDefault(m => defaultOptions.MimeInputTypes.ContainsKey(m)) ?? defaultOptions.DefaultContentType; if (preferredContentType.Trim() == string.Empty) preferredContentType = context.Request.Headers["Content-Type"]; } else preferredContentType = context.Request.Headers["Content-Type"]; if (preferredContentType == "application/json") { context.Response.ContentType = preferredContentType; XmlDocument doc = new XmlDocument(); doc.LoadXml(retVal); retVal = JsonConvert.SerializeXmlNode(doc); } else { string luceneTransform = string.Empty; luceneTransform = xdoc.SelectSingleNode("/MOOPData/luceneSearch/transform").InnerText; if (luceneTransform != string.Empty && luceneTransform != null) { xmlFragment = abs.GetBlob(fragmentLocation, luceneTransform, "", ref ar, ""); if (ar.Succeeded) { XsltUtil xslu = new XsltUtil(); retVal = XsltUtil.TransformXml(retVal, System.Text.ASCIIEncoding.ASCII.GetString(xmlFragment)); string transformContentType = "text/html"; try { transformContentType = xdoc.SelectSingleNode("/MOOPData/luceneSearch/transform").Attributes["contentType"].Value; } catch { } context.Response.ContentType = transformContentType; } } else { context.Response.ContentType = "text/xml"; } } } } catch (Exception ex) { FrameworkUtility u = new FrameworkUtility(new Utility().ResolveDataConnection("sqlAzureConnection")); FrameworkUtility.ParsedURI pUri = new FrameworkUtility.ParsedURI(context.Request.RawUrl); u.LogData(pUri.domainName, pUri.folderName, pUri.pageName, context.Request.HttpMethod, context.Request.UserHostAddress, "", "QueryError", ex.ToString(), u.nvc2XML(context.Request.Headers)); //retVal = string.Format("<!-- {0} -->", ex.ToString()); context.Response.StatusDescription = "An error occured but it was logged for later review"; context.Response.StatusCode = 400; } finally { if (retVal == string.Empty) retVal = "<root />"; } context.Response.Cache.SetExpires(DateTime.Now.AddSeconds(expirationSeconds)); context.Response.Cache.SetCacheability(hc); context.Response.Write(retVal); } catch (Exception ex) { context.Response.StatusCode = 404; context.Response.Close(); } }
public AzureIndexInput(AzureDirectory azuredirectory, BlobClient blob, string resourceDescription) : base(resourceDescription) { _name = blob.Uri.Segments[blob.Uri.Segments.Length - 1]; #if FULLDEBUG Debug.WriteLine(String.Format("opening {0} ", _name)); #endif _fileMutex = BlobMutexManager.GrabMutex(_name); _fileMutex.WaitOne(); try { _azureDirectory = azuredirectory; _blobContainer = azuredirectory.BlobContainer; _blob = blob; var fileName = _name; var fFileNeeded = false; if (!CacheDirectory.FileExists(fileName)) { fFileNeeded = true; } else { long cachedLength = CacheDirectory.FileLength(fileName); string blobLengthMetadata; var properties = blob.GetProperties().Value; bool hasMetadataValue = properties.Metadata.TryGetValue("CachedLength", out blobLengthMetadata); long blobLength = properties.ContentLength; if (hasMetadataValue) { long.TryParse(blobLengthMetadata, out blobLength); } string blobLastModifiedMetadata; long longLastModified = 0; DateTime blobLastModifiedUTC = properties.LastModified.UtcDateTime; if (properties.Metadata.TryGetValue("CachedLastModified", out blobLastModifiedMetadata)) { if (long.TryParse(blobLastModifiedMetadata, out longLastModified)) { blobLastModifiedUTC = DateTime.FromFileTimeUtc(longLastModified); } //blobLastModifiedUTC = new DateTime(longLastModified).ToUniversalTime(); } if (cachedLength != blobLength) { fFileNeeded = true; } else { // cachedLastModifiedUTC was not ouputting with a date (just time) and the time was always off var filePath = Path.Combine(_azureDirectory.CatalogPath, fileName); var lastModified = File.GetLastWriteTimeUtc(filePath); //long unixDate = CacheDirectory.FileModified(fileName); //DateTime start = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); //var cachedLastModifiedUTC = start.AddMilliseconds(unixDate).ToUniversalTime(); var cachedLastModifiedUTC = lastModified; if (cachedLastModifiedUTC != blobLastModifiedUTC) { var timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC); if (timeSpan.TotalSeconds > 1) { fFileNeeded = true; } else { #if FULLDEBUG Debug.WriteLine(timeSpan.TotalSeconds); #endif // file not needed } } } } // if the file does not exist // or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage) if (fFileNeeded) { if (_azureDirectory.ShouldCompressFile(_name)) { InflateStream(fileName); } else { using (var fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName)) { // get the blob _blob.DownloadTo(fileStream); fileStream.Flush(); Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length)); } } // and open it as an input _indexInput = CacheDirectory.OpenInput(fileName, IOContext.DEFAULT); } else { #if FULLDEBUG Debug.WriteLine(String.Format("Using cached file for {0}", _name)); #endif // open the file in read only mode _indexInput = CacheDirectory.OpenInput(fileName, IOContext.DEFAULT); } } finally { _fileMutex.ReleaseMutex(); } }
protected override void Dispose(bool disposing) { _fileMutex.WaitOne(); try { #if FULLDEBUG Debug.WriteLine(String.Format("CLOSED READSTREAM local {0}", _name)); #endif _indexInput.Dispose(); _indexInput = null; _azureDirectory = null; _blobContainer = null; _blob = null; GC.SuppressFinalize(this); } finally { _fileMutex.ReleaseMutex(); } }
public AzureLock(string lockFile, AzureDirectory directory) { _lockFile = lockFile; _azureDirectory = directory; }
public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob) { _name = blob.Uri.Segments[blob.Uri.Segments.Length - 1]; #if FULLDEBUG Debug.WriteLine(String.Format("opening {0} ", _name)); #endif _fileMutex = BlobMutexManager.GrabMutex(_name); _fileMutex.WaitOne(); try { _azureDirectory = azuredirectory; _blobContainer = azuredirectory.BlobContainer; _blob = blob; string fileName = _name; bool fFileNeeded = false; if (!CacheDirectory.FileExists(fileName)) { fFileNeeded = true; } else { long cachedLength = CacheDirectory.FileLength(fileName); long blobLength = blob.Properties.Length; long.TryParse(blob.Metadata["CachedLength"], out blobLength); long longLastModified = 0; DateTime blobLastModifiedUTC = blob.Properties.LastModified.Value.UtcDateTime; if (long.TryParse(blob.Metadata["CachedLastModified"], out longLastModified)) { // normalize RAMDirectory and FSDirectory times if (longLastModified > ticks1970) { longLastModified -= ticks1970; } blobLastModifiedUTC = new DateTime(longLastModified).ToUniversalTime(); } if (cachedLength != blobLength) { fFileNeeded = true; } else { // there seems to be an error of 1 tick which happens every once in a while // for now we will say that if they are within 1 tick of each other and same length var elapsed = CacheDirectory.FileModified(fileName); // normalize RAMDirectory and FSDirectory times if (elapsed > ticks1970) { elapsed -= ticks1970; } DateTime cachedLastModifiedUTC = new DateTime(elapsed, DateTimeKind.Local).ToUniversalTime(); if (cachedLastModifiedUTC != blobLastModifiedUTC) { TimeSpan timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC); if (timeSpan.TotalSeconds > 1) { fFileNeeded = true; } else { #if FULLDEBUG Debug.WriteLine(timeSpan.TotalSeconds); #endif // file not needed } } } } // if the file does not exist // or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage) if (fFileNeeded) { #if COMPRESSBLOBS if (_azureDirectory.ShouldCompressFile(_name)) { // then we will get it fresh into local deflatedName // StreamOutput deflatedStream = new StreamOutput(CacheDirectory.CreateOutput(deflatedName)); MemoryStream deflatedStream = new MemoryStream(); // get the deflated blob _blob.DownloadToStream(deflatedStream); Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, deflatedStream.Length)); // seek back to begininng deflatedStream.Seek(0, SeekOrigin.Begin); // open output file for uncompressed contents StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName); // create decompressor DeflateStream decompressor = new DeflateStream(deflatedStream, CompressionMode.Decompress); byte[] bytes = new byte[65535]; int nRead = 0; do { nRead = decompressor.Read(bytes, 0, 65535); if (nRead > 0) { fileStream.Write(bytes, 0, nRead); } } while (nRead == 65535); decompressor.Close(); // this should close the deflatedFileStream too fileStream.Close(); } else #endif { StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName); // get the blob _blob.DownloadToStream(fileStream); fileStream.Flush(); Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length)); fileStream.Close(); } // and open it as an input _indexInput = CacheDirectory.OpenInput(fileName); } else { #if FULLDEBUG Debug.WriteLine(String.Format("Using cached file for {0}", _name)); #endif // open the file in read only mode _indexInput = CacheDirectory.OpenInput(fileName); } } finally { _fileMutex.ReleaseMutex(); } }
public AzureIndexInput(AzureIndexInput cloneInput) { _fileMutex = new Mutex(false, cloneInput._name); _fileMutex.WaitOne(); try { #if FULLDEBUG Debug.WriteLine(String.Format("Creating clone for {0}", cloneInput._name)); #endif _azureDirectory = cloneInput._azureDirectory; _blobContainer = cloneInput._blobContainer; _blob = cloneInput._blob; _indexInput = cloneInput._indexInput.Clone() as IndexInput; } catch (Exception) { // sometimes we get access denied on the 2nd stream...but not always. I haven't tracked it down yet // but this covers our tail until I do Debug.WriteLine(String.Format("Dagnabbit, falling back to memory clone for {0}", cloneInput._name)); } finally { _fileMutex.ReleaseMutex(); } }
public DataTable MyFullTextSearcher(String myQuery) { DataTable dochits = new DataTable(); dochits.Columns.Add(new DataColumn("MyUserName", typeof(String))); String result = CreateMyIndexDoc(); // Create the AzureDirectory for blob storage try { dochits.Rows.Add(result); String storageConnectionString = "DefaultEndpointsProtocol=https;" + "AccountName=classifiedfilestorage;" + "AccountKey=P/GSa/P8vmBicBT45Jgv7pmRHdYWYfJeTFJ963Q1aEHlJDZBCzYkTBcjH1JoGgl+k34x9koBW4lsgYhCym0JLQ=="; CloudStorageAccount cloudAccount = CloudStorageAccount.Parse(storageConnectionString); var cacheDirectory = new RAMDirectory(); AzureDirectory azureDirectory = new AzureDirectory(cloudAccount, "JobCat",cacheDirectory); // Create the IndexSearcher //IndexReader indexReader = DirectoryReader.Open(azureDirectory,true); IndexSearcher indexSearcher = new IndexSearcher(azureDirectory); // Create the QueryParser //MoreLikeThisQuery mltq = new MoreLikeThisQuery(myQuery, new String[] { "title", "jobCategory" }, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30)); //MoreLikeThis mlt = new MoreLikeThis(indexReader); //mlt.SetFieldNames(new String[] { "title", "jobCategory" }); //mlt.MinDocFreq = MoreLikeThis.DEFAULT_MIN_DOC_FREQ; //mlt.MinTermFreq = MoreLikeThis.DEFAULT_MIN_TERM_FREQ; //TextReader reader = new StringReader(myQuery); //Query query = mlt.Like(reader); MultiFieldQueryParser parser = new MultiFieldQueryParser(Lucene.Net.Util.Version.LUCENE_30, new String[] { "title", "jobCategory" }, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30)); // Create a query from the Parser Query query = parser.Parse(myQuery+"*"); // Retrieve matching hits var hits = indexSearcher.Search(query, 100); // Loop through the matching hits, retrieving the document for (int i = 0; i < hits.TotalHits; i++) { Document doc = indexSearcher.Doc(hits.ScoreDocs[i].Doc); dochits.Rows.Add(doc.Get("jobID")); } // indexReader.Dispose(); } catch (Exception e) { result = "" + e; dochits.Rows.Add(result); } return dochits; }
public void processRequest(string queryName, NameValueCollection htKeys) { string retVal = string.Empty; XmlDocument xdoc = new XmlDocument(); xdoc.LoadXml(new Utility().getConfigXML()); XmlNode xNode = xdoc.SelectSingleNode(string.Format("//blobdata[@name='default']")); string azureAccount = xNode.Attributes["account"].Value; string azureEndpoint = xNode.Attributes["endpoint"].Value; string azureSharedKey = xNode.Attributes["accountSharedKey"].Value; string blobStorage = xNode.Attributes["endpoint"].Value; xNode = xdoc.SelectSingleNode(string.Format("//fragmentData/Setting[@name='HandlerFragments']")); string fragmentLocation = xNode.Attributes["value"].Value; try { AzureBlobStorage abs = new AzureBlobStorage(azureAccount, blobStorage, azureSharedKey, "SharedKey"); azureResults ar = new azureResults(); // Get the page name and replace the .q extension with .xml if (!queryName.ToLower().EndsWith(".xml")) queryName += ".xml"; byte[] xmlFragment = abs.GetBlob(fragmentLocation, queryName, "", ref ar, ""); if (!ar.Succeeded) { NotifyError(new Exception(ar.StatusCode.ToString())); } else { xdoc = new XmlDocument(); System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding(); xdoc.LoadXml(enc.GetString(xmlFragment)); /* * http://azure-architect.com/portals/16/MOOPData.xsd */ XmlNode xn = xdoc.SelectSingleNode("//storedProcedure[1]"); string storedProcedureName = xn.Attributes["procedureName"].Value; string connectionStringName = xn.Attributes["connectionName"].Value; SqlCommand cmd = new SqlCommand(storedProcedureName, new SqlConnection(new Utility().ResolveDataConnection(connectionStringName))); cmd.CommandType = CommandType.StoredProcedure; XmlNodeList xnl = xdoc.SelectNodes("/MOOPData/luceneData/field"); Field.Store[] fieldStore = new Field.Store[xnl.Count]; Field.Index[] indexType = new Field.Index[xnl.Count]; string[] luceneName = new string[xnl.Count]; string[] dataName = new string[xnl.Count]; bool[] isIncludedInOlioSearchFlag = new bool[xnl.Count]; bool[] isKeyFieldFlag = new bool[xnl.Count]; string olioSearchFieldName = string.Empty; string azureContainerName = string.Empty; olioSearchFieldName = xdoc.SelectSingleNode("//MOOPData/luceneData/olioSearchFieldName[1]").InnerText; azureContainerName = xdoc.SelectSingleNode("//MOOPData/luceneData/azureContainer[1]").InnerText; for (int i = 0; i < xnl.Count; i++) { XmlNode node = xnl[i]; switch (node.Attributes["store"].Value.ToLower()) { case "compress": fieldStore[i] = Field.Store.COMPRESS; break; case "no": fieldStore[i] = Field.Store.NO; break; case "yes": fieldStore[i] = Field.Store.YES; break; default: fieldStore[i] = Field.Store.NO; break; } switch (node.Attributes["index"].Value.ToLower()) { case "analyzed": indexType[i] = Field.Index.ANALYZED; break; case "analyzed_no_norms": indexType[i] = Field.Index.ANALYZED_NO_NORMS; break; case "no": indexType[i] = Field.Index.NO; break; case "no_norms": indexType[i] = Field.Index.NOT_ANALYZED_NO_NORMS; break; case "not_analyzed": indexType[i] = Field.Index.NOT_ANALYZED; break; case "not_analyzed_no_norms": indexType[i] = Field.Index.NOT_ANALYZED_NO_NORMS; break; case "tokenized": indexType[i] = Field.Index.ANALYZED; break; case "un_tokenized": indexType[i] = Field.Index.NOT_ANALYZED; break; default: indexType[i] = Field.Index.NO; break; } dataName[i] = node.Attributes["dataName"].Value; luceneName[i] = node.Attributes["luceneName"].Value; isKeyFieldFlag[i] = node.Attributes["isKeyField"].Value == "true"; isKeyFieldFlag[i] = node.Attributes["isKeyField"].Value == "true"; isIncludedInOlioSearchFlag[i] = node.Attributes["isIncludedInOlioSearch"].Value == "true"; } xnl = xdoc.SelectNodes("//parameter"); foreach (XmlNode node in xnl) { string parameterName = node.Attributes["parameterName"].Value; string urlParameterName = node.Attributes["urlParameterName"].Value; string dataType = node.Attributes["dataType"].Value; string dataLength = node.Attributes["dataLength"].Value; string defaultValue = node.Attributes["defaultValue"].Value; if (!parameterName.StartsWith("@")) parameterName = "@" + parameterName; SqlParameter sp = new SqlParameter(); sp.ParameterName = parameterName; switch (dataType) { case "bigint": sp.SqlDbType = SqlDbType.BigInt; break; case "binary": sp.SqlDbType = SqlDbType.Binary; break; case "bit": sp.SqlDbType = SqlDbType.Bit; break; case "char": sp.SqlDbType = SqlDbType.Char; break; case "date": sp.SqlDbType = SqlDbType.Date; break; case "datetime": sp.SqlDbType = SqlDbType.DateTime; break; case "datetime2": sp.SqlDbType = SqlDbType.DateTime2; break; case "datetimeoffset": sp.SqlDbType = SqlDbType.DateTimeOffset; break; case "decimal": sp.SqlDbType = SqlDbType.Decimal; break; case "float": sp.SqlDbType = SqlDbType.Float; break; case "geography": sp.SqlDbType = SqlDbType.Structured; break; case "geometry": sp.SqlDbType = SqlDbType.Structured; break; case "hierarchyid": sp.SqlDbType = SqlDbType.Structured; break; case "image": sp.SqlDbType = SqlDbType.Image; break; case "int": sp.SqlDbType = SqlDbType.Int; break; case "money": sp.SqlDbType = SqlDbType.Money; break; case "nchar": sp.SqlDbType = SqlDbType.NChar; break; case "ntext": sp.SqlDbType = SqlDbType.NText; break; case "nvarchar": sp.SqlDbType = SqlDbType.NVarChar; break; case "real": sp.SqlDbType = SqlDbType.Real; break; case "smalldatetime": sp.SqlDbType = SqlDbType.SmallDateTime; break; case "smallint": sp.SqlDbType = SqlDbType.SmallInt; break; case "smallmoney": sp.SqlDbType = SqlDbType.SmallMoney; break; case "sql_variant": sp.SqlDbType = SqlDbType.Variant; break; case "text": sp.SqlDbType = SqlDbType.Text; break; case "time": sp.SqlDbType = SqlDbType.Time; break; case "timestamp": sp.SqlDbType = SqlDbType.Timestamp; break; case "tinyint": sp.SqlDbType = SqlDbType.TinyInt; break; case "uniqueidentifier": sp.SqlDbType = SqlDbType.UniqueIdentifier; break; case "varbinary": sp.SqlDbType = SqlDbType.VarBinary; break; case "varchar": sp.SqlDbType = SqlDbType.VarChar; break; case "xml": sp.SqlDbType = SqlDbType.Xml; break; default: sp.SqlDbType = SqlDbType.Variant; break; } switch (urlParameterName.ToLower()) { case "ipaddress": sp.Value = "127.0.0.1"; break; case "domainname": sp.Value = ""; break; default: if (htKeys[urlParameterName] != null) sp.Value = htKeys[urlParameterName]; else sp.Value = (defaultValue.ToLower() == "dbnull" ? DBNull.Value : (object)defaultValue); break; } cmd.Parameters.Add(sp); } cmd.Connection.Open(); SqlDataReader dr = cmd.ExecuteReader(); Microsoft.WindowsAzure.StorageCredentialsAccountAndKey scaak = new Microsoft.WindowsAzure.StorageCredentialsAccountAndKey(azureAccount, azureSharedKey); Microsoft.WindowsAzure.CloudStorageAccount csa = new Microsoft.WindowsAzure.CloudStorageAccount(scaak, false); AzureDirectory azureDirectory = new AzureDirectory(csa, azureContainerName, new RAMDirectory()); bool findexExists = false; try { findexExists = IndexReader.IndexExists(azureDirectory); if ((findexExists) && IndexWriter.IsLocked(azureDirectory)) azureDirectory.ClearLock("write.lock"); } catch (Exception e) { Trace.WriteLine(e.ToString()); return; } IndexWriter idxW = new IndexWriter(azureDirectory, new SnowballAnalyzer("English"), !findexExists, new IndexWriter.MaxFieldLength(1024)); idxW.SetRAMBufferSizeMB(10.0); idxW.SetUseCompoundFile(false); idxW.SetMaxMergeDocs(10000); idxW.SetMergeFactor(100); while (dr.Read()) { StringBuilder olioSearch = new StringBuilder(); Document doc = new Document(); for (int i = 0; i <= dataName.GetUpperBound(0); i++) { if (isKeyFieldFlag[i]) { NotifyCaller(string.Format("Processing {0}", dr[dataName[i]].ToString().ToLower())); idxW.DeleteDocuments(new Term(luceneName[i], dr[dataName[i]].ToString().ToLower())); doc.Add(new Field(luceneName[i], dr[dataName[i]].ToString().ToLower(), Field.Store.YES, Field.Index.NOT_ANALYZED)); } else try { doc.Add(new Field(luceneName[i], dr[dataName[i]].ToString(), fieldStore[i], indexType[i])); if (isIncludedInOlioSearchFlag[i]) olioSearch.AppendFormat("\r\n{0}", dr[dataName[i]].ToString()); } catch (Exception ex) { NotifyError(ex); } } if (olioSearch.ToString() != string.Empty && olioSearchFieldName != string.Empty) doc.Add(new Field(olioSearchFieldName, olioSearch.ToString(), Field.Store.NO, Field.Index.ANALYZED)); idxW.AddDocument(doc); } idxW.Commit(); idxW.Close(); } } catch (Exception ex) { MOOPFramework.FrameworkUtility u = new MOOPFramework.FrameworkUtility(new Utility().ResolveDataConnection("sqlAzureConnection")); u.LogData("localhost", "quoteSearchLoader", "testing", string.Empty, string.Empty, "", "QueryError", ex.ToString(), u.nvc2XML(htKeys)); //retVal = string.Format("<!-- {0} -->", ex.ToString()); NotifyError(new Exception("An error occured but it was logged for later review")); } finally { if (retVal == string.Empty) retVal = "<root />"; } }
static void Main(string[] args) { // get settings from azure settings or app.config CloudStorageAccount.SetConfigurationSettingPublisher((configName, configSetter) => { try { configSetter(RoleEnvironment.GetConfigurationSettingValue(configName)); } catch (Exception) { // for a console app, reading from App.config configSetter(System.Configuration.ConfigurationManager.AppSettings[configName]); } }); // default AzureDirectory stores cache in local temp folder AzureDirectory azureDirectory = new AzureDirectory(CloudStorageAccount.FromConfigurationSetting("blobStorage"), "TestCatalog"); bool findexExists = false; try { findexExists = IndexReader.IndexExists(azureDirectory); if ((findexExists) && IndexReader.IsLocked(azureDirectory)) azureDirectory.ClearLock("write.lock"); } catch (Exception e) { Console.WriteLine(e.ToString()); return; } IndexWriter indexWriter = new IndexWriter(azureDirectory, new Lucene.Net.Analysis.Standard.StandardAnalyzer(), !findexExists); indexWriter.SetRAMBufferSizeMB(10.0); indexWriter.SetUseCompoundFile(false); indexWriter.SetMaxMergeDocs(10000); indexWriter.SetMergeFactor(100); fExit = true; for (int iDoc = 0; iDoc < 100; iDoc++) { if (fExit) break; if (iDoc % 10 == 0) Console.WriteLine(iDoc); Document doc = new Document(); doc.Add(new Field("id", DateTime.Now.ToFileTimeUtc().ToString(), Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.NO)); doc.Add(new Field("Title", GeneratePhrase(10), Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.NO)); doc.Add(new Field("Body", GeneratePhrase(40), Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.NO)); indexWriter.AddDocument(doc); } Console.WriteLine("Total docs is {0}", indexWriter.DocCount()); indexWriter.Close(); IndexSearcher searcher; using (new AutoStopWatch("Creating searcher")) { searcher = new IndexSearcher(azureDirectory); // IndexReader.Open( } SearchForPhrase(searcher, "dog"); SearchForPhrase(searcher, _random.Next(32768).ToString()); SearchForPhrase(searcher, _random.Next(32768).ToString()); }
public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob) : base("Azure" + azuredirectory.ToString()) { _name = blob.Uri.Segments[blob.Uri.Segments.Length - 1]; #if FULLDEBUG Debug.WriteLine(String.Format("opening {0} ", _name)); #endif _fileMutex = BlobMutexManager.GrabMutex(_name); _fileMutex.WaitOne(); try { _azureDirectory = azuredirectory; _blobContainer = azuredirectory.BlobContainer; _blob = blob; var fileName = _name; var fFileNeeded = false; if (!CacheDirectory.FileExists(fileName)) { fFileNeeded = true; } else { long cachedLength = CacheDirectory.FileLength(fileName); string blobLengthMetadata; bool hasMetadataValue = blob.Metadata.TryGetValue("CachedLength", out blobLengthMetadata); long blobLength = blob.Properties.Length; if (hasMetadataValue) { long.TryParse(blobLengthMetadata, out blobLength); } string blobLastModifiedMetadata; long longLastModified = 0; DateTime blobLastModifiedUTC = blob.Properties.LastModified.Value.UtcDateTime; if (blob.Metadata.TryGetValue("CachedLastModified", out blobLastModifiedMetadata)) { if (long.TryParse(blobLastModifiedMetadata, out longLastModified)) { blobLastModifiedUTC = new DateTime(longLastModified).ToUniversalTime(); } } if (cachedLength != blobLength) { fFileNeeded = true; } else { // cachedLastModifiedUTC was not ouputting with a date (just time) and the time was always off long unixDate = _azureDirectory.FileModified(fileName); // DateTime start = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); var cachedLastModifiedUTC = new DateTime(unixDate).ToUniversalTime(); if (cachedLastModifiedUTC != blobLastModifiedUTC) { var timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC); if (timeSpan.TotalSeconds > 1) { fFileNeeded = true; } else { #if FULLDEBUG Debug.WriteLine(timeSpan.TotalSeconds); #endif // file not needed } } } } // if the file does not exist // or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage) if (fFileNeeded) { if (_azureDirectory.ShouldCompressFile(_name)) { InflateStream(fileName); } else { using (var fileStream = _azureDirectory.CreateOutput(fileName, IOContext.DEFAULT)) { // get the blob _blob.FetchAttributes(); long fileByteLength = _blob.Properties.Length; byte[] fileContent = new byte[fileByteLength]; for (int i = 0; i < fileByteLength; i++) { fileContent[i] = 0x20; } _blob.DownloadToByteArray(fileContent, 0); fileStream.WriteBytes(fileContent, (int)fileByteLength); fileStream.Flush(); Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length)); } } // and open it as an input _indexInput = CacheDirectory.OpenInput(fileName, IOContext.DEFAULT); } else { #if FULLDEBUG Debug.WriteLine(String.Format("Using cached file for {0}", _name)); #endif // open the file in read only mode _indexInput = CacheDirectory.OpenInput(fileName, IOContext.DEFAULT); } } finally { _fileMutex.ReleaseMutex(); } }
public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob) { _name = blob.Uri.Segments[blob.Uri.Segments.Length - 1]; #if FULLDEBUG Debug.WriteLine(String.Format("opening {0} ", _name)); #endif _fileMutex = BlobMutexManager.GrabMutex(_name); _fileMutex.WaitOne(); try { _azureDirectory = azuredirectory; _blobContainer = azuredirectory.BlobContainer; _blob = blob; var fileName = _name; var fFileNeeded = false; if (!CacheDirectory.FileExists(fileName)) { fFileNeeded = true; } else { long cachedLength = CacheDirectory.FileLength(fileName); string blobLengthMetadata; bool hasMetadataValue = blob.Metadata.TryGetValue("CachedLength", out blobLengthMetadata); long blobLength = blob.Properties.Length; if (hasMetadataValue) { long.TryParse(blobLengthMetadata, out blobLength); } string blobLastModifiedMetadata; long longLastModified = 0; DateTime blobLastModifiedUTC = blob.Properties.LastModified.Value.UtcDateTime; if (blob.Metadata.TryGetValue("CachedLastModified", out blobLastModifiedMetadata)) { if (long.TryParse(blobLastModifiedMetadata, out longLastModified)) { blobLastModifiedUTC = new DateTime(longLastModified).ToUniversalTime(); } } if (cachedLength != blobLength) { fFileNeeded = true; } else { // there seems to be an error of 1 tick which happens every once in a while // for now we will say that if they are within 1 tick of each other and same length var cachedLastModifiedUTC = new DateTime(CacheDirectory.FileModified(fileName), DateTimeKind.Local).ToUniversalTime(); if (cachedLastModifiedUTC != blobLastModifiedUTC) { var timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC); if (timeSpan.TotalSeconds > 1) { fFileNeeded = true; } else { #if FULLDEBUG Debug.WriteLine(timeSpan.TotalSeconds); #endif // file not needed } } } } // if the file does not exist // or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage) if (fFileNeeded) { if (_azureDirectory.ShouldCompressFile(_name)) { InflateStream(fileName); } else { using (var fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName)) { // get the blob _blob.DownloadToStream(fileStream); fileStream.Flush(); Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length)); } } // and open it as an input _indexInput = CacheDirectory.OpenInput(fileName); } else { #if FULLDEBUG Debug.WriteLine(String.Format("Using cached file for {0}", _name)); #endif // open the file in read only mode _indexInput = CacheDirectory.OpenInput(fileName); } } finally { _fileMutex.ReleaseMutex(); } }
public override void close() { _fileMutex.WaitOne(); try { _indexInput.close(); _indexInput = null; _azureDirectory = null; _blobContainer = null; _blob = null; GC.SuppressFinalize(this); } finally { _fileMutex.ReleaseMutex(); } }
static void Main() { // default AzureDirectory stores cache in local temp folder CloudStorageAccount cloudStorageAccount; CloudStorageAccount.TryParse(CloudConfigurationManager.GetSetting("blobStorage"), out cloudStorageAccount); //AzureDirectory azureDirectory = new AzureDirectory(cloudStorageAccount, "TestTest", new RAMDirectory()); //AzureDirectory azureDirectory = new AzureDirectory(cloudStorageAccount, "TestTest", FSDirectory.Open(@"c:\test")); var azureDirectory = new AzureDirectory(cloudStorageAccount, "TestTest" /* default is FSDirectory.Open(@"%temp%/AzureDirectory/TestTest"); */ ); IndexWriter indexWriter = null; while (indexWriter == null) { try { var config = new IndexWriterConfig(org.apache.lucene.util.Version.LUCENE_CURRENT, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT)); indexWriter = new IndexWriter(azureDirectory, config); } catch (LockObtainFailedException) { Console.WriteLine("Lock is taken, waiting for timeout..."); Thread.Sleep(1000); } } Console.WriteLine("IndexWriter lock obtained, this process has exclusive write access to index"); //indexWriter.setRAMBufferSizeMB(10.0); //indexWriter.SetUseCompoundFile(false); //indexWriter.SetMaxMergeDocs(10000); //indexWriter.SetMergeFactor(100); for (int iDoc = 0; iDoc < 10000; iDoc++) { if (iDoc % 10 == 0) Console.WriteLine(iDoc); var doc = new Document(); doc.add(new TextField("id", DateTime.Now.ToFileTimeUtc().ToString(CultureInfo.InvariantCulture), Field.Store.YES)); doc.add(new TextField("Title", GeneratePhrase(10), Field.Store.YES)); doc.add(new TextField("Body", GeneratePhrase(40), Field.Store.YES)); indexWriter.addDocument(doc); } Console.WriteLine("Total docs is {0}", indexWriter.numDocs()); Console.Write("Flushing and disposing writer..."); // Potentially Expensive: this ensures that all writes are commited to blob storage indexWriter.commit(); indexWriter.close(); Console.WriteLine("done"); Console.WriteLine("Hit Key to search again"); Console.ReadKey(); IndexSearcher searcher; using (new AutoStopWatch("Creating searcher")) { searcher = new IndexSearcher(DirectoryReader.open(azureDirectory)); } SearchForPhrase(searcher, "dog"); SearchForPhrase(searcher, Random.Next(32768).ToString(CultureInfo.InvariantCulture)); SearchForPhrase(searcher, Random.Next(32768).ToString(CultureInfo.InvariantCulture)); Console.WriteLine("Hit a key to dispose and exit"); Console.ReadKey(); }
public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob) { _name = blob.Uri.Segments[blob.Uri.Segments.Length - 1]; #if FULLDEBUG Debug.WriteLine(String.Format("opening {0} ", _name)); #endif _fileMutex = BlobMutexManager.GrabMutex(_name); _fileMutex.WaitOne(); try { _azureDirectory = azuredirectory; _blobContainer = azuredirectory.BlobContainer; _blob = blob; string fileName = _name; bool fFileNeeded = false; if (!CacheDirectory.FileExists(fileName)) { fFileNeeded = true; } else { long cachedLength = CacheDirectory.FileLength(fileName); long blobLength = blob.Properties.Length; long.TryParse(blob.Metadata["CachedLength"], out blobLength); long longLastModified = 0; DateTime blobLastModifiedUTC = blob.Properties.LastModified.Value.UtcDateTime; //if (long.TryParse(blob.Metadata["CachedLastModified"], out longLastModified)) // blobLastModifiedUTC = new DateTime(longLastModified, DateTimeKind.Local).ToUniversalTime(); if (cachedLength != blobLength) fFileNeeded = true; else { // there seems to be an error of 1 tick which happens every once in a while // for now we will say that if they are within 1 tick of each other and same length DateTime cachedLastModifiedUTC = DateTime.MinValue; if (CacheDirectory is RAMDirectory) { cachedLastModifiedUTC = new DateTime(CacheDirectory.FileModified(fileName) * TimeSpan.TicksPerMillisecond, DateTimeKind.Local).ToUniversalTime(); } else if (CacheDirectory is FSDirectory) { cachedLastModifiedUTC = new DateTime(CacheDirectory.FileModified(fileName) * TimeSpan.TicksPerMillisecond, DateTimeKind.Utc).ToUniversalTime().AddYears(1969); } if (cachedLastModifiedUTC != blobLastModifiedUTC) { TimeSpan timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC); if (timeSpan.TotalSeconds > 1) fFileNeeded = true; else { #if FULLDEBUG Debug.WriteLine(timeSpan.TotalSeconds); #endif // file not needed } } } } // if the file does not exist // or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage) if (fFileNeeded) { #if COMPRESSBLOBS if (_azureDirectory.ShouldCompressFile(_name)) { // then we will get it fresh into local deflatedName // StreamOutput deflatedStream = new StreamOutput(CacheDirectory.CreateOutput(deflatedName)); MemoryStream deflatedStream = new MemoryStream(); // get the deflated blob _blob.DownloadToStream(deflatedStream); Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, deflatedStream.Length)); // seek back to begininng deflatedStream.Seek(0, SeekOrigin.Begin); // open output file for uncompressed contents StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName); // create decompressor DeflateStream decompressor = new DeflateStream(deflatedStream, CompressionMode.Decompress); byte[] bytes = new byte[65535]; int nRead = 0; do { nRead = decompressor.Read(bytes, 0, 65535); if (nRead > 0) fileStream.Write(bytes, 0, nRead); } while (nRead == 65535); decompressor.Close(); // this should close the deflatedFileStream too fileStream.Close(); } else #endif { StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName); // get the blob _blob.DownloadToStream(fileStream); fileStream.Flush(); Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length)); fileStream.Close(); } // and open it as an input _indexInput = CacheDirectory.OpenInput(fileName); } else { #if FULLDEBUG Debug.WriteLine(String.Format("Using cached file for {0}", _name)); #endif // open the file in read only mode _indexInput = CacheDirectory.OpenInput(fileName); } } finally { _fileMutex.ReleaseMutex(); } }
public void optimizeIndex(string azureContainerName) { XmlDocument xdoc = new XmlDocument(); xdoc.LoadXml(new Utility().getConfigXML()); XmlNode xNode = xdoc.SelectSingleNode(string.Format("//blobdata[@name='default']")); string azureAccount = xNode.Attributes["account"].Value; string azureEndpoint = xNode.Attributes["endpoint"].Value; string azureSharedKey = xNode.Attributes["accountSharedKey"].Value; string blobStorage = xNode.Attributes["endpoint"].Value; xNode = xdoc.SelectSingleNode(string.Format("//fragmentData/Setting[@name='HandlerFragments']")); string fragmentLocation = xNode.Attributes["value"].Value; Microsoft.WindowsAzure.StorageCredentialsAccountAndKey scaak = new Microsoft.WindowsAzure.StorageCredentialsAccountAndKey(azureAccount, azureSharedKey); Microsoft.WindowsAzure.CloudStorageAccount csa = new Microsoft.WindowsAzure.CloudStorageAccount(scaak, false); AzureDirectory azureDirectory = new AzureDirectory(csa, azureContainerName, new RAMDirectory()); bool findexExists = false; try { findexExists = IndexReader.IndexExists(azureDirectory); if ((findexExists) && IndexWriter.IsLocked(azureDirectory)) azureDirectory.ClearLock("write.lock"); } catch (Exception e) { Trace.WriteLine(e.ToString()); return; } IndexWriter idxW = new IndexWriter(azureDirectory, new SnowballAnalyzer("English"), !findexExists, new IndexWriter.MaxFieldLength(1024)); idxW.SetRAMBufferSizeMB(10.0); idxW.SetUseCompoundFile(false); idxW.SetMaxMergeDocs(10000); idxW.SetMergeFactor(100); idxW.Optimize(); }
// Saten public String CreateMyIndexDoc() { String status = ""; try { if (dbConnection.State.ToString() == "Closed") { dbConnection.Open(); } String storageConnectionString = "DefaultEndpointsProtocol=https;" + "AccountName=classifiedfilestorage;" + "AccountKey=P/GSa/P8vmBicBT45Jgv7pmRHdYWYfJeTFJ963Q1aEHlJDZBCzYkTBcjH1JoGgl+k34x9koBW4lsgYhCym0JLQ=="; CloudStorageAccount cloudAccount = CloudStorageAccount.Parse(storageConnectionString); var cacheDirectory = new RAMDirectory(); AzureDirectory azureDirectory = new AzureDirectory(cloudAccount, "JobCat",cacheDirectory); bool findexExists = IndexReader.IndexExists(azureDirectory); IndexWriter indexWriter = null; while (indexWriter == null) { try { indexWriter = new IndexWriter(azureDirectory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30),findexExists, new Lucene.Net.Index.IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH)); } catch (LockObtainFailedException) { Thread.Sleep(1000); } }; indexWriter.SetRAMBufferSizeMB(10.0); String query = "SELECT jobID,title,jobCategory from jobs"; SqlCommand command = new SqlCommand(query, dbConnection); SqlDataReader reader = command.ExecuteReader(); if (reader.HasRows) { while (reader.Read()) { Document docx = new Document(); docx.Add(new Field("jobID", reader["jobID"].ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); docx.Add(new Field("title", reader["title"].ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); docx.Add(new Field("jobCategory", reader["jobCategory"].ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); indexWriter.AddDocument(docx); } } indexWriter.Close(); dbConnection.Close(); status = "Success"; } catch (Exception e) { status = "" + e; } return status; }
private IndexSearcher CreateIndexSearcher() { CloudStorageAccount cloudStorageAccount; CloudStorageAccount.TryParse(CloudConfigurationManager.GetSetting("blobStorage"), out cloudStorageAccount); var cacheDirectory = new RAMDirectory(); var azureDirectory = new AzureDirectory(cloudStorageAccount, "FarmCatalog", cacheDirectory); return new IndexSearcher(azureDirectory, true); }