private void InitializeEnvironmentCacheAndSearchProviders() { CfTrace.Current.Information(TraceCode.AppBuildSearchIndex, "Started {0:MMM dd HH.mm.ss}", DateTime.Now); if (!RoleEnvironment.IsAvailable) { CfCacheIndex.Initialize(); CfPerfCache.Initialize(); new SearchManager().BuildIndex(null); var directoryPath = AppDomain.CurrentDomain.BaseDirectory + @"App_Data\"; var dir = Lucene.Net.Store.FSDirectory.Open(new System.IO.DirectoryInfo(directoryPath)); _siteSearchEngine = new CfLuceneIndexSearcher(dir); } else { CfCacheIndex.Initialize(new Level2MemcachedCacheIndex()); CfPerfCache.Initialize(new Level2MemcachedPerfCache()); var connectionString = ConfigurationManager.ConnectionStrings["CfCloudStorage"].ConnectionString; var cloudStorage = CloudStorageAccount.Parse(connectionString); var dir = new Lucene.Net.Store.Azure.AzureDirectory(cloudStorage, "SearchCatalog", new RAMDirectory()); _siteSearchEngine = new CfLuceneIndexSearcher(dir); lastIndexRefresh = DateTime.UtcNow; } CfTrace.Current.Information(TraceCode.AppBuildSearchIndex, "Ended {0:MMM dd HH.mm.ss}", DateTime.Now); }
/// <summary> /// Had to move azure role initialization here /// See http://social.msdn.microsoft.com/Forums/en-US/windowsazuredevelopment/thread/10d042da-50b1-4930-b0c0-aff22e4144f9 /// and http://social.msdn.microsoft.com/Forums/en-US/windowsazuredevelopment/thread/ab6d56dc-154d-4aba-8bde-2b7f7df121c1/#89264b8c-7e25-455a-8fd6-20f547ab545b /// </summary> protected void Application_BeginRequest() { //-- Search index is updated every hour + time taken to build cache and index on the cache server var indexNeedsUpdating = lastIndexRefresh.HasValue && lastIndexRefresh > DateTime.UtcNow.AddMinutes(65); if (_initialized && !indexNeedsUpdating) { return; } lock (_gate) { if (!_initialized) { InitializeEnvironmentCacheAndSearchProviders(); _initialized = true; } else if (indexNeedsUpdating) { _siteSearchEngine.Dispose(); var connectionString = ConfigurationManager.ConnectionStrings["CfCloudStorage"].ConnectionString; var cloudStorage = CloudStorageAccount.Parse(connectionString); var dir = new Lucene.Net.Store.Azure.AzureDirectory(cloudStorage, "SearchCatalog", new RAMDirectory()); _siteSearchEngine = new CfLuceneIndexSearcher(dir); Mail.MailMan.SendAppEvent(TraceCode.AppBuildSearchIndex, "Search index reloaded", "", Stgs.JskID, "*****@*****.**", false); lastIndexRefresh = DateTime.UtcNow; } } }
/// <summary> /// Had to move azure role initialization here /// See http://social.msdn.microsoft.com/Forums/en-US/windowsazuredevelopment/thread/10d042da-50b1-4930-b0c0-aff22e4144f9 /// and http://social.msdn.microsoft.com/Forums/en-US/windowsazuredevelopment/thread/ab6d56dc-154d-4aba-8bde-2b7f7df121c1/#89264b8c-7e25-455a-8fd6-20f547ab545b /// </summary> protected void Application_BeginRequest() { //-- Search index is updated every hour + time taken to build cache and index on the cache server var indexNeedsUpdating = lastIndexRefresh.HasValue && lastIndexRefresh > DateTime.UtcNow.AddMinutes(65); if (_initialized && !indexNeedsUpdating) { return; } lock (_gate) { if (!_initialized) { InitializeEnvironmentCacheAndSearchProviders(); _initialized = true; } else if (indexNeedsUpdating) { _siteSearchEngine.Dispose(); var connectionString = ConfigurationManager.ConnectionStrings["CfCloudStorage"].ConnectionString; var cloudStorage = CloudStorageAccount.Parse(connectionString); var dir = new Lucene.Net.Store.Azure.AzureDirectory(cloudStorage, "SearchCatalog", new RAMDirectory()); _siteSearchEngine = new CfLuceneIndexSearcher(dir); Mail.MailMan.SendAppEvent(TraceCode.AppBuildSearchIndex, "Search index reloaded", "", Stgs.JskID, "*****@*****.**", false); lastIndexRefresh = DateTime.UtcNow; } } }
public async Task <LuceneIndexAndMetadata> GetReaderInternal(CancellationToken cancellationToken) { var resultObject = new LuceneIndexAndMetadata(); //Azure configuration var accountSAS = new Microsoft.Azure.Storage.Auth.StorageCredentials(_azureLuceneConfiguration.SASToken); var accountWithSAS = new Microsoft.Azure.Storage.CloudStorageAccount(accountSAS, _azureLuceneConfiguration.AzureStorageAccountName, endpointSuffix: null, useHttps: true); _logger.LogInformation("_azureLuceneConfiguration.TempDirectory = {0}", _azureLuceneConfiguration.TempDirectory); var tempLocation = _azureLuceneConfiguration.TempDirectory ?? "temp"; _logger.LogDebug("Lucene IndexReader is located in {0} azure storage account (container '{1}')" , _azureLuceneConfiguration.AzureStorageAccountName , _azureLuceneConfiguration.Container); var azureDirectory = new Lucene.Net.Store.Azure.AzureDirectory(accountWithSAS, tempLocation, containerName: _azureLuceneConfiguration.Container); //ensure RAMDirectory azureDirectory.CacheDirectory = new Lucene.Net.Store.RAMDirectory(); var reader = DirectoryReader.Open(azureDirectory); _logger.LogDebug("Lucene IndexReader is acquired."); resultObject.Index = reader; using (var dbConnection = await _SQLservice.GetConnection(cancellationToken)) { //we need last sync point only if it is not full rebuild var dbCommand = @"SELECT TOP 1 LastSyncPoint FROM [dbo].[FTS_Config]"; var cmd = new SqlCommand(dbCommand, dbConnection); try { var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken); var lastSyncPointNullable = untyped as DateTimeOffset?; if (lastSyncPointNullable.HasValue) { resultObject.LastIndexOffset = lastSyncPointNullable.Value; } _logger.LogDebug("Last sync point is {0}", lastSyncPointNullable.HasValue ? lastSyncPointNullable.Value.ToString() : "'never'"); } catch (Exception ex) { _logger.LogError(ex, "unexpected failure to acquire LastSyncPoint from database"); throw; } } return(resultObject); }
public void Can_Clear_And_Rewrite_Search_Index_ToCloudStorage() { var dir = new Lucene.Net.Store.Azure.AzureDirectory(CfCloudStorage, "SearchCatalog", new Lucene.Net.Store.RAMDirectory()); foreach (var b in dir.BlobContainer.ListBlobs()) { dir.BlobContainer.GetBlobReference(b.Uri.ToString()).Delete(); } new cf.Content.Search.SearchManager().BuildIndex(dir); var expected = blobRepo.GetBlobWithProperties("searchcatalog", "_0.cfs"); Assert.IsTrue(expected.Properties.LastModifiedUtc > DateTime.UtcNow.AddMinutes(-1)); }
public void Can_Clear_And_Rewrite_Search_Index_ToCloudStorage() { var dir = new Lucene.Net.Store.Azure.AzureDirectory(CfCloudStorage, "SearchCatalog", new Lucene.Net.Store.RAMDirectory()); foreach (var b in dir.BlobContainer.ListBlobs()) { dir.BlobContainer.GetBlobReference(b.Uri.ToString()).Delete(); } new cf.Content.Search.SearchManager().BuildIndex(dir); var expected = blobRepo.GetBlobWithProperties("searchcatalog", "_0.cfs"); Assert.IsTrue(expected.Properties.LastModifiedUtc > DateTime.UtcNow.AddMinutes(-1)); }
/// <summary> /// /// </summary> private void RebuildLuceneIndex() { try { var dir = new Lucene.Net.Store.Azure.AzureDirectory(CfCloudStorage, "SearchCatalog", new Lucene.Net.Store.RAMDirectory()); //-- The magic code... delete all the blobs before rebuilding the index foreach (var b in dir.BlobContainer.ListBlobs()) { dir.BlobContainer.GetBlobReference(b.Uri.ToString()).Delete(); } new cf.Content.Search.SearchManager().BuildIndex(dir); } catch (Exception ex) { CfTrace.Error(ex); } }
private void InitializeEnvironmentCacheAndSearchProviders() { CfTrace.Current.Information(TraceCode.AppBuildSearchIndex, "Started {0:MMM dd HH.mm.ss}", DateTime.Now); if (!RoleEnvironment.IsAvailable) { CfCacheIndex.Initialize(); CfPerfCache.Initialize(); new SearchManager().BuildIndex(null); var directoryPath = AppDomain.CurrentDomain.BaseDirectory + @"App_Data\"; var dir = Lucene.Net.Store.FSDirectory.Open(new System.IO.DirectoryInfo(directoryPath)); _siteSearchEngine = new CfLuceneIndexSearcher(dir); } else { CfCacheIndex.Initialize(new Level2MemcachedCacheIndex()); CfPerfCache.Initialize(new Level2MemcachedPerfCache()); var connectionString = ConfigurationManager.ConnectionStrings["CfCloudStorage"].ConnectionString; var cloudStorage = CloudStorageAccount.Parse(connectionString); var dir = new Lucene.Net.Store.Azure.AzureDirectory(cloudStorage, "SearchCatalog", new RAMDirectory()); _siteSearchEngine = new CfLuceneIndexSearcher(dir); lastIndexRefresh = DateTime.UtcNow; } CfTrace.Current.Information(TraceCode.AppBuildSearchIndex, "Ended {0:MMM dd HH.mm.ss}", DateTime.Now); }
public async Task <RebuildIndicesResponse> Handle(RebuildIndicesRequest request, CancellationToken cancellationToken) { _logger.LogDebug("RebuildIndicesResponseHandler started."); cancellationToken.ThrowIfCancellationRequested(); IndexWriter writer = null; Lucene.Net.Store.Azure.AzureDirectory azureDirectory = null; DateTimeOffset lastSyncPoint = DateTimeOffset.MinValue; DateTimeOffset currentSyncPoint = DateTimeOffset.Now; int? updatedCount = null; int? deletedCount = null; try { // Ensures index backwards compatibility var AppLuceneVersion = LuceneVersion.LUCENE_48; //Azure configuration var accountSAS = new Microsoft.Azure.Storage.Auth.StorageCredentials(AzureLuceneConfiguration.SASToken); var accountWithSAS = new Microsoft.Azure.Storage.CloudStorageAccount(accountSAS, AzureLuceneConfiguration.AzureStorageAccountName, endpointSuffix: null, useHttps: true); var tempLocation = AzureLuceneConfiguration.TempDirectory ?? "temp"; _logger.LogTrace("tempLocation: {0}", tempLocation); azureDirectory = new Lucene.Net.Store.Azure.AzureDirectory(accountWithSAS, tempLocation, containerName: AzureLuceneConfiguration.Container); //ensure RAMDirectory azureDirectory.CacheDirectory = new RAMDirectory(); //create an analyzer to process the text var analyzer = new StandardAnalyzer(AppLuceneVersion); //create an index writer var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer); writer = new IndexWriter(azureDirectory, indexConfig); //used to be dir _logger.LogTrace("IndexWriter is initialized"); if (request.FullRebuild) { _logger.LogInformation("Full Rebuild is requested. Deleting indices"); writer.DeleteAll(); writer.Commit(); _logger.LogTrace("Full Rebuild is committed."); } using (var dbConnection = await _SQLservice.GetConnection(cancellationToken)) { SqlCommand cmd; if (!request.FullRebuild) { //we need last sync point only if it is not full rebuild var dbCommand = @"SELECT TOP 1 LastSyncPoint FROM [dbo].[FTS_Config]"; cmd = new SqlCommand(dbCommand, dbConnection); try { var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken); var lastSyncPointNullable = untyped as DateTimeOffset?; if (lastSyncPointNullable.HasValue) { lastSyncPoint = lastSyncPointNullable.Value; } _logger.LogDebug("Last sync point is {0}", lastSyncPointNullable.HasValue ? lastSyncPointNullable.Value.ToString() : "'never'"); } catch (Exception ex) { _logger.LogError(ex, "unexpected failure to acquire LastSyncPoint from database"); throw; } } else { lastSyncPoint = DateTimeOffset.MinValue; } //determine number of records that will need to be processed var dbCountCommand = @"SELECT COUNT(Id) from [dbo].[Test_Data] WHERE UpdatedAt >= @lastSyncPoint AND UpdatedAt < @currentSyncPoint AND DeletedAt IS NULL"; cmd = new SqlCommand(dbCountCommand, dbConnection); cmd.Parameters.Add("@lastSyncPoint", System.Data.SqlDbType.DateTimeOffset); cmd.Parameters["@lastSyncPoint"].Value = lastSyncPoint; cmd.Parameters.Add("@currentSyncPoint", System.Data.SqlDbType.DateTimeOffset); cmd.Parameters["@currentSyncPoint"].Value = currentSyncPoint; try { var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken); updatedCount = untyped as int?; _logger.LogDebug("Expected number of updated documents {0}", updatedCount.HasValue ? updatedCount.Value.ToString() : "'none'"); } catch (Exception ex) { _logger.LogError(ex, "unexpected failure to acquire number of documents to be updated from database"); throw; } //working on deleted documents if (!request.FullRebuild) { //also need to remove "Deleted" documents. Only if not full rebuild of indices var dbDeletedCountCommand = @"SELECT COUNT(Id) from [dbo].[Test_Data] WHERE DeletedAt >= @lastSyncPoint AND DeletedAt<=@currentSyncPoint AND DeletedAt IS NOT NULL"; cmd = new SqlCommand(dbDeletedCountCommand, dbConnection); cmd.Parameters.Add("@lastSyncPoint", System.Data.SqlDbType.DateTimeOffset); cmd.Parameters["@lastSyncPoint"].Value = lastSyncPoint; cmd.Parameters.Add("@currentSyncPoint", System.Data.SqlDbType.DateTimeOffset); cmd.Parameters["@currentSyncPoint"].Value = currentSyncPoint; try { var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken); deletedCount = untyped as int?; _logger.LogDebug("Expected number of deleted documents {0}", deletedCount.HasValue ? updatedCount.Value.ToString() : "'none'"); } catch (Exception ex) { _logger.LogError(ex, "unexpected failure to acquire 'number of documents to be delete from indicies' from database"); throw; } } } var atLeastOneUpdate = false; if (updatedCount.HasValue && updatedCount.Value > 0) { _logger.LogDebug("Expected number of updated documents: {0}", updatedCount.Value); //Start updating 'Updated records' await UpdateIndicesWithAddedDocuments(lastSyncPoint, currentSyncPoint, updatedCount.Value, writer, cancellationToken); atLeastOneUpdate = true; } else { _logger.LogDebug("Expected number of updated documents: none "); } if (deletedCount.HasValue && deletedCount.Value > 0) { _logger.LogDebug("Expected number of deleted documents: {0}", deletedCount.Value); await UpdateIndicesWithDeletedDocuments(lastSyncPoint, currentSyncPoint, deletedCount.Value, writer, cancellationToken); atLeastOneUpdate = true; } else { _logger.LogDebug("Expected number of updated documents: none "); } if (atLeastOneUpdate) { _logger.LogDebug("Expected number of updated documents: none "); _luceneReaderService.Evict(); writer.Flush(triggerMerge: true, applyAllDeletes: true); _logger.LogInformation("Indexes are updated"); } //update LastSyncPoint using (var dbConnection = await _SQLservice.GetConnection(cancellationToken)) { var dbCommand = @"UPDATE [dbo].[FTS_Config] SET LastSyncPoint = @currentSyncPoint"; var cmd = new SqlCommand(dbCommand, dbConnection); cmd.Parameters.Add("@currentSyncPoint", System.Data.SqlDbType.DateTimeOffset); cmd.Parameters["@currentSyncPoint"].Value = currentSyncPoint; try { await _SQLservice.ExecuteNonQueryWithRetryAsync(cmd, cancellationToken); _logger.LogDebug("Last sync point is set to {0}", currentSyncPoint); } catch (Exception ex) { _logger.LogError(ex, "unexpected failure to update LastSyncPoint in database"); throw; } } var result = new RebuildIndicesResponse { IsValid = true, Success = true, NumberOfUpdates = updatedCount, NumberOfDeletes = deletedCount, CurrentSyncPoint = currentSyncPoint }; return(result); } catch (LockObtainFailedException) { var result = new RebuildIndicesResponse(); result.IsValid = false; result.Errors = new List <string>(); result.Errors.Add("Failed to lock full text search index file. Probaly there is another job is running. Please try again later."); return(result); } catch (Exception ex) { var result = new RebuildIndicesResponse(); result.IsValid = false; result.Errors = new List <string>(); result.Errors.Add("Unexpected error occured: " + ex.Message); return(result); } finally { if (writer != null) { writer.Dispose(); } if (azureDirectory != null) { azureDirectory.Dispose(); } } }