private void InitializeEnvironmentCacheAndSearchProviders()
{
CfTrace.Current.Information(TraceCode.AppBuildSearchIndex, "Started {0:MMM dd HH.mm.ss}", DateTime.Now);
if (!RoleEnvironment.IsAvailable)
{
CfCacheIndex.Initialize();
CfPerfCache.Initialize();
new SearchManager().BuildIndex(null);
var directoryPath = AppDomain.CurrentDomain.BaseDirectory + @"App_Data\";
var dir = Lucene.Net.Store.FSDirectory.Open(new System.IO.DirectoryInfo(directoryPath));
_siteSearchEngine = new CfLuceneIndexSearcher(dir);
}
else
{
CfCacheIndex.Initialize(new Level2MemcachedCacheIndex());
CfPerfCache.Initialize(new Level2MemcachedPerfCache());
var connectionString = ConfigurationManager.ConnectionStrings["CfCloudStorage"].ConnectionString;
var cloudStorage = CloudStorageAccount.Parse(connectionString);
var dir = new Lucene.Net.Store.Azure.AzureDirectory(cloudStorage, "SearchCatalog", new RAMDirectory());
_siteSearchEngine = new CfLuceneIndexSearcher(dir);
lastIndexRefresh = DateTime.UtcNow;
}
CfTrace.Current.Information(TraceCode.AppBuildSearchIndex, "Ended {0:MMM dd HH.mm.ss}", DateTime.Now);
}
/// <summary>
/// Had to move azure role initialization here
/// See http://social.msdn.microsoft.com/Forums/en-US/windowsazuredevelopment/thread/10d042da-50b1-4930-b0c0-aff22e4144f9
/// and http://social.msdn.microsoft.com/Forums/en-US/windowsazuredevelopment/thread/ab6d56dc-154d-4aba-8bde-2b7f7df121c1/#89264b8c-7e25-455a-8fd6-20f547ab545b
/// </summary>
protected void Application_BeginRequest()
{
//-- Search index is updated every hour + time taken to build cache and index on the cache server
var indexNeedsUpdating = lastIndexRefresh.HasValue && lastIndexRefresh > DateTime.UtcNow.AddMinutes(65);
if (_initialized && !indexNeedsUpdating)
{
return;
}
lock (_gate)
{
if (!_initialized)
{
InitializeEnvironmentCacheAndSearchProviders();
_initialized = true;
}
else if (indexNeedsUpdating)
{
_siteSearchEngine.Dispose();
var connectionString = ConfigurationManager.ConnectionStrings["CfCloudStorage"].ConnectionString;
var cloudStorage = CloudStorageAccount.Parse(connectionString);
var dir = new Lucene.Net.Store.Azure.AzureDirectory(cloudStorage, "SearchCatalog", new RAMDirectory());
_siteSearchEngine = new CfLuceneIndexSearcher(dir);
Mail.MailMan.SendAppEvent(TraceCode.AppBuildSearchIndex, "Search index reloaded", "", Stgs.JskID, "*****@*****.**", false);
lastIndexRefresh = DateTime.UtcNow;
}
}
}
/// <summary>
/// Had to move azure role initialization here
/// See http://social.msdn.microsoft.com/Forums/en-US/windowsazuredevelopment/thread/10d042da-50b1-4930-b0c0-aff22e4144f9
/// and http://social.msdn.microsoft.com/Forums/en-US/windowsazuredevelopment/thread/ab6d56dc-154d-4aba-8bde-2b7f7df121c1/#89264b8c-7e25-455a-8fd6-20f547ab545b
/// </summary>
protected void Application_BeginRequest()
{
//-- Search index is updated every hour + time taken to build cache and index on the cache server
var indexNeedsUpdating = lastIndexRefresh.HasValue && lastIndexRefresh > DateTime.UtcNow.AddMinutes(65);
if (_initialized && !indexNeedsUpdating)
{
return;
}
lock (_gate)
{
if (!_initialized)
{
InitializeEnvironmentCacheAndSearchProviders();
_initialized = true;
}
else if (indexNeedsUpdating)
{
_siteSearchEngine.Dispose();
var connectionString = ConfigurationManager.ConnectionStrings["CfCloudStorage"].ConnectionString;
var cloudStorage = CloudStorageAccount.Parse(connectionString);
var dir = new Lucene.Net.Store.Azure.AzureDirectory(cloudStorage, "SearchCatalog", new RAMDirectory());
_siteSearchEngine = new CfLuceneIndexSearcher(dir);
Mail.MailMan.SendAppEvent(TraceCode.AppBuildSearchIndex, "Search index reloaded", "", Stgs.JskID, "*****@*****.**", false);
lastIndexRefresh = DateTime.UtcNow;
}
}
}
public async Task <LuceneIndexAndMetadata> GetReaderInternal(CancellationToken cancellationToken)
{
var resultObject = new LuceneIndexAndMetadata();
//Azure configuration
var accountSAS = new Microsoft.Azure.Storage.Auth.StorageCredentials(_azureLuceneConfiguration.SASToken);
var accountWithSAS = new Microsoft.Azure.Storage.CloudStorageAccount(accountSAS, _azureLuceneConfiguration.AzureStorageAccountName, endpointSuffix: null, useHttps: true);
_logger.LogInformation("_azureLuceneConfiguration.TempDirectory = {0}", _azureLuceneConfiguration.TempDirectory);
var tempLocation = _azureLuceneConfiguration.TempDirectory ?? "temp";
_logger.LogDebug("Lucene IndexReader is located in {0} azure storage account (container '{1}')"
, _azureLuceneConfiguration.AzureStorageAccountName
, _azureLuceneConfiguration.Container);
var azureDirectory = new Lucene.Net.Store.Azure.AzureDirectory(accountWithSAS, tempLocation, containerName: _azureLuceneConfiguration.Container);
//ensure RAMDirectory
azureDirectory.CacheDirectory = new Lucene.Net.Store.RAMDirectory();
var reader = DirectoryReader.Open(azureDirectory);
_logger.LogDebug("Lucene IndexReader is acquired.");
resultObject.Index = reader;
using (var dbConnection = await _SQLservice.GetConnection(cancellationToken))
{
//we need last sync point only if it is not full rebuild
var dbCommand = @"SELECT TOP 1 LastSyncPoint FROM [dbo].[FTS_Config]";
var cmd = new SqlCommand(dbCommand, dbConnection);
try
{
var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken);
var lastSyncPointNullable = untyped as DateTimeOffset?;
if (lastSyncPointNullable.HasValue)
{
resultObject.LastIndexOffset = lastSyncPointNullable.Value;
}
_logger.LogDebug("Last sync point is {0}", lastSyncPointNullable.HasValue ? lastSyncPointNullable.Value.ToString() : "'never'");
}
catch (Exception ex)
{
_logger.LogError(ex, "unexpected failure to acquire LastSyncPoint from database");
throw;
}
}
return(resultObject);
}
public void Can_Clear_And_Rewrite_Search_Index_ToCloudStorage()
{
var dir = new Lucene.Net.Store.Azure.AzureDirectory(CfCloudStorage, "SearchCatalog", new Lucene.Net.Store.RAMDirectory());
foreach (var b in dir.BlobContainer.ListBlobs())
{
dir.BlobContainer.GetBlobReference(b.Uri.ToString()).Delete();
}
new cf.Content.Search.SearchManager().BuildIndex(dir);
var expected = blobRepo.GetBlobWithProperties("searchcatalog", "_0.cfs");
Assert.IsTrue(expected.Properties.LastModifiedUtc > DateTime.UtcNow.AddMinutes(-1));
}
public void Can_Clear_And_Rewrite_Search_Index_ToCloudStorage()
{
var dir = new Lucene.Net.Store.Azure.AzureDirectory(CfCloudStorage, "SearchCatalog", new Lucene.Net.Store.RAMDirectory());
foreach (var b in dir.BlobContainer.ListBlobs())
{
dir.BlobContainer.GetBlobReference(b.Uri.ToString()).Delete();
}
new cf.Content.Search.SearchManager().BuildIndex(dir);
var expected = blobRepo.GetBlobWithProperties("searchcatalog", "_0.cfs");
Assert.IsTrue(expected.Properties.LastModifiedUtc > DateTime.UtcNow.AddMinutes(-1));
}
/// <summary>
///
/// </summary>
private void RebuildLuceneIndex()
{
try
{
var dir = new Lucene.Net.Store.Azure.AzureDirectory(CfCloudStorage, "SearchCatalog", new Lucene.Net.Store.RAMDirectory());
//-- The magic code... delete all the blobs before rebuilding the index
foreach (var b in dir.BlobContainer.ListBlobs())
{
dir.BlobContainer.GetBlobReference(b.Uri.ToString()).Delete();
}
new cf.Content.Search.SearchManager().BuildIndex(dir);
}
catch (Exception ex)
{
CfTrace.Error(ex);
}
}
private void InitializeEnvironmentCacheAndSearchProviders()
{
CfTrace.Current.Information(TraceCode.AppBuildSearchIndex, "Started {0:MMM dd HH.mm.ss}", DateTime.Now);
if (!RoleEnvironment.IsAvailable)
{
CfCacheIndex.Initialize();
CfPerfCache.Initialize();
new SearchManager().BuildIndex(null);
var directoryPath = AppDomain.CurrentDomain.BaseDirectory + @"App_Data\";
var dir = Lucene.Net.Store.FSDirectory.Open(new System.IO.DirectoryInfo(directoryPath));
_siteSearchEngine = new CfLuceneIndexSearcher(dir);
}
else
{
CfCacheIndex.Initialize(new Level2MemcachedCacheIndex());
CfPerfCache.Initialize(new Level2MemcachedPerfCache());
var connectionString = ConfigurationManager.ConnectionStrings["CfCloudStorage"].ConnectionString;
var cloudStorage = CloudStorageAccount.Parse(connectionString);
var dir = new Lucene.Net.Store.Azure.AzureDirectory(cloudStorage, "SearchCatalog", new RAMDirectory());
_siteSearchEngine = new CfLuceneIndexSearcher(dir);
lastIndexRefresh = DateTime.UtcNow;
}
CfTrace.Current.Information(TraceCode.AppBuildSearchIndex, "Ended {0:MMM dd HH.mm.ss}", DateTime.Now);
}
public async Task <RebuildIndicesResponse> Handle(RebuildIndicesRequest request, CancellationToken cancellationToken)
{
_logger.LogDebug("RebuildIndicesResponseHandler started.");
cancellationToken.ThrowIfCancellationRequested();
IndexWriter writer = null;
Lucene.Net.Store.Azure.AzureDirectory azureDirectory = null;
DateTimeOffset lastSyncPoint = DateTimeOffset.MinValue;
DateTimeOffset currentSyncPoint = DateTimeOffset.Now;
int? updatedCount = null;
int? deletedCount = null;
try
{
// Ensures index backwards compatibility
var AppLuceneVersion = LuceneVersion.LUCENE_48;
//Azure configuration
var accountSAS = new Microsoft.Azure.Storage.Auth.StorageCredentials(AzureLuceneConfiguration.SASToken);
var accountWithSAS = new Microsoft.Azure.Storage.CloudStorageAccount(accountSAS, AzureLuceneConfiguration.AzureStorageAccountName, endpointSuffix: null, useHttps: true);
var tempLocation = AzureLuceneConfiguration.TempDirectory ?? "temp";
_logger.LogTrace("tempLocation: {0}", tempLocation);
azureDirectory = new Lucene.Net.Store.Azure.AzureDirectory(accountWithSAS, tempLocation, containerName: AzureLuceneConfiguration.Container);
//ensure RAMDirectory
azureDirectory.CacheDirectory = new RAMDirectory();
//create an analyzer to process the text
var analyzer = new StandardAnalyzer(AppLuceneVersion);
//create an index writer
var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer);
writer = new IndexWriter(azureDirectory, indexConfig); //used to be dir
_logger.LogTrace("IndexWriter is initialized");
if (request.FullRebuild)
{
_logger.LogInformation("Full Rebuild is requested. Deleting indices");
writer.DeleteAll();
writer.Commit();
_logger.LogTrace("Full Rebuild is committed.");
}
using (var dbConnection = await _SQLservice.GetConnection(cancellationToken))
{
SqlCommand cmd;
if (!request.FullRebuild)
{
//we need last sync point only if it is not full rebuild
var dbCommand = @"SELECT TOP 1 LastSyncPoint FROM [dbo].[FTS_Config]";
cmd = new SqlCommand(dbCommand, dbConnection);
try
{
var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken);
var lastSyncPointNullable = untyped as DateTimeOffset?;
if (lastSyncPointNullable.HasValue)
{
lastSyncPoint = lastSyncPointNullable.Value;
}
_logger.LogDebug("Last sync point is {0}", lastSyncPointNullable.HasValue ? lastSyncPointNullable.Value.ToString() : "'never'");
}
catch (Exception ex)
{
_logger.LogError(ex, "unexpected failure to acquire LastSyncPoint from database");
throw;
}
}
else
{
lastSyncPoint = DateTimeOffset.MinValue;
}
//determine number of records that will need to be processed
var dbCountCommand = @"SELECT COUNT(Id) from [dbo].[Test_Data] WHERE UpdatedAt >= @lastSyncPoint AND UpdatedAt < @currentSyncPoint AND DeletedAt IS NULL";
cmd = new SqlCommand(dbCountCommand, dbConnection);
cmd.Parameters.Add("@lastSyncPoint", System.Data.SqlDbType.DateTimeOffset);
cmd.Parameters["@lastSyncPoint"].Value = lastSyncPoint;
cmd.Parameters.Add("@currentSyncPoint", System.Data.SqlDbType.DateTimeOffset);
cmd.Parameters["@currentSyncPoint"].Value = currentSyncPoint;
try
{
var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken);
updatedCount = untyped as int?;
_logger.LogDebug("Expected number of updated documents {0}", updatedCount.HasValue ? updatedCount.Value.ToString() : "'none'");
}
catch (Exception ex)
{
_logger.LogError(ex, "unexpected failure to acquire number of documents to be updated from database");
throw;
}
//working on deleted documents
if (!request.FullRebuild)
{
//also need to remove "Deleted" documents. Only if not full rebuild of indices
var dbDeletedCountCommand = @"SELECT COUNT(Id) from [dbo].[Test_Data] WHERE DeletedAt >= @lastSyncPoint AND DeletedAt<=@currentSyncPoint AND DeletedAt IS NOT NULL";
cmd = new SqlCommand(dbDeletedCountCommand, dbConnection);
cmd.Parameters.Add("@lastSyncPoint", System.Data.SqlDbType.DateTimeOffset);
cmd.Parameters["@lastSyncPoint"].Value = lastSyncPoint;
cmd.Parameters.Add("@currentSyncPoint", System.Data.SqlDbType.DateTimeOffset);
cmd.Parameters["@currentSyncPoint"].Value = currentSyncPoint;
try
{
var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken);
deletedCount = untyped as int?;
_logger.LogDebug("Expected number of deleted documents {0}", deletedCount.HasValue ? updatedCount.Value.ToString() : "'none'");
}
catch (Exception ex)
{
_logger.LogError(ex, "unexpected failure to acquire 'number of documents to be delete from indicies' from database");
throw;
}
}
}
var atLeastOneUpdate = false;
if (updatedCount.HasValue && updatedCount.Value > 0)
{
_logger.LogDebug("Expected number of updated documents: {0}", updatedCount.Value);
//Start updating 'Updated records'
await UpdateIndicesWithAddedDocuments(lastSyncPoint, currentSyncPoint, updatedCount.Value, writer, cancellationToken);
atLeastOneUpdate = true;
}
else
{
_logger.LogDebug("Expected number of updated documents: none ");
}
if (deletedCount.HasValue && deletedCount.Value > 0)
{
_logger.LogDebug("Expected number of deleted documents: {0}", deletedCount.Value);
await UpdateIndicesWithDeletedDocuments(lastSyncPoint, currentSyncPoint, deletedCount.Value, writer, cancellationToken);
atLeastOneUpdate = true;
}
else
{
_logger.LogDebug("Expected number of updated documents: none ");
}
if (atLeastOneUpdate)
{
_logger.LogDebug("Expected number of updated documents: none ");
_luceneReaderService.Evict();
writer.Flush(triggerMerge: true, applyAllDeletes: true);
_logger.LogInformation("Indexes are updated");
}
//update LastSyncPoint
using (var dbConnection = await _SQLservice.GetConnection(cancellationToken))
{
var dbCommand = @"UPDATE [dbo].[FTS_Config] SET LastSyncPoint = @currentSyncPoint";
var cmd = new SqlCommand(dbCommand, dbConnection);
cmd.Parameters.Add("@currentSyncPoint", System.Data.SqlDbType.DateTimeOffset);
cmd.Parameters["@currentSyncPoint"].Value = currentSyncPoint;
try
{
await _SQLservice.ExecuteNonQueryWithRetryAsync(cmd, cancellationToken);
_logger.LogDebug("Last sync point is set to {0}", currentSyncPoint);
}
catch (Exception ex)
{
_logger.LogError(ex, "unexpected failure to update LastSyncPoint in database");
throw;
}
}
var result = new RebuildIndicesResponse
{
IsValid = true,
Success = true,
NumberOfUpdates = updatedCount,
NumberOfDeletes = deletedCount,
CurrentSyncPoint = currentSyncPoint
};
return(result);
}
catch (LockObtainFailedException)
{
var result = new RebuildIndicesResponse();
result.IsValid = false;
result.Errors = new List <string>();
result.Errors.Add("Failed to lock full text search index file. Probaly there is another job is running. Please try again later.");
return(result);
}
catch (Exception ex)
{
var result = new RebuildIndicesResponse();
result.IsValid = false;
result.Errors = new List <string>();
result.Errors.Add("Unexpected error occured: " + ex.Message);
return(result);
}
finally
{
if (writer != null)
{
writer.Dispose();
}
if (azureDirectory != null)
{
azureDirectory.Dispose();
}
}
}
