/// <summary>
/// Creates a connection to Azure Storage from an account name, key, and endpoints
/// </summary>
public AzureStorageAccount(
string accountName,
string accountKey,
Uri?blobStorageEndpoint,
Uri?queueStorageEndpoint,
Uri?tableStorageEndpoint,
Uri?fileStorageEndpoint)
{
if ((blobStorageEndpoint == null) &&
(queueStorageEndpoint == null) &&
(tableStorageEndpoint == null) &&
(fileStorageEndpoint == null))
{
throw new ArgumentException($"{nameof(AzureStorageAccount)} must have at least one remote URI to connect to");
}
Microsoft.Azure.Storage.Auth.StorageCredentials credentials = new Microsoft.Azure.Storage.Auth.StorageCredentials(accountName, accountKey);
_account = new Microsoft.Azure.Storage.CloudStorageAccount(credentials, blobStorageEndpoint, queueStorageEndpoint, tableStorageEndpoint, fileStorageEndpoint);
_storageContainers = new Dictionary <string, AzureBlobStorage>();
if (tableStorageEndpoint != null)
{
Microsoft.Azure.Cosmos.Table.StorageCredentials tableCredentials = new Microsoft.Azure.Cosmos.Table.StorageCredentials(accountName, accountKey);
_tableAccount = new Microsoft.Azure.Cosmos.Table.CloudStorageAccount(tableCredentials, tableStorageEndpoint);
}
}
public async Task <bool> UploadPhoto(Stream photo, string sharedAccessSignature)
{
try
{
var creds = new Microsoft.Azure.Storage.Auth.StorageCredentials(sharedAccessSignature);
storageAccountName = DefaultSettings.StorageAccountName;
var account = new CloudStorageAccount(creds, storageAccountName, "core.windows.net", true);
var blobClient = account.CreateCloudBlobClient();
var container = blobClient.GetContainerReference("wishlist");
var blockBlob = container.GetBlockBlobReference($"TT-{Guid.NewGuid()}.jpg");
await blockBlob.UploadFromStreamAsync(photo);
}
catch (Exception ex)
{
System.Diagnostics.Debug.WriteLine(ex);
return(false);
}
return(true);
}
public async Task <bool> UploadPhoto(Stream photo, string sharedAccessSignature)
{
try
{
var creds = new Microsoft.Azure.Storage.Auth.StorageCredentials(sharedAccessSignature);
var storageAccountName = Preferences.Get(PreferencesConstants.StorageAccountNameKey,
PreferencesConstants.DemoStorageName);
var account = new CloudStorageAccount(creds, storageAccountName, "core.windows.net", true);
var blobClient = account.CreateCloudBlobClient();
var container = blobClient.GetContainerReference("wishlist");
var blockBlob = container.GetBlockBlobReference($"TT-{Guid.NewGuid()}.jpg");
await blockBlob.UploadFromStreamAsync(photo);
}
catch (Exception ex)
{
Crashes.TrackError(ex, new Dictionary <string, string> {
{ "Function", "AzureStorageService.UploadPhoto" }
});
return(false);
}
return(true);
}
public StorageAccount(string storageName, StorageCredentialType type, string keyOrToken)
{
fileURI = new Uri(string.Format("https://{0}.file.core.windows.net/"));
queueURI = new Uri(string.Format("https://{0}.queue.core.windows.net/"));
blobURI = new Uri(string.Format("https://{0}.blob.core.windows.net/"));
tableURI = new Uri(string.Format("https://{0}.table.core.windows.net/"));
try
{
if (type == StorageCredentialType.SAS)
{
creds = new Microsoft.Azure.Storage.Auth.StorageCredentials(keyOrToken);
tableCreds = new StorageCredentials(keyOrToken);
}
else
{
creds = new Microsoft.Azure.Storage.Auth.StorageCredentials(storageName, keyOrToken);
tableCreds = new StorageCredentials(storageName, keyOrToken);
}
}
catch (Exception e)
{
throw new StorageAccountException("Error while attempting to authenticate", e);
}
}
public async Task <LuceneIndexAndMetadata> GetReaderInternal(CancellationToken cancellationToken)
{
var resultObject = new LuceneIndexAndMetadata();
//Azure configuration
var accountSAS = new Microsoft.Azure.Storage.Auth.StorageCredentials(_azureLuceneConfiguration.SASToken);
var accountWithSAS = new Microsoft.Azure.Storage.CloudStorageAccount(accountSAS, _azureLuceneConfiguration.AzureStorageAccountName, endpointSuffix: null, useHttps: true);
_logger.LogInformation("_azureLuceneConfiguration.TempDirectory = {0}", _azureLuceneConfiguration.TempDirectory);
var tempLocation = _azureLuceneConfiguration.TempDirectory ?? "temp";
_logger.LogDebug("Lucene IndexReader is located in {0} azure storage account (container '{1}')"
, _azureLuceneConfiguration.AzureStorageAccountName
, _azureLuceneConfiguration.Container);
var azureDirectory = new Lucene.Net.Store.Azure.AzureDirectory(accountWithSAS, tempLocation, containerName: _azureLuceneConfiguration.Container);
//ensure RAMDirectory
azureDirectory.CacheDirectory = new Lucene.Net.Store.RAMDirectory();
var reader = DirectoryReader.Open(azureDirectory);
_logger.LogDebug("Lucene IndexReader is acquired.");
resultObject.Index = reader;
using (var dbConnection = await _SQLservice.GetConnection(cancellationToken))
{
//we need last sync point only if it is not full rebuild
var dbCommand = @"SELECT TOP 1 LastSyncPoint FROM [dbo].[FTS_Config]";
var cmd = new SqlCommand(dbCommand, dbConnection);
try
{
var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken);
var lastSyncPointNullable = untyped as DateTimeOffset?;
if (lastSyncPointNullable.HasValue)
{
resultObject.LastIndexOffset = lastSyncPointNullable.Value;
}
_logger.LogDebug("Last sync point is {0}", lastSyncPointNullable.HasValue ? lastSyncPointNullable.Value.ToString() : "'never'");
}
catch (Exception ex)
{
_logger.LogError(ex, "unexpected failure to acquire LastSyncPoint from database");
throw;
}
}
return(resultObject);
}
/// <summary>
/// Initialise the batch and storage clients for communication with Microsoft's APIs.
/// </summary>
private void Initialise()
{
Licence licence = new Licence(AzureSettings.Default.LicenceFilePath);
// Setup Azure batch/storage clients using the given credentials.
var credentials = new Microsoft.Azure.Storage.Auth.StorageCredentials(licence.StorageAccount, licence.StorageKey);
var storageAccount = new CloudStorageAccount(credentials, true);
storageClient = storageAccount.CreateCloudBlobClient();
var sharedCredentials = new BatchSharedKeyCredentials(licence.BatchUrl, licence.BatchAccount, licence.BatchKey);
batchClient = BatchClient.Open(sharedCredentials);
}
/// <summary>
/// Sets metadata for a particular container in Azure's cloud storage.
/// </summary>
/// <param name="containerName">Name of the container</param>
/// <param name="key">Metadata key/name</param>
/// <param name="val">Data to write</param>
private void SetAzureMetaData(string containerName, string key, string val)
{
try
{
var credentials = new Microsoft.Azure.Storage.Auth.StorageCredentials(
(string)AzureSettings.Default["StorageAccount"],
(string)AzureSettings.Default["StorageKey"]);
var storageAccount = new CloudStorageAccount(credentials, true);
var blobClient = Microsoft.Azure.Storage.Blob.BlobAccountExtensions.CreateCloudBlobClient(storageAccount);
var containerRef = blobClient.GetContainerReference(containerName);
containerRef.CreateIfNotExists();
containerRef.Metadata.Add(key, val);
containerRef.SetMetadata();
}
catch (Exception err)
{
ShowError(err);
}
}
/// <summary>
/// Upload a file to Azure's cloud storage if it does not already exist.
/// </summary>
/// <param name="containerName">Name of the container to upload the file to</param>
/// <param name="filePath">Path to the file on disk</param>
private void UploadFileIfNeeded(string containerName, string filePath)
{
var credentials = new Microsoft.Azure.Storage.Auth.StorageCredentials(
(string)AzureSettings.Default["StorageAccount"],
(string)AzureSettings.Default["StorageKey"]);
var storageAccount = new CloudStorageAccount(credentials, true);
var blobClient = Microsoft.Azure.Storage.Blob.BlobAccountExtensions.CreateCloudBlobClient(storageAccount);
var containerRef = blobClient.GetContainerReference(containerName);
containerRef.CreateIfNotExists();
var blobRef = containerRef.GetBlockBlobReference(Path.GetFileName(filePath));
var md5 = GetFileMd5(filePath);
// if blob exists and md5 matches, there is no need to upload the file
if (blobRef.Exists() && string.Equals(md5, blobRef.Properties.ContentMD5, StringComparison.InvariantCultureIgnoreCase))
{
return;
}
blobRef.Properties.ContentMD5 = md5;
blobRef.UploadFromFileAsync(filePath);
}
public async Task <RebuildIndicesResponse> Handle(RebuildIndicesRequest request, CancellationToken cancellationToken)
{
_logger.LogDebug("RebuildIndicesResponseHandler started.");
cancellationToken.ThrowIfCancellationRequested();
IndexWriter writer = null;
Lucene.Net.Store.Azure.AzureDirectory azureDirectory = null;
DateTimeOffset lastSyncPoint = DateTimeOffset.MinValue;
DateTimeOffset currentSyncPoint = DateTimeOffset.Now;
int? updatedCount = null;
int? deletedCount = null;
try
{
// Ensures index backwards compatibility
var AppLuceneVersion = LuceneVersion.LUCENE_48;
//Azure configuration
var accountSAS = new Microsoft.Azure.Storage.Auth.StorageCredentials(AzureLuceneConfiguration.SASToken);
var accountWithSAS = new Microsoft.Azure.Storage.CloudStorageAccount(accountSAS, AzureLuceneConfiguration.AzureStorageAccountName, endpointSuffix: null, useHttps: true);
var tempLocation = AzureLuceneConfiguration.TempDirectory ?? "temp";
_logger.LogTrace("tempLocation: {0}", tempLocation);
azureDirectory = new Lucene.Net.Store.Azure.AzureDirectory(accountWithSAS, tempLocation, containerName: AzureLuceneConfiguration.Container);
//ensure RAMDirectory
azureDirectory.CacheDirectory = new RAMDirectory();
//create an analyzer to process the text
var analyzer = new StandardAnalyzer(AppLuceneVersion);
//create an index writer
var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer);
writer = new IndexWriter(azureDirectory, indexConfig); //used to be dir
_logger.LogTrace("IndexWriter is initialized");
if (request.FullRebuild)
{
_logger.LogInformation("Full Rebuild is requested. Deleting indices");
writer.DeleteAll();
writer.Commit();
_logger.LogTrace("Full Rebuild is committed.");
}
using (var dbConnection = await _SQLservice.GetConnection(cancellationToken))
{
SqlCommand cmd;
if (!request.FullRebuild)
{
//we need last sync point only if it is not full rebuild
var dbCommand = @"SELECT TOP 1 LastSyncPoint FROM [dbo].[FTS_Config]";
cmd = new SqlCommand(dbCommand, dbConnection);
try
{
var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken);
var lastSyncPointNullable = untyped as DateTimeOffset?;
if (lastSyncPointNullable.HasValue)
{
lastSyncPoint = lastSyncPointNullable.Value;
}
_logger.LogDebug("Last sync point is {0}", lastSyncPointNullable.HasValue ? lastSyncPointNullable.Value.ToString() : "'never'");
}
catch (Exception ex)
{
_logger.LogError(ex, "unexpected failure to acquire LastSyncPoint from database");
throw;
}
}
else
{
lastSyncPoint = DateTimeOffset.MinValue;
}
//determine number of records that will need to be processed
var dbCountCommand = @"SELECT COUNT(Id) from [dbo].[Test_Data] WHERE UpdatedAt >= @lastSyncPoint AND UpdatedAt < @currentSyncPoint AND DeletedAt IS NULL";
cmd = new SqlCommand(dbCountCommand, dbConnection);
cmd.Parameters.Add("@lastSyncPoint", System.Data.SqlDbType.DateTimeOffset);
cmd.Parameters["@lastSyncPoint"].Value = lastSyncPoint;
cmd.Parameters.Add("@currentSyncPoint", System.Data.SqlDbType.DateTimeOffset);
cmd.Parameters["@currentSyncPoint"].Value = currentSyncPoint;
try
{
var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken);
updatedCount = untyped as int?;
_logger.LogDebug("Expected number of updated documents {0}", updatedCount.HasValue ? updatedCount.Value.ToString() : "'none'");
}
catch (Exception ex)
{
_logger.LogError(ex, "unexpected failure to acquire number of documents to be updated from database");
throw;
}
//working on deleted documents
if (!request.FullRebuild)
{
//also need to remove "Deleted" documents. Only if not full rebuild of indices
var dbDeletedCountCommand = @"SELECT COUNT(Id) from [dbo].[Test_Data] WHERE DeletedAt >= @lastSyncPoint AND DeletedAt<=@currentSyncPoint AND DeletedAt IS NOT NULL";
cmd = new SqlCommand(dbDeletedCountCommand, dbConnection);
cmd.Parameters.Add("@lastSyncPoint", System.Data.SqlDbType.DateTimeOffset);
cmd.Parameters["@lastSyncPoint"].Value = lastSyncPoint;
cmd.Parameters.Add("@currentSyncPoint", System.Data.SqlDbType.DateTimeOffset);
cmd.Parameters["@currentSyncPoint"].Value = currentSyncPoint;
try
{
var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken);
deletedCount = untyped as int?;
_logger.LogDebug("Expected number of deleted documents {0}", deletedCount.HasValue ? updatedCount.Value.ToString() : "'none'");
}
catch (Exception ex)
{
_logger.LogError(ex, "unexpected failure to acquire 'number of documents to be delete from indicies' from database");
throw;
}
}
}
var atLeastOneUpdate = false;
if (updatedCount.HasValue && updatedCount.Value > 0)
{
_logger.LogDebug("Expected number of updated documents: {0}", updatedCount.Value);
//Start updating 'Updated records'
await UpdateIndicesWithAddedDocuments(lastSyncPoint, currentSyncPoint, updatedCount.Value, writer, cancellationToken);
atLeastOneUpdate = true;
}
else
{
_logger.LogDebug("Expected number of updated documents: none ");
}
if (deletedCount.HasValue && deletedCount.Value > 0)
{
_logger.LogDebug("Expected number of deleted documents: {0}", deletedCount.Value);
await UpdateIndicesWithDeletedDocuments(lastSyncPoint, currentSyncPoint, deletedCount.Value, writer, cancellationToken);
atLeastOneUpdate = true;
}
else
{
_logger.LogDebug("Expected number of updated documents: none ");
}
if (atLeastOneUpdate)
{
_logger.LogDebug("Expected number of updated documents: none ");
_luceneReaderService.Evict();
writer.Flush(triggerMerge: true, applyAllDeletes: true);
_logger.LogInformation("Indexes are updated");
}
//update LastSyncPoint
using (var dbConnection = await _SQLservice.GetConnection(cancellationToken))
{
var dbCommand = @"UPDATE [dbo].[FTS_Config] SET LastSyncPoint = @currentSyncPoint";
var cmd = new SqlCommand(dbCommand, dbConnection);
cmd.Parameters.Add("@currentSyncPoint", System.Data.SqlDbType.DateTimeOffset);
cmd.Parameters["@currentSyncPoint"].Value = currentSyncPoint;
try
{
await _SQLservice.ExecuteNonQueryWithRetryAsync(cmd, cancellationToken);
_logger.LogDebug("Last sync point is set to {0}", currentSyncPoint);
}
catch (Exception ex)
{
_logger.LogError(ex, "unexpected failure to update LastSyncPoint in database");
throw;
}
}
var result = new RebuildIndicesResponse
{
IsValid = true,
Success = true,
NumberOfUpdates = updatedCount,
NumberOfDeletes = deletedCount,
CurrentSyncPoint = currentSyncPoint
};
return(result);
}
catch (LockObtainFailedException)
{
var result = new RebuildIndicesResponse();
result.IsValid = false;
result.Errors = new List <string>();
result.Errors.Add("Failed to lock full text search index file. Probaly there is another job is running. Please try again later.");
return(result);
}
catch (Exception ex)
{
var result = new RebuildIndicesResponse();
result.IsValid = false;
result.Errors = new List <string>();
result.Errors.Add("Unexpected error occured: " + ex.Message);
return(result);
}
finally
{
if (writer != null)
{
writer.Dispose();
}
if (azureDirectory != null)
{
azureDirectory.Dispose();
}
}
}
