/// <summary> /// Creates a connection to Azure Storage from an account name, key, and endpoints /// </summary> public AzureStorageAccount( string accountName, string accountKey, Uri?blobStorageEndpoint, Uri?queueStorageEndpoint, Uri?tableStorageEndpoint, Uri?fileStorageEndpoint) { if ((blobStorageEndpoint == null) && (queueStorageEndpoint == null) && (tableStorageEndpoint == null) && (fileStorageEndpoint == null)) { throw new ArgumentException($"{nameof(AzureStorageAccount)} must have at least one remote URI to connect to"); } Microsoft.Azure.Storage.Auth.StorageCredentials credentials = new Microsoft.Azure.Storage.Auth.StorageCredentials(accountName, accountKey); _account = new Microsoft.Azure.Storage.CloudStorageAccount(credentials, blobStorageEndpoint, queueStorageEndpoint, tableStorageEndpoint, fileStorageEndpoint); _storageContainers = new Dictionary <string, AzureBlobStorage>(); if (tableStorageEndpoint != null) { Microsoft.Azure.Cosmos.Table.StorageCredentials tableCredentials = new Microsoft.Azure.Cosmos.Table.StorageCredentials(accountName, accountKey); _tableAccount = new Microsoft.Azure.Cosmos.Table.CloudStorageAccount(tableCredentials, tableStorageEndpoint); } }
public async Task <bool> UploadPhoto(Stream photo, string sharedAccessSignature) { try { var creds = new Microsoft.Azure.Storage.Auth.StorageCredentials(sharedAccessSignature); storageAccountName = DefaultSettings.StorageAccountName; var account = new CloudStorageAccount(creds, storageAccountName, "core.windows.net", true); var blobClient = account.CreateCloudBlobClient(); var container = blobClient.GetContainerReference("wishlist"); var blockBlob = container.GetBlockBlobReference($"TT-{Guid.NewGuid()}.jpg"); await blockBlob.UploadFromStreamAsync(photo); } catch (Exception ex) { System.Diagnostics.Debug.WriteLine(ex); return(false); } return(true); }
public async Task <bool> UploadPhoto(Stream photo, string sharedAccessSignature) { try { var creds = new Microsoft.Azure.Storage.Auth.StorageCredentials(sharedAccessSignature); var storageAccountName = Preferences.Get(PreferencesConstants.StorageAccountNameKey, PreferencesConstants.DemoStorageName); var account = new CloudStorageAccount(creds, storageAccountName, "core.windows.net", true); var blobClient = account.CreateCloudBlobClient(); var container = blobClient.GetContainerReference("wishlist"); var blockBlob = container.GetBlockBlobReference($"TT-{Guid.NewGuid()}.jpg"); await blockBlob.UploadFromStreamAsync(photo); } catch (Exception ex) { Crashes.TrackError(ex, new Dictionary <string, string> { { "Function", "AzureStorageService.UploadPhoto" } }); return(false); } return(true); }
public StorageAccount(string storageName, StorageCredentialType type, string keyOrToken) { fileURI = new Uri(string.Format("https://{0}.file.core.windows.net/")); queueURI = new Uri(string.Format("https://{0}.queue.core.windows.net/")); blobURI = new Uri(string.Format("https://{0}.blob.core.windows.net/")); tableURI = new Uri(string.Format("https://{0}.table.core.windows.net/")); try { if (type == StorageCredentialType.SAS) { creds = new Microsoft.Azure.Storage.Auth.StorageCredentials(keyOrToken); tableCreds = new StorageCredentials(keyOrToken); } else { creds = new Microsoft.Azure.Storage.Auth.StorageCredentials(storageName, keyOrToken); tableCreds = new StorageCredentials(storageName, keyOrToken); } } catch (Exception e) { throw new StorageAccountException("Error while attempting to authenticate", e); } }
public async Task <LuceneIndexAndMetadata> GetReaderInternal(CancellationToken cancellationToken) { var resultObject = new LuceneIndexAndMetadata(); //Azure configuration var accountSAS = new Microsoft.Azure.Storage.Auth.StorageCredentials(_azureLuceneConfiguration.SASToken); var accountWithSAS = new Microsoft.Azure.Storage.CloudStorageAccount(accountSAS, _azureLuceneConfiguration.AzureStorageAccountName, endpointSuffix: null, useHttps: true); _logger.LogInformation("_azureLuceneConfiguration.TempDirectory = {0}", _azureLuceneConfiguration.TempDirectory); var tempLocation = _azureLuceneConfiguration.TempDirectory ?? "temp"; _logger.LogDebug("Lucene IndexReader is located in {0} azure storage account (container '{1}')" , _azureLuceneConfiguration.AzureStorageAccountName , _azureLuceneConfiguration.Container); var azureDirectory = new Lucene.Net.Store.Azure.AzureDirectory(accountWithSAS, tempLocation, containerName: _azureLuceneConfiguration.Container); //ensure RAMDirectory azureDirectory.CacheDirectory = new Lucene.Net.Store.RAMDirectory(); var reader = DirectoryReader.Open(azureDirectory); _logger.LogDebug("Lucene IndexReader is acquired."); resultObject.Index = reader; using (var dbConnection = await _SQLservice.GetConnection(cancellationToken)) { //we need last sync point only if it is not full rebuild var dbCommand = @"SELECT TOP 1 LastSyncPoint FROM [dbo].[FTS_Config]"; var cmd = new SqlCommand(dbCommand, dbConnection); try { var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken); var lastSyncPointNullable = untyped as DateTimeOffset?; if (lastSyncPointNullable.HasValue) { resultObject.LastIndexOffset = lastSyncPointNullable.Value; } _logger.LogDebug("Last sync point is {0}", lastSyncPointNullable.HasValue ? lastSyncPointNullable.Value.ToString() : "'never'"); } catch (Exception ex) { _logger.LogError(ex, "unexpected failure to acquire LastSyncPoint from database"); throw; } } return(resultObject); }
/// <summary> /// Initialise the batch and storage clients for communication with Microsoft's APIs. /// </summary> private void Initialise() { Licence licence = new Licence(AzureSettings.Default.LicenceFilePath); // Setup Azure batch/storage clients using the given credentials. var credentials = new Microsoft.Azure.Storage.Auth.StorageCredentials(licence.StorageAccount, licence.StorageKey); var storageAccount = new CloudStorageAccount(credentials, true); storageClient = storageAccount.CreateCloudBlobClient(); var sharedCredentials = new BatchSharedKeyCredentials(licence.BatchUrl, licence.BatchAccount, licence.BatchKey); batchClient = BatchClient.Open(sharedCredentials); }
/// <summary> /// Sets metadata for a particular container in Azure's cloud storage. /// </summary> /// <param name="containerName">Name of the container</param> /// <param name="key">Metadata key/name</param> /// <param name="val">Data to write</param> private void SetAzureMetaData(string containerName, string key, string val) { try { var credentials = new Microsoft.Azure.Storage.Auth.StorageCredentials( (string)AzureSettings.Default["StorageAccount"], (string)AzureSettings.Default["StorageKey"]); var storageAccount = new CloudStorageAccount(credentials, true); var blobClient = Microsoft.Azure.Storage.Blob.BlobAccountExtensions.CreateCloudBlobClient(storageAccount); var containerRef = blobClient.GetContainerReference(containerName); containerRef.CreateIfNotExists(); containerRef.Metadata.Add(key, val); containerRef.SetMetadata(); } catch (Exception err) { ShowError(err); } }
/// <summary> /// Upload a file to Azure's cloud storage if it does not already exist. /// </summary> /// <param name="containerName">Name of the container to upload the file to</param> /// <param name="filePath">Path to the file on disk</param> private void UploadFileIfNeeded(string containerName, string filePath) { var credentials = new Microsoft.Azure.Storage.Auth.StorageCredentials( (string)AzureSettings.Default["StorageAccount"], (string)AzureSettings.Default["StorageKey"]); var storageAccount = new CloudStorageAccount(credentials, true); var blobClient = Microsoft.Azure.Storage.Blob.BlobAccountExtensions.CreateCloudBlobClient(storageAccount); var containerRef = blobClient.GetContainerReference(containerName); containerRef.CreateIfNotExists(); var blobRef = containerRef.GetBlockBlobReference(Path.GetFileName(filePath)); var md5 = GetFileMd5(filePath); // if blob exists and md5 matches, there is no need to upload the file if (blobRef.Exists() && string.Equals(md5, blobRef.Properties.ContentMD5, StringComparison.InvariantCultureIgnoreCase)) { return; } blobRef.Properties.ContentMD5 = md5; blobRef.UploadFromFileAsync(filePath); }
public async Task <RebuildIndicesResponse> Handle(RebuildIndicesRequest request, CancellationToken cancellationToken) { _logger.LogDebug("RebuildIndicesResponseHandler started."); cancellationToken.ThrowIfCancellationRequested(); IndexWriter writer = null; Lucene.Net.Store.Azure.AzureDirectory azureDirectory = null; DateTimeOffset lastSyncPoint = DateTimeOffset.MinValue; DateTimeOffset currentSyncPoint = DateTimeOffset.Now; int? updatedCount = null; int? deletedCount = null; try { // Ensures index backwards compatibility var AppLuceneVersion = LuceneVersion.LUCENE_48; //Azure configuration var accountSAS = new Microsoft.Azure.Storage.Auth.StorageCredentials(AzureLuceneConfiguration.SASToken); var accountWithSAS = new Microsoft.Azure.Storage.CloudStorageAccount(accountSAS, AzureLuceneConfiguration.AzureStorageAccountName, endpointSuffix: null, useHttps: true); var tempLocation = AzureLuceneConfiguration.TempDirectory ?? "temp"; _logger.LogTrace("tempLocation: {0}", tempLocation); azureDirectory = new Lucene.Net.Store.Azure.AzureDirectory(accountWithSAS, tempLocation, containerName: AzureLuceneConfiguration.Container); //ensure RAMDirectory azureDirectory.CacheDirectory = new RAMDirectory(); //create an analyzer to process the text var analyzer = new StandardAnalyzer(AppLuceneVersion); //create an index writer var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer); writer = new IndexWriter(azureDirectory, indexConfig); //used to be dir _logger.LogTrace("IndexWriter is initialized"); if (request.FullRebuild) { _logger.LogInformation("Full Rebuild is requested. Deleting indices"); writer.DeleteAll(); writer.Commit(); _logger.LogTrace("Full Rebuild is committed."); } using (var dbConnection = await _SQLservice.GetConnection(cancellationToken)) { SqlCommand cmd; if (!request.FullRebuild) { //we need last sync point only if it is not full rebuild var dbCommand = @"SELECT TOP 1 LastSyncPoint FROM [dbo].[FTS_Config]"; cmd = new SqlCommand(dbCommand, dbConnection); try { var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken); var lastSyncPointNullable = untyped as DateTimeOffset?; if (lastSyncPointNullable.HasValue) { lastSyncPoint = lastSyncPointNullable.Value; } _logger.LogDebug("Last sync point is {0}", lastSyncPointNullable.HasValue ? lastSyncPointNullable.Value.ToString() : "'never'"); } catch (Exception ex) { _logger.LogError(ex, "unexpected failure to acquire LastSyncPoint from database"); throw; } } else { lastSyncPoint = DateTimeOffset.MinValue; } //determine number of records that will need to be processed var dbCountCommand = @"SELECT COUNT(Id) from [dbo].[Test_Data] WHERE UpdatedAt >= @lastSyncPoint AND UpdatedAt < @currentSyncPoint AND DeletedAt IS NULL"; cmd = new SqlCommand(dbCountCommand, dbConnection); cmd.Parameters.Add("@lastSyncPoint", System.Data.SqlDbType.DateTimeOffset); cmd.Parameters["@lastSyncPoint"].Value = lastSyncPoint; cmd.Parameters.Add("@currentSyncPoint", System.Data.SqlDbType.DateTimeOffset); cmd.Parameters["@currentSyncPoint"].Value = currentSyncPoint; try { var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken); updatedCount = untyped as int?; _logger.LogDebug("Expected number of updated documents {0}", updatedCount.HasValue ? updatedCount.Value.ToString() : "'none'"); } catch (Exception ex) { _logger.LogError(ex, "unexpected failure to acquire number of documents to be updated from database"); throw; } //working on deleted documents if (!request.FullRebuild) { //also need to remove "Deleted" documents. Only if not full rebuild of indices var dbDeletedCountCommand = @"SELECT COUNT(Id) from [dbo].[Test_Data] WHERE DeletedAt >= @lastSyncPoint AND DeletedAt<=@currentSyncPoint AND DeletedAt IS NOT NULL"; cmd = new SqlCommand(dbDeletedCountCommand, dbConnection); cmd.Parameters.Add("@lastSyncPoint", System.Data.SqlDbType.DateTimeOffset); cmd.Parameters["@lastSyncPoint"].Value = lastSyncPoint; cmd.Parameters.Add("@currentSyncPoint", System.Data.SqlDbType.DateTimeOffset); cmd.Parameters["@currentSyncPoint"].Value = currentSyncPoint; try { var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken); deletedCount = untyped as int?; _logger.LogDebug("Expected number of deleted documents {0}", deletedCount.HasValue ? updatedCount.Value.ToString() : "'none'"); } catch (Exception ex) { _logger.LogError(ex, "unexpected failure to acquire 'number of documents to be delete from indicies' from database"); throw; } } } var atLeastOneUpdate = false; if (updatedCount.HasValue && updatedCount.Value > 0) { _logger.LogDebug("Expected number of updated documents: {0}", updatedCount.Value); //Start updating 'Updated records' await UpdateIndicesWithAddedDocuments(lastSyncPoint, currentSyncPoint, updatedCount.Value, writer, cancellationToken); atLeastOneUpdate = true; } else { _logger.LogDebug("Expected number of updated documents: none "); } if (deletedCount.HasValue && deletedCount.Value > 0) { _logger.LogDebug("Expected number of deleted documents: {0}", deletedCount.Value); await UpdateIndicesWithDeletedDocuments(lastSyncPoint, currentSyncPoint, deletedCount.Value, writer, cancellationToken); atLeastOneUpdate = true; } else { _logger.LogDebug("Expected number of updated documents: none "); } if (atLeastOneUpdate) { _logger.LogDebug("Expected number of updated documents: none "); _luceneReaderService.Evict(); writer.Flush(triggerMerge: true, applyAllDeletes: true); _logger.LogInformation("Indexes are updated"); } //update LastSyncPoint using (var dbConnection = await _SQLservice.GetConnection(cancellationToken)) { var dbCommand = @"UPDATE [dbo].[FTS_Config] SET LastSyncPoint = @currentSyncPoint"; var cmd = new SqlCommand(dbCommand, dbConnection); cmd.Parameters.Add("@currentSyncPoint", System.Data.SqlDbType.DateTimeOffset); cmd.Parameters["@currentSyncPoint"].Value = currentSyncPoint; try { await _SQLservice.ExecuteNonQueryWithRetryAsync(cmd, cancellationToken); _logger.LogDebug("Last sync point is set to {0}", currentSyncPoint); } catch (Exception ex) { _logger.LogError(ex, "unexpected failure to update LastSyncPoint in database"); throw; } } var result = new RebuildIndicesResponse { IsValid = true, Success = true, NumberOfUpdates = updatedCount, NumberOfDeletes = deletedCount, CurrentSyncPoint = currentSyncPoint }; return(result); } catch (LockObtainFailedException) { var result = new RebuildIndicesResponse(); result.IsValid = false; result.Errors = new List <string>(); result.Errors.Add("Failed to lock full text search index file. Probaly there is another job is running. Please try again later."); return(result); } catch (Exception ex) { var result = new RebuildIndicesResponse(); result.IsValid = false; result.Errors = new List <string>(); result.Errors.Add("Unexpected error occured: " + ex.Message); return(result); } finally { if (writer != null) { writer.Dispose(); } if (azureDirectory != null) { azureDirectory.Dispose(); } } }