/// <summary>
/// Creates a connection to Azure Storage from an account name, key, and endpoints
/// </summary>
public AzureStorageAccount(
string accountName,
string accountKey,
Uri?blobStorageEndpoint,
Uri?queueStorageEndpoint,
Uri?tableStorageEndpoint,
Uri?fileStorageEndpoint)
{
if ((blobStorageEndpoint == null) &&
(queueStorageEndpoint == null) &&
(tableStorageEndpoint == null) &&
(fileStorageEndpoint == null))
{
throw new ArgumentException($"{nameof(AzureStorageAccount)} must have at least one remote URI to connect to");
}
Microsoft.Azure.Storage.Auth.StorageCredentials credentials = new Microsoft.Azure.Storage.Auth.StorageCredentials(accountName, accountKey);
_account = new Microsoft.Azure.Storage.CloudStorageAccount(credentials, blobStorageEndpoint, queueStorageEndpoint, tableStorageEndpoint, fileStorageEndpoint);
_storageContainers = new Dictionary <string, AzureBlobStorage>();
if (tableStorageEndpoint != null)
{
Microsoft.Azure.Cosmos.Table.StorageCredentials tableCredentials = new Microsoft.Azure.Cosmos.Table.StorageCredentials(accountName, accountKey);
_tableAccount = new Microsoft.Azure.Cosmos.Table.CloudStorageAccount(tableCredentials, tableStorageEndpoint);
}
}
public async Task OnPostAsync(string firstname, string lastname, IFormFile file)
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString);
var sbc = storageAccount.CreateCloudBlobClient();
//Blob Containers
var writeContainer = sbc.GetContainerReference("samples-images");
var readContainer = sbc.GetContainerReference("sample-images-sm");
await writeContainer.CreateIfNotExistsAsync();
await readContainer.CreateIfNotExistsAsync();
// Queue
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
CloudQueue queue = queueClient.GetQueueReference("bjornosqueue");
await PublishToQueueAsync(queue, new Person(firstname, lastname));
PublishToBlobStorage(writeContainer, $"{firstname}_{lastname}.png", file);
// De person die net is gepost zit nog in de queue deze moet nog door de azure function
// naar table gestorage gezet worden Oftewel wij lopen hier in principe 1 post achter En
// dat is prima voor test doeleinden
var selectAllQuery = new TableQuery <Person>();
var account = CosmosCloudStorageAccount.Parse(connectionString);
var client = account.CreateCloudTableClient();
var table = client.GetTableReference("persons");
Persons = table.ExecuteQuery(selectAllQuery).ToList();
}
/// <summary>
/// Constructs the data context
/// </summary>
/// <param name="account"></param>
/// <param name="storageAccount"></param>
public TableDataContext(CloudTableClient account, CloudStorageAccount storageAccount)
{
if (account == null)
{
throw new ArgumentNullException(nameof(account));
}
_storageAccount = storageAccount ?? throw new ArgumentNullException(nameof(storageAccount));
_entrantTable = account.GetTableReference("entrants");
}
public async Task <LuceneIndexAndMetadata> GetReaderInternal(CancellationToken cancellationToken)
{
var resultObject = new LuceneIndexAndMetadata();
//Azure configuration
var accountSAS = new Microsoft.Azure.Storage.Auth.StorageCredentials(_azureLuceneConfiguration.SASToken);
var accountWithSAS = new Microsoft.Azure.Storage.CloudStorageAccount(accountSAS, _azureLuceneConfiguration.AzureStorageAccountName, endpointSuffix: null, useHttps: true);
_logger.LogInformation("_azureLuceneConfiguration.TempDirectory = {0}", _azureLuceneConfiguration.TempDirectory);
var tempLocation = _azureLuceneConfiguration.TempDirectory ?? "temp";
_logger.LogDebug("Lucene IndexReader is located in {0} azure storage account (container '{1}')"
, _azureLuceneConfiguration.AzureStorageAccountName
, _azureLuceneConfiguration.Container);
var azureDirectory = new Lucene.Net.Store.Azure.AzureDirectory(accountWithSAS, tempLocation, containerName: _azureLuceneConfiguration.Container);
//ensure RAMDirectory
azureDirectory.CacheDirectory = new Lucene.Net.Store.RAMDirectory();
var reader = DirectoryReader.Open(azureDirectory);
_logger.LogDebug("Lucene IndexReader is acquired.");
resultObject.Index = reader;
using (var dbConnection = await _SQLservice.GetConnection(cancellationToken))
{
//we need last sync point only if it is not full rebuild
var dbCommand = @"SELECT TOP 1 LastSyncPoint FROM [dbo].[FTS_Config]";
var cmd = new SqlCommand(dbCommand, dbConnection);
try
{
var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken);
var lastSyncPointNullable = untyped as DateTimeOffset?;
if (lastSyncPointNullable.HasValue)
{
resultObject.LastIndexOffset = lastSyncPointNullable.Value;
}
_logger.LogDebug("Last sync point is {0}", lastSyncPointNullable.HasValue ? lastSyncPointNullable.Value.ToString() : "'never'");
}
catch (Exception ex)
{
_logger.LogError(ex, "unexpected failure to acquire LastSyncPoint from database");
throw;
}
}
return(resultObject);
}
public CommandAuditWriter(ICommandAuditOptions options = null)
{
auditContainer = new Lazy <CloudBlobContainer>(() => BlobCloudStorageAccount
.Parse((options ?? CommandAuditOptions.Default).ConnectionString)
.CreateCloudBlobClient().GetContainerReference(GetAuditContainerName(options)));
auditTable = new Lazy <CloudTable>(() => TableCloudStorageAccount
.Parse((options ?? CommandAuditOptions.Default).ConnectionString)
.CreateCloudTableClient().GetTableReference(GetAuditTableName(options)));
}
static CloudTable GetCosmosOrAzureDBTable()
{
Microsoft.Azure.Storage.CloudStorageAccount storageAccount = Microsoft.Azure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageAccount"));
Microsoft.Azure.CosmosDB.Table.CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference("customers");
//cleanup
table.DeleteIfExists();
table.CreateIfNotExists();
return(table);
}
public static AzureStorageContext GetStorageContextFromTrack1FileServiceClient(CloudFileClient fileServiceClient, IAzureContext DefaultContext = null)
{
Microsoft.Azure.Storage.CloudStorageAccount account = new Microsoft.Azure.Storage.CloudStorageAccount(
fileServiceClient.Credentials,
null, //blob Uri
null, //queue Uri
null, //talbe Uri
fileServiceClient.BaseUri); //file Uri
return(new AzureStorageContext(account,
fileServiceClient.Credentials.AccountName,
DefaultContext));
}
public CoreIdGenerator(IConfiguration config)
{
var storageConnection = config.GetSection("ConnectionStrings:AzureBlobStorage").Get <BlobStorageConnection>();
var batchSize = config.GetValue <int>("IdGeneratorBatchSize");
_cloudStorageAccount = Microsoft.Azure.Storage.CloudStorageAccount.Parse(storageConnection.ConnectionString);
_containerName = storageConnection.ContainerName;
_dataStore = BlobOptimisticDataStore.CreateAsync(_cloudStorageAccount, _containerName).Result;
_generator = new UniqueIdGenerator(_dataStore)
{
BatchSize = batchSize
};
}
public static Stream DownloadObject(string storageName, string objectName, string name, string serviceAccountName, string serviceAccountKey, string clientId, Stream objectStream)
{
if (objectStream is null)
{
throw new ArgumentNullException(nameof(objectStream));
}
StorageCredentials storageCredentials = new StorageCredentials(serviceAccountName, serviceAccountKey);
Microsoft.Azure.Storage.CloudStorageAccount account = new Microsoft.Azure.Storage.CloudStorageAccount(storageCredentials, true);
///TO DO : Get all other thins done download file or something
return(objectStream);
}
public AzureJobStorage(AzureStorageOptions options)
{
_options = options;
_cosmosAccount = CloudStorageAccount.Parse(options.ConnectionString);
_account = Microsoft.Azure.Storage.CloudStorageAccount.Parse(options.ConnectionString);
_tableClient = _cosmosAccount.CreateCloudTableClient();
_blobClient = _account.CreateCloudBlobClient();
_queueClient = _account.CreateCloudQueueClient();
// ensure the required tables / containers exists
GetTable(SERVER_TABLE).CreateIfNotExists();
GetTable(SETS_TABLE).CreateIfNotExists();
GetTable(LISTS_TABLE).CreateIfNotExists();
GetTable(JOBS_TABLE).CreateIfNotExists();
GetTable(HASHS_TABLE).CreateIfNotExists();
GetTable(COUNTERS_TABLE).CreateIfNotExists();
GetContainer(JOB_CONTAINER).CreateIfNotExists();
}
public BlobStorageService()
{
_cloudStorageAccount = CloudStorageAccount.DevelopmentStorageAccount;
_blobClient = _cloudStorageAccount.CreateCloudBlobClient();
}
public override void Configure(IFunctionsHostBuilder builder)
{
var services = builder.Services;
services.AddTransient <ISverigesRadioApiClient>(s => SverigesRadioApiClient.CreateClient());
services.AddTransient(x =>
{
var configuration = x.GetRequiredService <IConfiguration>();
var storageAccount = CloudStorageAccount.Parse(configuration["AzureStorage:BlobsConnectionString"]);
return(storageAccount.CreateCloudBlobClient());
});
services.AddTransient(x =>
{
var configuration = x.GetRequiredService <IConfiguration>();
var storageAccount = Microsoft.Azure.Cosmos.Table.CloudStorageAccount.Parse(configuration["AzureStorage:TablesConnectionString"]);
return(storageAccount.CreateCloudTableClient(new TableClientConfiguration()));
});
services.AddTransient(x =>
{
var configuration = x.GetRequiredService <IConfiguration>();
return(SpeechConfig.FromSubscription(configuration["AzureSpeech:Key"], configuration["AzureSpeech:Region"]));
});
services.AddTransient(x =>
{
var configuration = x.GetRequiredService <IConfiguration>();
return(SpeechBatchClient.CreateApiV2Client(configuration["AzureSpeech:Key"], configuration["AzureSpeech:Hostname"], 443));
});
services.AddTransient(x =>
{
var configuration = x.GetRequiredService <IConfiguration>();
var credentials = new ApiKeyServiceClientCredentials(configuration["AzureTextAnalytics:Key"]);
return(new TextAnalyticsClient(credentials)
{
Endpoint = configuration["AzureTextAnalytics:Endpoint"]
});
});
services.AddTransient(x =>
{
var configuration = x.GetRequiredService <IConfiguration>();
return(TranslatorClient.CreateClient(configuration["AzureTranslator:Key"], configuration["AzureTranslator:Endpoint"]));
});
services.AddTransient <IStorageTransfer, AzureStorageTransfer>();
services.AddTransient <IStorage, AzureTableStorage>(s =>
{
var configuration = s.GetRequiredService <IConfiguration>();
return(new AzureTableStorage(
s.GetRequiredService <CloudTableClient>(),
configuration["AzureStorage:EpisodeStatusesTableName"],
configuration["AzureStorage:EpisodesTableName"],
configuration["AzureStorage:EpisodeTranscriptionsTableName"],
configuration["AzureStorage:EpisodeTextAnalyticsTableName"],
configuration["AzureStorage:EpisodeSpeechTableName"]
));
});
services.AddTransient <ISummaryStorage, SummaryAzureTableStorage>(s =>
{
var configuration = s.GetRequiredService <IConfiguration>();
return(new SummaryAzureTableStorage(
s.GetRequiredService <CloudTableClient>(),
configuration["AzureStorage:EpisodeSummaryTableName"]
));
});
services.AddTransient <SrEpisodesLister>();
services.AddTransient(s =>
{
var configuration = s.GetRequiredService <IConfiguration>();
return(new SrEpisodeCollector(
configuration["AzureStorage:AudioContainerName"],
s.GetRequiredService <IStorageTransfer>(),
s.GetRequiredService <ISverigesRadioApiClient>(),
s.GetRequiredService <ILogger <SrEpisodeCollector> >(),
s.GetRequiredService <IStorage>()
));
});
services.AddTransient(s =>
{
var configuration = s.GetRequiredService <IConfiguration>();
return(new SrEpisodeTranscriber(
configuration["AzureStorage:EpisodeTranscriptionsContainerName"],
s.GetRequiredService <ISpeechBatchClientFactory>(),
s.GetRequiredService <IStorageTransfer>(),
s.GetRequiredService <ILogger <SrEpisodeCollector> >(),
s.GetRequiredService <IStorage>(),
s.GetRequiredService <CloudBlobClient>()
));
});
services.AddTransient <SrEpisodeTextEnricher>();
services.AddTransient <SrEpisodeSummarizer>();
services.AddTransient(s =>
{
var configuration = s.GetRequiredService <IConfiguration>();
return(new SrEpisodeSpeaker(
configuration["AzureStorage:EpisodeSpeechContainerName"],
s.GetRequiredService <ISpeechConfigFactory>(),
s.GetRequiredService <IStorage>(),
s.GetRequiredService <ILogger <SrEpisodeSpeaker> >(),
s.GetRequiredService <CloudBlobClient>()
));
});
services.AddTransient <SrWorker>();
}
/// <summary>
/// Creates an image manager along with storage dependencies, if needed
/// </summary>
/// <param name="tableStorageAccount">
/// The storage account used to save original and optimized images
/// </param>
/// <param name="imageTableName">
/// The name of the table storing the relationship between original
/// and optimized images
/// </param>
/// <param name="optimizer">
/// The image optimizer used to optimize images
/// </param>
/// <returns></returns>
public static async Task <ImageManager> CreateAsync(CloudStorageAccount tableStorageAccount, Microsoft.Azure.Storage.CloudStorageAccount blobStorageAccount, string imageTableName, IImageOptimizer optimizer)
{
var tableClient = tableStorageAccount.CreateCloudTableClient();
var table = tableClient.GetTableReference(imageTableName);
await table.CreateIfNotExistsAsync().ConfigureAwait(false);
return(new ImageManager(table, blobStorageAccount.CreateCloudBlobClient(), optimizer));
}
public static IHostBuilder CreateHostBuilder(string[] args) =>
Host.CreateDefaultBuilder(args)
.ConfigureServices((hostContext, services) =>
{
ServicePointManager.DefaultConnectionLimit = 100;
services.AddSingleton <ISverigesRadioApiClient>(s => SverigesRadioApiClient.CreateClient());
services.AddTransient(x =>
{
var storageAccount = CloudStorageAccount.Parse(hostContext.Configuration["AzureStorage:BlobsConnectionString"]);
return(storageAccount.CreateCloudBlobClient());
});
services.AddTransient(x =>
{
var storageAccount = Microsoft.Azure.Cosmos.Table.CloudStorageAccount.Parse(hostContext.Configuration["AzureStorage:TablesConnectionString"]);
return(storageAccount.CreateCloudTableClient(new TableClientConfiguration()));
});
var azureSpeechClients = new List <SpeechBatchClientOptions>();
hostContext.Configuration.GetSection("AzureSpeech:Clients").Bind(azureSpeechClients);
services.AddSingleton <ISpeechConfigFactory>(x => new RoundRobinSpeechConfigFactory(azureSpeechClients));
services.AddSingleton <ISpeechBatchClientFactory>(x => new RoundRobinSpeechBatchClientFactory(azureSpeechClients));
services.AddTransient(x =>
{
var credentials = new ApiKeyServiceClientCredentials(hostContext.Configuration["AzureTextAnalytics:Key"]);
return(new TextAnalyticsClient(credentials)
{
Endpoint = hostContext.Configuration["AzureTextAnalytics:Endpoint"]
});
});
services.AddTransient(x => TranslatorClient.CreateClient(hostContext.Configuration["AzureTranslator:Key"], hostContext.Configuration["AzureTranslator:Endpoint"]));
services.AddTransient <IStorageTransfer, AzureStorageTransfer>();
services.AddTransient <IStorage, AzureTableStorage>(s => new AzureTableStorage(
s.GetRequiredService <CloudTableClient>(),
hostContext.Configuration["AzureStorage:EpisodeStatusesTableName"],
hostContext.Configuration["AzureStorage:EpisodesTableName"],
hostContext.Configuration["AzureStorage:EpisodeTranscriptionsTableName"],
hostContext.Configuration["AzureStorage:EpisodeTextAnalyticsTableName"],
hostContext.Configuration["AzureStorage:EpisodeSpeechTableName"]
));
services.AddTransient <ISummaryStorage, SummaryAzureTableStorage>(s => new SummaryAzureTableStorage(
s.GetRequiredService <CloudTableClient>(),
hostContext.Configuration["AzureStorage:EpisodeSummaryTableName"]
));
services.AddTransient <IWordCountStorage, WordCounterAzureTableStorage>(s =>
{
var storageAccount = Microsoft.Azure.Cosmos.Table.CloudStorageAccount.Parse(hostContext.Configuration["AzureStorage:BlobsConnectionString"]);
var tableClient = storageAccount.CreateCloudTableClient(new TableClientConfiguration());
return(new WordCounterAzureTableStorage(
tableClient,
hostContext.Configuration["AzureStorage:EpisodeWordCountTableName"]
));
});
services.AddTransient <SrEpisodesLister>();
services.AddTransient(s => new SrEpisodeCollector(
hostContext.Configuration["AzureStorage:AudioContainerName"],
s.GetRequiredService <IStorageTransfer>(),
s.GetRequiredService <ISverigesRadioApiClient>(),
s.GetRequiredService <ILogger <SrEpisodeCollector> >(),
s.GetRequiredService <IStorage>(),
hostContext.Configuration["FFMpeg:Location"]
));
services.AddTransient(s => new SrEpisodeTranscriber(
hostContext.Configuration["AzureStorage:EpisodeTranscriptionsContainerName"],
s.GetRequiredService <ISpeechBatchClientFactory>(),
s.GetRequiredService <IStorageTransfer>(),
s.GetRequiredService <ILogger <SrEpisodeCollector> >(),
s.GetRequiredService <IStorage>(),
s.GetRequiredService <CloudBlobClient>()
));
services.AddTransient <SrEpisodeTextEnricher>();
services.AddTransient <SrEpisodeSummarizer>();
services.AddTransient(s => new SrEpisodeSpeaker(
hostContext.Configuration["AzureStorage:EpisodeSpeechContainerName"],
s.GetRequiredService <ISpeechConfigFactory>(),
s.GetRequiredService <IStorage>(),
s.GetRequiredService <ILogger <SrEpisodeSpeaker> >(),
s.GetRequiredService <CloudBlobClient>()
));
services.AddTransient <SrEpisodeWordCounter>();
services.AddTransient <SrWorker>();
services.AddHostedService <Worker>();
});
public async Task InitializeAsync()
{
string nowString = DateTime.UtcNow.ToString("yyMMdd-HHmmss");
string GetDestPath(int counter)
{
return(Path.Combine(Path.GetTempPath(), "FunctionsE2E", $"{nowString}_{counter}"));
}
// Prevent collisions.
int i = 0;
_copiedRootPath = GetDestPath(i++);
while (Directory.Exists(_copiedRootPath))
{
_copiedRootPath = GetDestPath(i++);
}
FileUtility.CopyDirectory(_rootPath, _copiedRootPath);
var extensionsToInstall = GetExtensionsToInstall();
if (extensionsToInstall != null && extensionsToInstall.Length > 0)
{
TestFunctionHost.WriteNugetPackageSources(_copiedRootPath, "http://www.myget.org/F/azure-appservice/api/v2", "https://www.myget.org/F/azure-appservice-staging/api/v2", "https://api.nuget.org/v3/index.json");
var options = new OptionsWrapper <ScriptJobHostOptions>(new ScriptJobHostOptions
{
RootScriptPath = _copiedRootPath
});
var manager = new ExtensionsManager(options, NullLogger <ExtensionsManager> .Instance, new TestExtensionBundleManager());
await manager.AddExtensions(extensionsToInstall);
}
string logPath = Path.Combine(Path.GetTempPath(), @"Functions");
if (!string.IsNullOrEmpty(_functionsWorkerRuntime))
{
Environment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, _functionsWorkerRuntime);
Environment.SetEnvironmentVariable(RpcWorkerConstants.FunctionsWorkerProcessCountSettingName, _workerProcessCount.ToString());
}
if (!string.IsNullOrEmpty(_functionsWorkerRuntimeVersion))
{
Environment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeVersionSettingName, _functionsWorkerRuntimeVersion);
}
FunctionsSyncManagerMock = new Mock <IFunctionsSyncManager>(MockBehavior.Strict);
FunctionsSyncManagerMock.Setup(p => p.TrySyncTriggersAsync(It.IsAny <bool>())).ReturnsAsync(new SyncTriggersResult {
Success = true
});
Host = new TestFunctionHost(_copiedRootPath, logPath,
configureScriptHostWebJobsBuilder: webJobsBuilder =>
{
ConfigureScriptHost(webJobsBuilder);
},
configureScriptHostServices: s =>
{
s.AddSingleton <IFunctionsSyncManager>(_ => FunctionsSyncManagerMock.Object);
s.AddSingleton <IMetricsLogger>(_ => MetricsLogger);
},
configureWebHostServices: s =>
{
s.AddSingleton <IEventGenerator>(_ => EventGenerator);
ConfigureWebHost(s);
});
string connectionString = Host.JobHostServices.GetService <IConfiguration>().GetWebJobsConnectionString(ConnectionStringNames.Storage);
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString);
QueueClient = storageAccount.CreateCloudQueueClient();
BlobClient = storageAccount.CreateCloudBlobClient();
TableStorageAccount tableStorageAccount = TableStorageAccount.Parse(connectionString);
TableClient = tableStorageAccount.CreateCloudTableClient();
await CreateTestStorageEntities();
MasterKey = await Host.GetMasterKeyAsync();
}
public async Task <RebuildIndicesResponse> Handle(RebuildIndicesRequest request, CancellationToken cancellationToken)
{
_logger.LogDebug("RebuildIndicesResponseHandler started.");
cancellationToken.ThrowIfCancellationRequested();
IndexWriter writer = null;
Lucene.Net.Store.Azure.AzureDirectory azureDirectory = null;
DateTimeOffset lastSyncPoint = DateTimeOffset.MinValue;
DateTimeOffset currentSyncPoint = DateTimeOffset.Now;
int? updatedCount = null;
int? deletedCount = null;
try
{
// Ensures index backwards compatibility
var AppLuceneVersion = LuceneVersion.LUCENE_48;
//Azure configuration
var accountSAS = new Microsoft.Azure.Storage.Auth.StorageCredentials(AzureLuceneConfiguration.SASToken);
var accountWithSAS = new Microsoft.Azure.Storage.CloudStorageAccount(accountSAS, AzureLuceneConfiguration.AzureStorageAccountName, endpointSuffix: null, useHttps: true);
var tempLocation = AzureLuceneConfiguration.TempDirectory ?? "temp";
_logger.LogTrace("tempLocation: {0}", tempLocation);
azureDirectory = new Lucene.Net.Store.Azure.AzureDirectory(accountWithSAS, tempLocation, containerName: AzureLuceneConfiguration.Container);
//ensure RAMDirectory
azureDirectory.CacheDirectory = new RAMDirectory();
//create an analyzer to process the text
var analyzer = new StandardAnalyzer(AppLuceneVersion);
//create an index writer
var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer);
writer = new IndexWriter(azureDirectory, indexConfig); //used to be dir
_logger.LogTrace("IndexWriter is initialized");
if (request.FullRebuild)
{
_logger.LogInformation("Full Rebuild is requested. Deleting indices");
writer.DeleteAll();
writer.Commit();
_logger.LogTrace("Full Rebuild is committed.");
}
using (var dbConnection = await _SQLservice.GetConnection(cancellationToken))
{
SqlCommand cmd;
if (!request.FullRebuild)
{
//we need last sync point only if it is not full rebuild
var dbCommand = @"SELECT TOP 1 LastSyncPoint FROM [dbo].[FTS_Config]";
cmd = new SqlCommand(dbCommand, dbConnection);
try
{
var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken);
var lastSyncPointNullable = untyped as DateTimeOffset?;
if (lastSyncPointNullable.HasValue)
{
lastSyncPoint = lastSyncPointNullable.Value;
}
_logger.LogDebug("Last sync point is {0}", lastSyncPointNullable.HasValue ? lastSyncPointNullable.Value.ToString() : "'never'");
}
catch (Exception ex)
{
_logger.LogError(ex, "unexpected failure to acquire LastSyncPoint from database");
throw;
}
}
else
{
lastSyncPoint = DateTimeOffset.MinValue;
}
//determine number of records that will need to be processed
var dbCountCommand = @"SELECT COUNT(Id) from [dbo].[Test_Data] WHERE UpdatedAt >= @lastSyncPoint AND UpdatedAt < @currentSyncPoint AND DeletedAt IS NULL";
cmd = new SqlCommand(dbCountCommand, dbConnection);
cmd.Parameters.Add("@lastSyncPoint", System.Data.SqlDbType.DateTimeOffset);
cmd.Parameters["@lastSyncPoint"].Value = lastSyncPoint;
cmd.Parameters.Add("@currentSyncPoint", System.Data.SqlDbType.DateTimeOffset);
cmd.Parameters["@currentSyncPoint"].Value = currentSyncPoint;
try
{
var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken);
updatedCount = untyped as int?;
_logger.LogDebug("Expected number of updated documents {0}", updatedCount.HasValue ? updatedCount.Value.ToString() : "'none'");
}
catch (Exception ex)
{
_logger.LogError(ex, "unexpected failure to acquire number of documents to be updated from database");
throw;
}
//working on deleted documents
if (!request.FullRebuild)
{
//also need to remove "Deleted" documents. Only if not full rebuild of indices
var dbDeletedCountCommand = @"SELECT COUNT(Id) from [dbo].[Test_Data] WHERE DeletedAt >= @lastSyncPoint AND DeletedAt<=@currentSyncPoint AND DeletedAt IS NOT NULL";
cmd = new SqlCommand(dbDeletedCountCommand, dbConnection);
cmd.Parameters.Add("@lastSyncPoint", System.Data.SqlDbType.DateTimeOffset);
cmd.Parameters["@lastSyncPoint"].Value = lastSyncPoint;
cmd.Parameters.Add("@currentSyncPoint", System.Data.SqlDbType.DateTimeOffset);
cmd.Parameters["@currentSyncPoint"].Value = currentSyncPoint;
try
{
var untyped = await _SQLservice.ExecuteScalarWithRetryAsync(cmd, cancellationToken);
deletedCount = untyped as int?;
_logger.LogDebug("Expected number of deleted documents {0}", deletedCount.HasValue ? updatedCount.Value.ToString() : "'none'");
}
catch (Exception ex)
{
_logger.LogError(ex, "unexpected failure to acquire 'number of documents to be delete from indicies' from database");
throw;
}
}
}
var atLeastOneUpdate = false;
if (updatedCount.HasValue && updatedCount.Value > 0)
{
_logger.LogDebug("Expected number of updated documents: {0}", updatedCount.Value);
//Start updating 'Updated records'
await UpdateIndicesWithAddedDocuments(lastSyncPoint, currentSyncPoint, updatedCount.Value, writer, cancellationToken);
atLeastOneUpdate = true;
}
else
{
_logger.LogDebug("Expected number of updated documents: none ");
}
if (deletedCount.HasValue && deletedCount.Value > 0)
{
_logger.LogDebug("Expected number of deleted documents: {0}", deletedCount.Value);
await UpdateIndicesWithDeletedDocuments(lastSyncPoint, currentSyncPoint, deletedCount.Value, writer, cancellationToken);
atLeastOneUpdate = true;
}
else
{
_logger.LogDebug("Expected number of updated documents: none ");
}
if (atLeastOneUpdate)
{
_logger.LogDebug("Expected number of updated documents: none ");
_luceneReaderService.Evict();
writer.Flush(triggerMerge: true, applyAllDeletes: true);
_logger.LogInformation("Indexes are updated");
}
//update LastSyncPoint
using (var dbConnection = await _SQLservice.GetConnection(cancellationToken))
{
var dbCommand = @"UPDATE [dbo].[FTS_Config] SET LastSyncPoint = @currentSyncPoint";
var cmd = new SqlCommand(dbCommand, dbConnection);
cmd.Parameters.Add("@currentSyncPoint", System.Data.SqlDbType.DateTimeOffset);
cmd.Parameters["@currentSyncPoint"].Value = currentSyncPoint;
try
{
await _SQLservice.ExecuteNonQueryWithRetryAsync(cmd, cancellationToken);
_logger.LogDebug("Last sync point is set to {0}", currentSyncPoint);
}
catch (Exception ex)
{
_logger.LogError(ex, "unexpected failure to update LastSyncPoint in database");
throw;
}
}
var result = new RebuildIndicesResponse
{
IsValid = true,
Success = true,
NumberOfUpdates = updatedCount,
NumberOfDeletes = deletedCount,
CurrentSyncPoint = currentSyncPoint
};
return(result);
}
catch (LockObtainFailedException)
{
var result = new RebuildIndicesResponse();
result.IsValid = false;
result.Errors = new List <string>();
result.Errors.Add("Failed to lock full text search index file. Probaly there is another job is running. Please try again later.");
return(result);
}
catch (Exception ex)
{
var result = new RebuildIndicesResponse();
result.IsValid = false;
result.Errors = new List <string>();
result.Errors.Add("Unexpected error occured: " + ex.Message);
return(result);
}
finally
{
if (writer != null)
{
writer.Dispose();
}
if (azureDirectory != null)
{
azureDirectory.Dispose();
}
}
}
/// <summary>
/// Creates a connection to Azure Storage from a connection string
/// </summary>
public AzureStorageAccount(string connectionString)
{
_account = Microsoft.Azure.Storage.CloudStorageAccount.Parse(connectionString);
_tableAccount = Microsoft.Azure.Cosmos.Table.CloudStorageAccount.Parse(connectionString);
_storageContainers = new Dictionary <string, AzureBlobStorage>();
}
public async Task TestAzureOnlyNoLocal()
{
string strFishPath = TestContent.GetContentFilePhysicalPath("/Images/Jellyfish.jpg");
Microsoft.Azure.Storage.CloudStorageAccount objCDNClient
= Microsoft.Azure.Storage.CloudStorageAccount.Parse(UnitTestContext.AzureStorageConnectionString);
IFileServer server1 = new FileServerAzure(objCDNClient, UnitTestContext.AzureBucket);
//Test 1: Delete a file if it exists, then store an image, check it's existance, get it's properties, and compare file versions
IFileQuery file1 = new FileQuery("Jellyfish.jpg", "Images");
if (server1.FileExistsInCDN(file1) || server1.FileExistsLocal(file1))
{
server1.Delete(file1);
}
Assert.IsFalse(server1.FileExistsInCDN(file1));
Assert.IsFalse(server1.FileExistsLocal(file1));
file1.MetaData.Add("test", "Value");
server1.StoreImage(strFishPath, file1);
var props = server1.GetFilePropertiesFromCDN(file1);
props.MetaData.Remove("nothing");
Assert.IsFalse(server1.FileExistsLocal(file1));
Assert.IsTrue(server1.FileExistsInCDN(file1));
var props1CDN = server1.GetFilePropertiesFromCDN(file1);
var props1Local = server1.GetFilePropertiesLocal(file1);
Assert.IsNotNull(props1CDN);
Assert.IsNull(props1Local);
//Test 2: Store a file from a Unicode text string, then retreive it and compare the strings
IFileQuery file2 = new FileQuery("Test.txt", "Scripts");
server1.StoreFileFromString("this is some text § how was that", file2);
//server2.StoreFileFromString("some text", file2);
string strFile2Text = server1.LoadFileText(file2);
Assert.AreEqual(strFile2Text, "this is some text § how was that");
server1.DeleteFileLocal(file2);
Assert.IsFalse(server1.FileExistsLocal(file2));
string test = server1.LoadFileText(file2);
Assert.AreEqual(test, "this is some text § how was that");
//Test 3: Make sure that this file, which was never uploaded, reflects that reality
IFileQuery file3 = new FileQuery("Redfish.jpg", "uploads/images");
Assert.IsFalse(server1.FileExistsLocal(file3));
Assert.IsFalse(server1.FileExistsInCDN(file3));
var strURL3 = server1.GetCDNURL(file3);
Assert.IsTrue(Model.URL.IsValid(strURL3));
var props2 = server1.GetFilePropertiesFromCDN(file3);
Assert.IsNull(props2);
//Test 4: Test each upload method
string strKoalaPath = TestContent.GetContentFilePhysicalPath("/Images/Koala.jpg");
System.Drawing.Image imgKoala = System.Drawing.Image.FromFile(strKoalaPath);
IFileQuery file4 = new FileQuery("Upload.jpg", "Images");
string strURL;
file4.FileName = "UploadFromFile.jpg";
strURL = server1.GetCDNURL(file4);
server1.StoreImage(strKoalaPath, file4);
Assert.IsFalse(server1.FileExistsLocal(file4));
Assert.IsTrue(server1.FileExistsInCDN(file4));
file4.FileName = "UploadFromImage.jpg";
strURL = server1.GetCDNURL(file4);
server1.StoreImage(imgKoala, file4);
Assert.IsFalse(server1.FileExistsLocal(file4));
Assert.IsTrue(server1.FileExistsInCDN(file4));
file4.FileName = "UploadFromImage.png";
strURL = server1.GetCDNURL(file4);
server1.StoreImage(imgKoala, file4, System.Drawing.Imaging.ImageFormat.Png);
Assert.IsFalse(server1.FileExistsLocal(file4));
Assert.IsTrue(server1.FileExistsInCDN(file4));
file4.FileName = "UploadFromStream.png";
strURL = server1.GetCDNURL(file4);
var stream = new System.IO.MemoryStream();
imgKoala.Save(stream, System.Drawing.Imaging.ImageFormat.Png);
stream.Position = 0;
server1.StoreImage(stream, file4);
Assert.IsFalse(server1.FileExistsLocal(file4));
Assert.IsTrue(server1.FileExistsInCDN(file4));
file4.FileName = "UploadFromFileAsFile.jpg";
strURL = server1.GetCDNURL(file4);
server1.StoreFile(strKoalaPath, file4);
Assert.IsFalse(server1.FileExistsLocal(file4));
Assert.IsTrue(server1.FileExistsInCDN(file4));
file4.FileName = "UploadFromStreamAsFile.jpg";
strURL = server1.GetCDNURL(file4);
var stream2 = new System.IO.MemoryStream();
imgKoala.Save(stream2, System.Drawing.Imaging.ImageFormat.Jpeg);
stream2.Position = 0;
server1.StoreFile(stream2, file4);
Assert.IsFalse(server1.FileExistsLocal(file4));
Assert.IsTrue(server1.FileExistsInCDN(file4));
}
public static Microsoft.AspNetCore.DataProtection.IDataProtectionBuilder PersistKeysToAzureBlobStorage(this Microsoft.AspNetCore.DataProtection.IDataProtectionBuilder builder, Microsoft.Azure.Storage.CloudStorageAccount storageAccount, string relativePath)
{
throw null;
}
