private async Task AssertTableHasExpectedColumnsAsync(QueuedTaskMapping mapping)
{
bool tableHasColumns = await TableHasColumnsAsync(mapping.ResultsQueueTableName,
QueueResultTableDbAssetSetup.TaskIdColumnName,
QueueResultTableDbAssetSetup.TaskTypeColumnName,
QueueResultTableDbAssetSetup.TaskStatusColumnName,
QueueResultTableDbAssetSetup.TaskSourceColumnName,
QueueResultTableDbAssetSetup.TaskPayloadColumnName,
QueueResultTableDbAssetSetup.TaskPriorityColumnName,
QueueResultTableDbAssetSetup.TaskLastErrorColumnName,
QueueResultTableDbAssetSetup.TaskErrorCountColumnName,
QueueResultTableDbAssetSetup.TaskLastErrorIsRecoverableColumnName,
QueueResultTableDbAssetSetup.TaskProcessingTimeMillisecondsColumnName,
QueueResultTableDbAssetSetup.TaskPostedAtColumnName,
QueueResultTableDbAssetSetup.TaskFirstProcessingAttemptedAtColumnName,
QueueResultTableDbAssetSetup.TaskLastProcessingAttemptedAtColumnName,
QueueResultTableDbAssetSetup.TaskProcessingFinalizedAtColumnName);
Assert.IsTrue(tableHasColumns,
"Table {0} does not have all expected columns!",
mapping.ResultsQueueTableName);
}
private string GetDequeueFunctionCreationScript(QueuedTaskMapping mapping)
{
return($@"CREATE OR REPLACE FUNCTION public.{mapping.DequeueFunctionName}(
{SelectTypesParamName} character varying[],
{ExcludeIdsParamName} uuid[],
{RefNowParamName} timestamp with time zone)
RETURNS TABLE({TaskIdTableParamName} uuid,
{TaskLockHandleIdTableParamName} bigint,
{TaskTypeTableParamName} character varying,
{TaskSourceTableParamName} character varying,
{TaskPayloadTableParamName} text,
{TaskPriorityTableParamName} integer,
{TaskPostedAtTableParamName} timestamp with time zone,
{TaskLockedUntilTableParamName} timestamp with time zone)
LANGUAGE 'plpgsql'
AS $BODY$
DECLARE
n_select_types integer = CARDINALITY({SelectTypesParamName});
BEGIN
RETURN QUERY
WITH sk_dequeued_task AS
(DELETE FROM {mapping.QueueTableName} td WHERE td.task_id = (
SELECT t0.task_id
FROM {mapping.QueueTableName} t0
WHERE (t0.task_type = ANY({SelectTypesParamName}) OR n_select_types = 0)
AND t0.task_locked_until_ts < {RefNowParamName}
AND t0.task_id <> ALL({ExcludeIdsParamName})
ORDER BY t0.task_priority ASC,
t0.task_locked_until_ts ASC,
t0.task_lock_handle_id ASC
LIMIT 1
FOR UPDATE SKIP LOCKED
) RETURNING *) SELECT sdt.* FROM sk_dequeued_task sdt;
END;
$BODY$;" );
}
private StakhanoviseSetupDefaults MergeDefaultsFromConfig(StakhanoviseSetupDefaults targetDefaults,
StakhanoviseSetupDefaultsConfig defaultsConfig,
IConfiguration config)
{
ScriptOptions parseOptions = ConstructParseOptions();
Assembly[] executorAssemblies = ParseExecutorAssembliesFromConfig(defaultsConfig);
if (executorAssemblies != null)
{
targetDefaults.ExecutorAssemblies = executorAssemblies;
}
Func <IQueuedTaskToken, long> calculateDelayTicksTaskAfterFailureFn =
CompileCalculateDelayTicksTaskAfterFailureFnFromConfig(defaultsConfig, parseOptions);
if (calculateDelayTicksTaskAfterFailureFn != null)
{
targetDefaults.CalculateDelayMillisecondsTaskAfterFailure = calculateDelayTicksTaskAfterFailureFn;
}
Func <IQueuedTask, Exception, bool> isTaskErrorRecoverableFn =
CompileIsTaskErrorRecoverableFnFromConfig(defaultsConfig, parseOptions);
if (isTaskErrorRecoverableFn != null)
{
targetDefaults.IsTaskErrorRecoverable = isTaskErrorRecoverableFn;
}
if (!string.IsNullOrWhiteSpace(defaultsConfig.ConnectionStringName))
{
targetDefaults.ConnectionString = config.GetConnectionString(defaultsConfig.ConnectionStringName);
}
QueuedTaskMapping mappingFromConfig =
GetQueudTaskMappingFromDefaultsConfig(defaultsConfig);
targetDefaults.Mapping = MergeMappingFromConfig(targetDefaults.Mapping,
mappingFromConfig);
if (defaultsConfig.WorkerCount > 0)
{
targetDefaults.WorkerCount = defaultsConfig.WorkerCount;
}
if (defaultsConfig.FaultErrorThresholdCount > 0)
{
targetDefaults.FaultErrorThresholdCount = defaultsConfig.FaultErrorThresholdCount;
}
if (defaultsConfig.AppMetricsCollectionIntervalMilliseconds > 0)
{
targetDefaults.AppMetricsCollectionIntervalMilliseconds = defaultsConfig.AppMetricsCollectionIntervalMilliseconds;
}
if (defaultsConfig.AppMetricsMonitoringEnabled.HasValue)
{
targetDefaults.AppMetricsMonitoringEnabled = defaultsConfig.AppMetricsMonitoringEnabled.Value;
}
if (defaultsConfig.SetupBuiltInDbAsssets.HasValue)
{
targetDefaults.SetupBuiltInDbAsssets = defaultsConfig.SetupBuiltInDbAsssets.Value;
}
return(targetDefaults);
}
public async Task Test_CanCreateDequeueFunction_WithNonDefaultMapping()
{
QueuedTaskMapping mapping = GenerateNonDefaultMapping();
await RunDbAssetSetupTestsAsync(mapping);
}
public NpgsqlDataReaderExtensionsTests()
{
mMapping = TestOptions.DefaultMapping;
mOperations = new PostgreSqlTaskQueueDbOperations(ConnectionString, mMapping);
}
public PostgreSqlAppMetricsMonitorWriterOptions(ConnectionOptions connectionOptions, QueuedTaskMapping mapping)
{
ConnectionOptions = connectionOptions
?? throw new ArgumentNullException(nameof(connectionOptions));
Mapping = mapping
?? throw new ArgumentNullException(nameof(mapping));
}
private Mock <ISetupDbAsset> CreateDbAssetSetupMockWithoutError(ConnectionOptions connectionOptions, QueuedTaskMapping mapping)
{
Mock <ISetupDbAsset> dbAssetMock = new Mock <ISetupDbAsset>();
dbAssetMock.Setup(m => m.SetupDbAssetAsync(connectionOptions, mapping))
.Returns(Task.CompletedTask)
.Verifiable();
return(dbAssetMock);
}
public async Task SetupDbAssetAsync(ConnectionOptions queueConnectionOptions, QueuedTaskMapping mapping)
{
if (queueConnectionOptions == null)
{
throw new ArgumentNullException(nameof(queueConnectionOptions));
}
if (mapping == null)
{
throw new ArgumentNullException(nameof(mapping));
}
using (NpgsqlConnection conn = await queueConnectionOptions.TryOpenConnectionAsync())
{
using (NpgsqlCommand cmdTable = new NpgsqlCommand(GetDbTableCreationScript(mapping), conn))
await cmdTable.ExecuteNonQueryAsync();
}
}
public TaskQueueInfoOptions(ConnectionOptions connectionOptions,
QueuedTaskMapping mapping)
: base(connectionOptions, mapping)
{
return;
}
public Program()
{
mMapping = new QueuedTaskMapping();
mConfiguration = GetConfig();
}
private string GetTaskAcquireSql(QueuedTaskMapping mapping)
{
return($@"DELETE FROM {mapping.QueueTableName}
WHERE task_id = @t_id" );
}
private string GetTaskDequeueSql(QueuedTaskMapping mapping)
{
//See https://dba.stackexchange.com/questions/69471/postgres-update-limit-1/69497#69497
return($@"SELECT tq.* FROM { mapping.DequeueFunctionName }(@types, @excluded, @ref_now) tq");
}
public StakhanoviseSetup(StakhanoviseSetupDefaults defaults)
{
if (defaults == null)
{
throw new ArgumentNullException(nameof(defaults));
}
mMapping = defaults.Mapping;
mSetupBuiltInDbAsssets = defaults
.SetupBuiltInDbAsssets;
mAppMetricsMonitoringEnabled = defaults
.AppMetricsMonitoringEnabled;
StandardConnectionSetup queueConsumerConnectionSetup =
new StandardConnectionSetup(defaults);
StandardConnectionSetup queueProducerConnectionSetup =
new StandardConnectionSetup(defaults);
StandardConnectionSetup queueInfoConnectionSetup =
new StandardConnectionSetup(defaults);
StandardConnectionSetup builtInTimingBeltConnectionSetup =
new StandardConnectionSetup(defaults);
StandardConnectionSetup builtInWriterConnectionSetup =
new StandardConnectionSetup(defaults);
StandardConnectionSetup builtInAppMetricsWriterConnectionSetup =
new StandardConnectionSetup(defaults);
mSetupDbAssetsConnectionSetup =
new StandardConnectionSetup(defaults);
mTaskQueueProducerSetup = new StandardTaskQueueProducerSetup(queueProducerConnectionSetup,
defaults);
mTaskQueueInfoSetup = new StandardTaskQueueInfoSetup(queueInfoConnectionSetup,
defaults);
mCommonTaskQueueConnectionSetup = new CollectiveConnectionSetup(queueConsumerConnectionSetup,
queueProducerConnectionSetup,
queueInfoConnectionSetup,
builtInTimingBeltConnectionSetup,
builtInWriterConnectionSetup,
builtInAppMetricsWriterConnectionSetup,
mSetupDbAssetsConnectionSetup);
mTaskQueueConsumerSetup = new StandardTaskQueueConsumerSetup(queueConsumerConnectionSetup,
defaults);
mTaskEngineSetup = new StandardTaskEngineSetup(mTaskQueueConsumerSetup,
defaults);
mPerformanceMonitorWriterSetup = new StandardExecutionPerformanceMonitorWriterSetup(builtInWriterConnectionSetup,
defaults);
mAppMetricsMonitorSetup = new StandardAppMetricsMonitorSetup(defaults);
mAppMetricsMonitorWriterSetup = new StandardAppMetricsMonitorWriterSetup(builtInAppMetricsWriterConnectionSetup,
defaults);
}
private async Task RunDbAssetFactoryTests(ConnectionOptions connectionOptions, QueuedTaskMapping mapping, int count)
{
List <Mock <ISetupDbAsset> > dbAssetSetupsMocks = CreateDbAssetSetupMocksWithoutError(connectionOptions,
mapping,
count);
IList <ISetupDbAsset> dbAssetSetups = dbAssetSetupsMocks
.Select(m => m.Object)
.ToList();
DbAssetFactory factory = new DbAssetFactory(dbAssetSetups,
connectionOptions,
mapping);
await factory.CreateDbAssetsAsync();
foreach (Mock <ISetupDbAsset> mock in dbAssetSetupsMocks)
{
mock.Verify();
mock.VerifyNoOtherCalls();
}
}
private QueuedTaskMapping MergeMappingFromConfig(QueuedTaskMapping targeMapping, QueuedTaskMapping mappingFromConfig)
{
if (!string.IsNullOrWhiteSpace(mappingFromConfig.DequeueFunctionName))
{
targeMapping.DequeueFunctionName = mappingFromConfig.DequeueFunctionName;
}
if (!string.IsNullOrWhiteSpace(mappingFromConfig.ExecutionTimeStatsTableName))
{
targeMapping.ExecutionTimeStatsTableName = mappingFromConfig.ExecutionTimeStatsTableName;
}
if (!string.IsNullOrWhiteSpace(mappingFromConfig.MetricsTableName))
{
targeMapping.MetricsTableName = mappingFromConfig.MetricsTableName;
}
if (!string.IsNullOrWhiteSpace(mappingFromConfig.NewTaskNotificationChannelName))
{
targeMapping.NewTaskNotificationChannelName = mappingFromConfig.NewTaskNotificationChannelName;
}
if (!string.IsNullOrWhiteSpace(mappingFromConfig.QueueTableName))
{
targeMapping.QueueTableName = mappingFromConfig.QueueTableName;
}
if (!string.IsNullOrWhiteSpace(mappingFromConfig.ResultsQueueTableName))
{
targeMapping.ResultsQueueTableName = mappingFromConfig.ResultsQueueTableName;
}
return(targeMapping);
}
public async Task SetupDbAssetAsync(ConnectionOptions queueConnectionOptions, QueuedTaskMapping mapping)
{
if (queueConnectionOptions == null)
{
throw new ArgumentNullException(nameof(queueConnectionOptions));
}
if (mapping == null)
{
throw new ArgumentNullException(nameof(mapping));
}
using (NpgsqlConnection conn = await queueConnectionOptions.TryOpenConnectionAsync())
{
using (NpgsqlCommand cmdLockHandleIdSeq = new NpgsqlCommand(GetLockHandleIdSequenceCreationScript(mapping),
conn))
await cmdLockHandleIdSeq.ExecuteNonQueryAsync();
using (NpgsqlCommand cmdTable = new NpgsqlCommand(GetDbTableCreationScript(mapping),
conn))
await cmdTable.ExecuteNonQueryAsync();
if (mCreateSortIndex)
{
using (NpgsqlCommand cmdCreateSortIndex = new NpgsqlCommand(GetSortIndexCreationScript(mapping),
conn))
await cmdCreateSortIndex.ExecuteNonQueryAsync();
}
if (mCreateFilterIndex)
{
using (NpgsqlCommand cmdCreateFilterIndex = new NpgsqlCommand(GetFilterIndexCreationScript(mapping),
conn))
await cmdCreateFilterIndex.ExecuteNonQueryAsync();
}
}
}
public PostgreSqlTaskQueueDbOperations(string connectionString,
QueuedTaskMapping mapping)
{
mConnectionString = connectionString;
mMapping = mapping;
}
private string GetLockHandleIdSequenceName(QueuedTaskMapping mapping)
{
return(string.Format(LockHandleIdSequenceNameFormat, mapping.QueueTableName));
}
public async Task Test_CanSetupDbAsset_WithNonDefaultMapping()
{
QueuedTaskMapping mapping = GenerateNonDefaultMapping();
await RunDbAssetSetupTestsAsync(mapping);
}
private List <Mock <ISetupDbAsset> > CreateDbAssetSetupMocksWithoutError(ConnectionOptions connectionOptions, QueuedTaskMapping mapping, int count)
{
List <Mock <ISetupDbAsset> > dbAssetMocks = new List <Mock <ISetupDbAsset> >(count);
for (int i = 0; i < count; i++)
{
dbAssetMocks.Add(CreateDbAssetSetupMockWithoutError(connectionOptions, mapping));
}
return(dbAssetMocks);
}
