private async Task StartTransactions(List <TimeSpan> startingTransactions, List <TaskCompletionSource <long> > startCompletions)
{
try
{
StartTransactionsResponse startResponse = await this.tmService.StartTransactions(startingTransactions);
List <long> startedIds = startResponse.TransactionId;
// reply to clients with results
for (int i = 0; i < startCompletions.Count; i++)
{
TransactionsStatisticsGroup.OnTransactionStarted();
startCompletions[i].SetResult(startedIds[i]);
}
// Refresh cached values using new values from TM.
this.ReadOnlyTransactionId = Math.Max(this.ReadOnlyTransactionId, startResponse.ReadOnlyTransactionId);
this.abortLowerBound = Math.Max(this.abortLowerBound, startResponse.AbortLowerBound);
logger.Verbose(ErrorCode.Transactions_ReceivedTMResponse, "{0} Transactions started. readOnlyTransactionId {1}, abortLowerBound {2}", startingTransactions.Count, ReadOnlyTransactionId, abortLowerBound);
}
catch (Exception e)
{
logger.Error(ErrorCode.Transactions_TMError, "Transaction manager failed to start transactions.", e);
foreach (var completion in startCompletions)
{
TransactionsStatisticsGroup.OnTransactionStartFailed();
completion.SetException(new OrleansStartTransactionFailedException(e));
}
}
startingTransactions.Clear();
startCompletions.Clear();
}
public SiloStatisticsManager(
IOptions <SiloStatisticsOptions> statisticsOptions,
IOptions <LoadSheddingOptions> loadSheddingOptions,
IOptions <StorageOptions> azureStorageOptions,
ILocalSiloDetails siloDetails,
SerializationManager serializationManager,
ITelemetryProducer telemetryProducer,
IHostEnvironmentStatistics hostEnvironmentStatistics,
IAppEnvironmentStatistics appEnvironmentStatistics,
ILoggerFactory loggerFactory,
IOptions <SiloMessagingOptions> messagingOptions)
{
this.siloDetails = siloDetails;
this.storageOptions = azureStorageOptions.Value;
MessagingStatisticsGroup.Init(true);
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init(true);
ApplicationRequestsStatisticsGroup.Init(messagingOptions.Value.ResponseTimeout);
SchedulerStatisticsGroup.Init(loggerFactory);
StorageStatisticsGroup.Init();
TransactionsStatisticsGroup.Init();
this.logger = loggerFactory.CreateLogger <SiloStatisticsManager>();
this.hostEnvironmentStatistics = hostEnvironmentStatistics;
this.logStatistics = new LogStatistics(statisticsOptions.Value.LogWriteInterval, true, serializationManager, loggerFactory);
this.MetricsTable = new SiloPerformanceMetrics(this.hostEnvironmentStatistics, appEnvironmentStatistics, loggerFactory, loadSheddingOptions);
this.countersPublisher = new CountersStatistics(statisticsOptions.Value.PerfCountersWriteInterval, telemetryProducer, loggerFactory);
}
public async Task Commit(ITransactionInfo info)
{
var transactionInfo = (TransactionInfo)info;
TransactionsStatisticsGroup.OnTransactionCommitRequest();
if (transactionInfo.IsReadOnly)
{
return;
}
var completion = new TaskCompletionSource <bool>();
bool canCommit = true;
List <Task <bool> > prepareTasks = new List <Task <bool> >(transactionInfo.WriteSet.Count);
foreach (var g in transactionInfo.WriteSet.Keys)
{
TransactionalResourceVersion write = TransactionalResourceVersion.Create(transactionInfo.TransactionId, transactionInfo.WriteSet[g]);
TransactionalResourceVersion?read = null;
if (transactionInfo.ReadSet.ContainsKey(g))
{
read = transactionInfo.ReadSet[g];
transactionInfo.ReadSet.Remove(g);
}
prepareTasks.Add(g.Prepare(transactionInfo.TransactionId, write, read));
}
foreach (var g in transactionInfo.ReadSet.Keys)
{
TransactionalResourceVersion read = transactionInfo.ReadSet[g];
prepareTasks.Add(g.Prepare(transactionInfo.TransactionId, null, read));
}
await Task.WhenAll(prepareTasks);
foreach (var t in prepareTasks)
{
if (!t.Result)
{
canCommit = false;
}
}
if (!canCommit)
{
TransactionsStatisticsGroup.OnTransactionAborted();
abortedTransactions.TryAdd(transactionInfo.TransactionId, 0);
throw new OrleansPrepareFailedException(transactionInfo.TransactionId);
}
commitCompletions.TryAdd(transactionInfo.TransactionId, completion);
transactionCommitQueue.Enqueue(transactionInfo);
await completion.Task;
}
public SiloStatisticsManager(
IOptions <StatisticsOptions> statisticsOptions,
SerializationStatisticsGroup serializationStatistics,
ITelemetryProducer telemetryProducer,
ILoggerFactory loggerFactory)
{
MessagingStatisticsGroup.Init(true);
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init(true);
StorageStatisticsGroup.Init();
TransactionsStatisticsGroup.Init();
this.logStatistics = new LogStatistics(statisticsOptions.Value.LogWriteInterval, true, serializationStatistics, loggerFactory);
this.countersPublisher = new CountersStatistics(statisticsOptions.Value.PerfCountersWriteInterval, telemetryProducer, loggerFactory);
}
public async Task <TransactionInfo> StartTransaction(bool readOnly, TimeSpan timeout)
{
if (readOnly)
{
return(new TransactionInfo(ReadOnlyTransactionId, true));
}
TransactionsStatisticsGroup.OnTransactionStartRequest();
var completion = new TaskCompletionSource <long>();
transactionStartQueue.Enqueue(new Tuple <TimeSpan, TaskCompletionSource <long> >(timeout, completion));
long id = await completion.Task;
return(new TransactionInfo(id));
}
public SiloStatisticsManager(NodeConfiguration nodeConfiguration, ILocalSiloDetails siloDetails, SerializationManager serializationManager, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory, IOptions <MessagingOptions> messagingOptions)
{
this.siloDetails = siloDetails;
MessagingStatisticsGroup.Init(true);
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init(true);
ApplicationRequestsStatisticsGroup.Init(messagingOptions.Value.ResponseTimeout);
SchedulerStatisticsGroup.Init(loggerFactory);
StorageStatisticsGroup.Init();
TransactionsStatisticsGroup.Init();
this.logger = loggerFactory.CreateLogger <SiloStatisticsManager>();
runtimeStats = new RuntimeStatisticsGroup(loggerFactory);
this.logStatistics = new LogStatistics(nodeConfiguration.StatisticsLogWriteInterval, true, serializationManager, loggerFactory);
this.MetricsTable = new SiloPerformanceMetrics(this.runtimeStats, loggerFactory, nodeConfiguration);
this.countersPublisher = new CountersStatistics(nodeConfiguration.StatisticsPerfCountersWriteInterval, telemetryProducer, loggerFactory);
}
public SiloStatisticsManager(SiloInitializationParameters initializationParams, SerializationManager serializationManager, ITelemetryProducer telemetryProducer)
{
MessagingStatisticsGroup.Init(true);
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init(true);
ApplicationRequestsStatisticsGroup.Init(initializationParams.GlobalConfig.ResponseTimeout);
SchedulerStatisticsGroup.Init();
StorageStatisticsGroup.Init();
TransactionsStatisticsGroup.Init();
runtimeStats = new RuntimeStatisticsGroup();
this.logStatistics = new LogStatistics(initializationParams.NodeConfig.StatisticsLogWriteInterval, true, serializationManager);
this.MetricsTable = new SiloPerformanceMetrics(this.runtimeStats, initializationParams.NodeConfig);
this.countersPublisher = new CountersStatistics(initializationParams.NodeConfig.StatisticsPerfCountersWriteInterval, telemetryProducer);
initializationParams.ClusterConfig.OnConfigChange(
"Defaults/LoadShedding",
() => this.MetricsTable.NodeConfig = initializationParams.NodeConfig,
false);
}
public SiloStatisticsManager(
IOptions <SiloStatisticsOptions> statisticsOptions,
IOptions <MonitoringStorageOptions> azureStorageOptions,
ILocalSiloDetails siloDetails,
SerializationManager serializationManager,
ITelemetryProducer telemetryProducer,
ILoggerFactory loggerFactory)
{
this.siloDetails = siloDetails;
this.storageOptions = azureStorageOptions.Value;
MessagingStatisticsGroup.Init(true);
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init(true);
ApplicationRequestsStatisticsGroup.Init();
SchedulerStatisticsGroup.Init(loggerFactory);
StorageStatisticsGroup.Init();
TransactionsStatisticsGroup.Init();
this.logger = loggerFactory.CreateLogger <SiloStatisticsManager>();
this.logStatistics = new LogStatistics(statisticsOptions.Value.LogWriteInterval, true, serializationManager, loggerFactory);
this.countersPublisher = new CountersStatistics(statisticsOptions.Value.PerfCountersWriteInterval, telemetryProducer, loggerFactory);
}
private async Task CommitTransactions(List <TransactionInfo> committingTransactions,
HashSet <long> outstandingCommits)
{
var stopWatch = Stopwatch.StartNew();
try
{
metrics.BatchCommitTransactionsRequestsCounter++;
metrics.BatchCommitTransactionsRequestSizeCounter += committingTransactions.Count;
CommitTransactionsResponse commitResponse;
try
{
commitResponse = await this.tmService.CommitTransactions(committingTransactions, outstandingCommits);
}
finally
{
stopWatch.Stop();
metrics.BatchCommitTransactionsRequestLatencyCounter += stopWatch.Elapsed;
}
var commitResults = commitResponse.CommitResult;
// reply to clients with the outcomes we received from the TM.
foreach (var completedId in commitResults.Keys)
{
outstandingCommits.Remove(completedId);
TaskCompletionSource <bool> completion;
if (commitCompletions.TryRemove(completedId, out completion))
{
if (commitResults[completedId].Success)
{
TransactionsStatisticsGroup.OnTransactionCommitted();
completion.SetResult(true);
}
else
{
if (commitResults[completedId].AbortingException != null)
{
TransactionsStatisticsGroup.OnTransactionAborted();
completion.SetException(commitResults[completedId].AbortingException);
}
else
{
TransactionsStatisticsGroup.OnTransactionInDoubt();
completion.SetException(new OrleansTransactionInDoubtException(completedId));
}
}
}
}
// Refresh cached values using new values from TM.
this.ReadOnlyTransactionId = Math.Max(this.ReadOnlyTransactionId,
commitResponse.ReadOnlyTransactionId);
this.abortLowerBound = Math.Max(this.abortLowerBound, commitResponse.AbortLowerBound);
logger.Debug(ErrorCode.Transactions_ReceivedTMResponse,
"{0} transactions committed. readOnlyTransactionId {1}, abortLowerBound {2}",
committingTransactions.Count, ReadOnlyTransactionId, abortLowerBound);
}
catch (Exception e)
{
logger.Error(ErrorCode.Transactions_TMError, "TM Error", e);
// Propagate the exception to every transaction in the request.
foreach (var tx in committingTransactions)
{
TransactionsStatisticsGroup.OnTransactionInDoubt();
TaskCompletionSource <bool> completion;
if (commitCompletions.TryRemove(tx.TransactionId, out completion))
{
outstandingCommits.Remove(tx.TransactionId);
completion.SetException(new OrleansTransactionInDoubtException(tx.TransactionId));
}
}
}
committingTransactions.Clear();
}
private async Task ProcessRequests(object args)
{
// NOTE: This code is a bit complicated because we want to issue both start and commit requests,
// but wait for each one separately in its own continuation. This can be significantly simplified
// if we can register a separate timer for start and commit.
List <TransactionInfo> committingTransactions = new List <TransactionInfo>();
List <TimeSpan> startingTransactions = new List <TimeSpan>();
List <TaskCompletionSource <long> > startCompletions = new List <TaskCompletionSource <long> >();
while (transactionCommitQueue.Count > 0 || transactionStartQueue.Count > 0 || outstandingCommits.Count > 0)
{
var initialAbortLowerBound = this.abortLowerBound;
await Task.Yield();
await WaitForWork();
int startCount = transactionStartQueue.Count;
while (startCount > 0 && startTransactionsTask.IsCompleted)
{
Tuple <TimeSpan, TaskCompletionSource <long> > elem;
transactionStartQueue.TryDequeue(out elem);
startingTransactions.Add(elem.Item1);
startCompletions.Add(elem.Item2);
startCount--;
}
int commitCount = transactionCommitQueue.Count;
while (commitCount > 0 && commitTransactionsTask.IsCompleted)
{
TransactionInfo elem;
transactionCommitQueue.TryDequeue(out elem);
committingTransactions.Add(elem);
outstandingCommits.Add(elem.TransactionId);
commitCount--;
}
if (startingTransactions.Count > 0 && startTransactionsTask.IsCompleted)
{
logger.Verbose(ErrorCode.Transactions_SendingTMRequest, "Calling TM to start {0} transactions", startingTransactions.Count);
startTransactionsTask = this.StartTransactions(startingTransactions, startCompletions);
}
if ((committingTransactions.Count > 0 || outstandingCommits.Count > 0) && commitTransactionsTask.IsCompleted)
{
logger.Verbose(ErrorCode.Transactions_SendingTMRequest, "Calling TM to commit {0} transactions", committingTransactions.Count);
commitTransactionsTask = this.tmService.CommitTransactions(committingTransactions, outstandingCommits).ContinueWith(
async commitRequest =>
{
try
{
var commitResponse = await commitRequest;
var commitResults = commitResponse.CommitResult;
// reply to clients with the outcomes we received from the TM.
foreach (var completedId in commitResults.Keys)
{
outstandingCommits.Remove(completedId);
TaskCompletionSource <bool> completion;
if (commitCompletions.TryRemove(completedId, out completion))
{
if (commitResults[completedId].Success)
{
TransactionsStatisticsGroup.OnTransactionCommitted();
completion.SetResult(true);
}
else
{
if (commitResults[completedId].AbortingException != null)
{
TransactionsStatisticsGroup.OnTransactionAborted();
completion.SetException(commitResults[completedId].AbortingException);
}
else
{
TransactionsStatisticsGroup.OnTransactionInDoubt();
completion.SetException(new OrleansTransactionInDoubtException(completedId));
}
}
}
}
// Refresh cached values using new values from TM.
this.ReadOnlyTransactionId = Math.Max(this.ReadOnlyTransactionId, commitResponse.ReadOnlyTransactionId);
this.abortLowerBound = Math.Max(this.abortLowerBound, commitResponse.AbortLowerBound);
logger.Verbose(ErrorCode.Transactions_ReceivedTMResponse, "{0} transactions committed. readOnlyTransactionId {1}, abortLowerBound {2}", committingTransactions.Count, ReadOnlyTransactionId, abortLowerBound);
}
catch (Exception e)
{
logger.Error(ErrorCode.Transactions_TMError, "", e);
// Propagate the exception to every transaction in the request.
foreach (var tx in committingTransactions)
{
TransactionsStatisticsGroup.OnTransactionInDoubt();
TaskCompletionSource <bool> completion;
if (commitCompletions.TryRemove(tx.TransactionId, out completion))
{
outstandingCommits.Remove(tx.TransactionId);
completion.SetException(new OrleansTransactionInDoubtException(tx.TransactionId));
}
}
}
committingTransactions.Clear();
});
// Removed transactions below the abort lower bound.
if (this.abortLowerBound != initialAbortLowerBound)
{
foreach (var aborted in this.abortedTransactions)
{
if (aborted.Key < this.abortLowerBound)
{
long ignored;
this.abortedTransactions.TryRemove(aborted.Key, out ignored);
}
}
}
}
}
}
