private async Task Start()
{
try
{
this.log.LogInformation(
(int)ErrorCode.MembershipStarting,
"MembershipOracle starting on host {HostName} with SiloAddress {SiloAddress} at {StartTime}",
this.localSiloDetails.DnsHostName,
this.myAddress,
LogFormatter.PrintDate(this.siloStartTime));
// Init the membership table.
await this.membershipTableProvider.InitializeMembershipTable(true);
// Perform an initial table read
var refreshed = await AsyncExecutorWithRetries.ExecuteWithRetries(
function : _ => this.RefreshInternal(requireCleanup: true),
maxNumSuccessTries : NUM_CONDITIONAL_WRITE_CONTENTION_ATTEMPTS,
maxNumErrorTries : NUM_CONDITIONAL_WRITE_ERROR_ATTEMPTS,
retryValueFilter : (value, i) => !value,
retryExceptionFilter : (exc, i) => true,
maxExecutionTime : this.clusterMembershipOptions.MaxJoinAttemptTime,
onSuccessBackOff : new ExponentialBackoff(EXP_BACKOFF_CONTENTION_MIN, EXP_BACKOFF_CONTENTION_MAX, EXP_BACKOFF_STEP),
onErrorBackOff : new ExponentialBackoff(EXP_BACKOFF_ERROR_MIN, EXP_BACKOFF_ERROR_MAX, EXP_BACKOFF_STEP));
if (!refreshed)
{
throw new OrleansException("Failed to perform initial membership refresh and cleanup.");
}
// read the table and look for my node migration occurrences
DetectNodeMigration(this.snapshot, this.localSiloDetails.DnsHostName);
}
catch (Exception exception)
{
this.log.LogError((int)ErrorCode.MembershipFailedToStart, "Membership failed to start: {Exception}", exception);
throw;
}
}
private async Task RegisterAsStreamProducer(StreamId streamId, StreamSequenceToken streamStartToken)
{
try
{
if (pubSub == null)
{
throw new NullReferenceException("Found pubSub reference not set up correctly in RetreaveNewStream");
}
IStreamProducerExtension meAsStreamProducer = this.AsReference <IStreamProducerExtension>();
ISet <PubSubSubscriptionState> streamData = null;
await AsyncExecutorWithRetries.ExecuteWithRetries(
async i => { streamData =
await PubsubRegisterProducer(pubSub, streamId, streamProviderName, meAsStreamProducer, logger); },
AsyncExecutorWithRetries.INFINITE_RETRIES,
(exception, i) => !IsShutdown,
Constants.INFINITE_TIMESPAN,
DeliveryBackoffProvider);
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug(ErrorCode.PersistentStreamPullingAgent_16, "Got back {0} Subscribers for stream {1}.", streamData.Count, streamId);
}
var addSubscriptionTasks = new List <Task>(streamData.Count);
foreach (PubSubSubscriptionState item in streamData)
{
addSubscriptionTasks.Add(AddSubscriber_Impl(item.SubscriptionId, item.Stream, item.Consumer, streamStartToken, item.Filter));
}
await Task.WhenAll(addSubscriptionTasks);
}
catch (Exception exc)
{
// RegisterAsStreamProducer is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception
logger.Error(ErrorCode.PersistentStreamPullingAgent_17, "Ignored RegisterAsStreamProducer Error", exc);
throw;
}
}
private async Task ExecuteWithRetries(Func <Task> functionToExecute, ErrorCode errorCode, string errorStr)
{
try
{
// Try to register/unregister the client in the directory.
// If failed, keep retrying with exponentially increasing time intervals in between, until:
// either succeeds or max time of orleansConfig.Globals.ClientRegistrationRefresh has reached.
// If failed to register after that time, it will be retried further on by clientRefreshTimer.
// In the unregsiter case just drop it. At the worst, where will be garbage in the directory.
await AsyncExecutorWithRetries.ExecuteWithRetries(
_ =>
{
return(functionToExecute());
},
AsyncExecutorWithRetries.INFINITE_RETRIES, // Do not limit the number of on-error retries, control it via "maxExecutionTime"
(exc, i) => true, // Retry on all errors.
orleansConfig.Globals.ClientRegistrationRefresh, // "maxExecutionTime"
new ExponentialBackoff(EXP_BACKOFF_ERROR_MIN, EXP_BACKOFF_ERROR_MAX, EXP_BACKOFF_STEP)); // how long to wait between error retries
}
catch (Exception exc)
{
logger.Error(errorCode, errorStr, exc);
}
}
private async Task RunConsumerCursor(StreamConsumerData consumerData, IStreamFilterPredicateWrapper filterWrapper)
{
try
{
// double check in case of interleaving
if (consumerData.State == StreamConsumerDataState.Active ||
consumerData.Cursor == null)
{
return;
}
consumerData.State = StreamConsumerDataState.Active;
while (consumerData.Cursor != null)
{
IBatchContainer batch = null;
Exception exceptionOccured = null;
try
{
Exception ignore;
if (!consumerData.Cursor.MoveNext())
{
break;
}
batch = consumerData.Cursor.GetCurrent(out ignore);
}
catch (Exception exc)
{
exceptionOccured = exc;
consumerData.SafeDisposeCursor(logger);
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, null);
}
// Apply filtering to this batch, if applicable
if (filterWrapper != null && batch != null)
{
try
{
// Apply batch filter to this input batch, to see whether we should deliver it to this consumer.
if (!batch.ShouldDeliver(
consumerData.StreamId,
filterWrapper.FilterData,
filterWrapper.ShouldReceive))
{
continue; // Skip this batch -- nothing to do
}
}
catch (Exception exc)
{
var message =
$"Ignoring exception while trying to evaluate subscription filter function {filterWrapper} on stream {consumerData.StreamId} in PersistentStreamPullingAgentGrain.RunConsumerCursor";
logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_13, message, exc);
}
}
try
{
numSentMessagesCounter.Increment();
if (batch != null)
{
StreamHandshakeToken newToken = await AsyncExecutorWithRetries.ExecuteWithRetries(
i => DeliverBatchToConsumer(consumerData, batch),
AsyncExecutorWithRetries.INFINITE_RETRIES,
(exception, i) => !(exception is ClientNotAvailableException),
config.MaxEventDeliveryTime,
DeliveryBackoffProvider);
if (newToken != null)
{
consumerData.LastToken = newToken;
IQueueCacheCursor newCursor = queueCache.GetCacheCursor(consumerData.StreamId, newToken.Token);
consumerData.SafeDisposeCursor(logger);
consumerData.Cursor = newCursor;
}
}
}
catch (Exception exc)
{
consumerData.Cursor?.RecordDeliveryFailure();
var message =
$"Exception while trying to deliver msgs to stream {consumerData.StreamId} in PersistentStreamPullingAgentGrain.RunConsumerCursor";
logger.Error(ErrorCode.PersistentStreamPullingAgent_14, message, exc);
exceptionOccured = exc is ClientNotAvailableException
? exc
: new StreamEventDeliveryFailureException(consumerData.StreamId);
}
// if we failed to deliver a batch
if (exceptionOccured != null)
{
bool faultedSubscription = await ErrorProtocol(consumerData, exceptionOccured, true, batch, batch?.SequenceToken);
if (faultedSubscription)
{
return;
}
}
}
consumerData.State = StreamConsumerDataState.Inactive;
}
catch (Exception exc)
{
// RunConsumerCursor is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception
logger.Error(ErrorCode.PersistentStreamPullingAgent_15, "Ignored RunConsumerCursor Error", exc);
consumerData.State = StreamConsumerDataState.Inactive;
throw;
}
}
private async Task <bool> DoHandshakeWithConsumer(
StreamConsumerData consumerData,
StreamSequenceToken cacheToken)
{
StreamHandshakeToken requestedHandshakeToken = null;
// if not cache, then we can't get cursor and there is no reason to ask consumer for token.
if (queueCache != null)
{
Exception exceptionOccured = null;
try
{
requestedHandshakeToken = await AsyncExecutorWithRetries.ExecuteWithRetries(
i => consumerData.StreamConsumer.GetSequenceToken(consumerData.SubscriptionId),
AsyncExecutorWithRetries.INFINITE_RETRIES,
(exception, i) => !(exception is ClientNotAvailableException),
config.MaxEventDeliveryTime,
DeliveryBackoffProvider);
if (requestedHandshakeToken != null)
{
consumerData.SafeDisposeCursor(logger);
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, requestedHandshakeToken.Token);
}
else
{
if (consumerData.Cursor == null) // if the consumer did not ask for a specific token and we already have a cursor, jsut keep using it.
{
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, cacheToken);
}
}
}
catch (Exception exception)
{
exceptionOccured = exception;
}
if (exceptionOccured != null)
{
bool faultedSubscription = await ErrorProtocol(consumerData, exceptionOccured, false, null, requestedHandshakeToken?.Token);
if (faultedSubscription)
{
return(false);
}
}
}
consumerData.LastToken = requestedHandshakeToken; // use what ever the consumer asked for as LastToken for next handshake (even if he asked for null).
// if we don't yet have a cursor (had errors in the handshake or data not available exc), get a cursor at the event that triggered that consumer subscription.
if (consumerData.Cursor == null && queueCache != null)
{
try
{
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, cacheToken);
}
catch (Exception)
{
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, null); // just in case last GetCacheCursor failed.
}
}
return(true);
}
private async Task RunConsumerCursor(StreamConsumerData consumerData)
{
try
{
// double check in case of interleaving
if (consumerData.State == StreamConsumerDataState.Active ||
consumerData.Cursor == null)
{
return;
}
consumerData.State = StreamConsumerDataState.Active;
while (consumerData.Cursor != null)
{
IBatchContainer batch = null;
Exception exceptionOccured = null;
try
{
batch = GetBatchForConsumer(consumerData.Cursor, consumerData.StreamId, consumerData.FilterData);
if (batch == null)
{
break;
}
}
catch (Exception exc)
{
exceptionOccured = exc;
consumerData.SafeDisposeCursor(logger);
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, null);
}
if (batch != null)
{
if (!ShouldDeliverBatch(consumerData.StreamId, batch, consumerData.FilterData))
{
continue;
}
}
try
{
numSentMessagesCounter.Increment();
if (batch != null)
{
StreamHandshakeToken newToken = await AsyncExecutorWithRetries.ExecuteWithRetries(
i => DeliverBatchToConsumer(consumerData, batch),
AsyncExecutorWithRetries.INFINITE_RETRIES,
(exception, i) => !(exception is ClientNotAvailableException) || IsShutdown,
this.options.MaxEventDeliveryTime,
DeliveryBackoffProvider);
if (newToken != null)
{
consumerData.LastToken = newToken;
IQueueCacheCursor newCursor = queueCache.GetCacheCursor(consumerData.StreamId, newToken.Token);
consumerData.SafeDisposeCursor(logger);
consumerData.Cursor = newCursor;
}
}
}
catch (Exception exc)
{
consumerData.Cursor?.RecordDeliveryFailure();
var message =
$"Exception while trying to deliver msgs to stream {consumerData.StreamId} in PersistentStreamPullingAgentGrain.RunConsumerCursor";
logger.Error(ErrorCode.PersistentStreamPullingAgent_14, message, exc);
exceptionOccured = exc is ClientNotAvailableException
? exc
: new StreamEventDeliveryFailureException(consumerData.StreamId);
}
// if we failed to deliver a batch
if (exceptionOccured != null)
{
bool faultedSubscription = await ErrorProtocol(consumerData, exceptionOccured, true, batch, batch?.SequenceToken);
if (faultedSubscription)
{
return;
}
}
}
consumerData.State = StreamConsumerDataState.Inactive;
}
catch (Exception exc)
{
// RunConsumerCursor is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception
logger.Error(ErrorCode.PersistentStreamPullingAgent_15, "Ignored RunConsumerCursor Error", exc);
consumerData.State = StreamConsumerDataState.Inactive;
throw;
}
}
/// <summary>
/// Read data entries and their corresponding eTags from the Azure table.
/// </summary>
/// <param name="predicate">Predicate function to use for querying the table and filtering the results.</param>
/// <returns>Enumeration of entries in the table which match the query condition.</returns>
internal async Task <IEnumerable <Tuple <T, string> > > ReadTableEntriesAndEtagsAsync(Expression <Func <T, bool> > predicate)
{
const string operation = "ReadTableEntriesAndEtags";
var startTime = DateTime.UtcNow;
try
{
TableServiceContext svc = tableOperationsClient.GetDataServiceContext();
// Improve performance when table name differs from class name
// http://www.gtrifonov.com/2011/06/15/improving-performance-for-windows-azure-tables/
svc.ResolveType = ResolveEntityType;
//IQueryable<T> query = svc.CreateQuery<T>(TableName).Where(predicate);
CloudTableQuery <T> cloudTableQuery = svc.CreateQuery <T>(TableName).Where(predicate).AsTableServiceQuery(); // turn IQueryable into CloudTableQuery
try
{
Func <Task <List <T> > > executeQueryHandleContinuations = async() =>
{
// Read table with continuation token
// http://convective.wordpress.com/2013/11/03/queries-in-the-windows-azure-storage-client-library-v2-1/
// 1) First wrong sync way to read:
// List<T> queryResults = query.ToList(); // ToList will actually execute the query and add entities to svc. However, this will not handle continuation tokens.
// 2) Second correct sync way to read:
// http://convective.wordpress.com/2010/02/06/queries-in-azure-tables/
// CloudTableQuery.Execute will properly retrieve all the records from a table through the automatic handling of continuation tokens:
Task <ResultSegment <T> > firstSegmentPromise = Task <ResultSegment <T> > .Factory.FromAsync(
cloudTableQuery.BeginExecuteSegmented,
cloudTableQuery.EndExecuteSegmented,
null);
// 3) Third wrong async way to read:
// return firstSegmentPromise;
// 4) Forth correct async way to read - handles continuation tokens:
var list = new List <T>();
Task <ResultSegment <T> > nextSegmentAsync = firstSegmentPromise;
while (true)
{
ResultSegment <T> resultSegment = await nextSegmentAsync;
var capture = resultSegment.Results;
if (capture != null) // don't call Count or Any or anything else that can potentialy cause multiple evaluations of the IEnumerable
{
list.AddRange(capture);
}
if (!resultSegment.HasMoreResults)
{
// All data was read successfully if we got to here
break;
}
// ask to read the next segment
nextSegmentAsync = Task <ResultSegment <T> > .Factory.FromAsync(
resultSegment.BeginGetNext,
resultSegment.EndGetNext,
null);
}
return(list);
};
IBackoffProvider backoff = new FixedBackoff(AzureTableDefaultPolicies.PauseBetweenTableOperationRetries);
List <T> results = await AsyncExecutorWithRetries.ExecuteWithRetries(
counter => executeQueryHandleContinuations(),
AzureTableDefaultPolicies.MaxTableOperationRetries,
(exc, counter) => AzureStorageUtils.AnalyzeReadException(exc.GetBaseException(), counter, TableName, Logger),
AzureTableDefaultPolicies.TableOperationTimeout,
backoff);
// Data was read successfully if we got to here
return(PairEntitiesWithEtags(svc, results));
}
catch (Exception exc)
{
// Out of retries...
var errorMsg = string.Format("Failed to read Azure storage table {0}: {1}", TableName, exc.Message);
if (!AzureStorageUtils.TableStorageDataNotFound(exc))
{
Logger.Warn(ErrorCode.AzureTable_09, errorMsg, exc);
}
throw new OrleansException(errorMsg, exc);
}
}
finally
{
CheckAlertSlowAccess(startTime, operation);
}
}
/// <summary>
/// Inserts a data entry in the Azure table: creates a new one if does not exists or overwrites (without eTag) an already existing version (the "update in place" semantincs).
/// </summary>
/// <param name="data">Data to be inserted or replaced in the table.</param>
/// <returns>Value promise with new Etag for this data entry after completing this storage operation.</returns>
public async Task <string> UpsertTableEntryAsync(T data)
{
const string operation = "UpsertTableEntry";
var startTime = DateTime.UtcNow;
if (Logger.IsVerbose2)
{
Logger.Verbose2("{0} entry {1} into table {2}", operation, data, TableName);
}
try
{
TableServiceContext svc = tableOperationsClient.GetDataServiceContext();
try
{
Task <DataServiceResponse> savePromise;
Func <int, Task <DataServiceResponse> > doSaveChanges = retryNum =>
{
if (retryNum > 0)
{
svc.Detach(data);
}
// Try to do update first
svc.AttachTo(TableName, data, ANY_ETAG);
svc.UpdateObject(data);
return(Task <DataServiceResponse> .Factory.FromAsync(
svc.BeginSaveChangesWithRetries,
svc.EndSaveChangesWithRetries,
SaveChangesOptions.ReplaceOnUpdate,
null));
};
if (AzureTableDefaultPolicies.MaxBusyRetries > 0)
{
IBackoffProvider backoff = new FixedBackoff(AzureTableDefaultPolicies.PauseBetweenBusyRetries);
savePromise = AsyncExecutorWithRetries.ExecuteWithRetries(
doSaveChanges,
AzureTableDefaultPolicies.MaxBusyRetries,
// Retry automatically iff we get ServerBusy reply from Azure storage
(exc, retryNum) => IsServerBusy(exc),
AzureTableDefaultPolicies.BusyRetriesTimeout,
backoff);
}
else
{
// Try single Write only once
savePromise = doSaveChanges(0);
}
await savePromise;
EntityDescriptor result = svc.GetEntityDescriptor(data);
return(result.ETag);
}
catch (Exception exc)
{
Logger.Warn(ErrorCode.AzureTable_06, String.Format("Intermediate error upserting entry {0} to the table {1}",
(data == null ? "null" : data.ToString()), TableName), exc);
throw;
}
}
finally
{
CheckAlertSlowAccess(startTime, operation);
}
}
private async Task RunConsumerCursor(StreamConsumerData consumerData, IStreamFilterPredicateWrapper filterWrapper)
{
try
{
// double check in case of interleaving
if (consumerData.State == StreamConsumerDataState.Active ||
consumerData.Cursor == null)
{
return;
}
consumerData.State = StreamConsumerDataState.Active;
while (consumerData.Cursor != null && consumerData.Cursor.MoveNext())
{
IBatchContainer batch = null;
Exception ex;
Task deliveryTask;
bool deliveryFailed = false;
try
{
batch = consumerData.Cursor.GetCurrent(out ex);
}
catch (DataNotAvailableException dataNotAvailable)
{
ex = dataNotAvailable;
}
// Apply filtering to this batch, if applicable
if (filterWrapper != null && batch != null)
{
try
{
// Apply batch filter to this input batch, to see whether we should deliver it to this consumer.
if (!batch.ShouldDeliver(
consumerData.StreamId,
filterWrapper.FilterData,
filterWrapper.ShouldReceive))
{
continue; // Skip this batch -- nothing to do
}
}
catch (Exception exc)
{
var message = string.Format("Ignoring exception while trying to evaluate subscription filter function {0} on stream {1} in PersistentStreamPullingAgentGrain.RunConsumerCursor", filterWrapper, consumerData.StreamId);
logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_13, message, exc);
}
}
if (batch != null)
{
deliveryTask = AsyncExecutorWithRetries.ExecuteWithRetries(i => DeliverBatchToConsumer(consumerData, batch),
AsyncExecutorWithRetries.INFINITE_RETRIES, (exception, i) => !(exception is DataNotAvailableException), config.MaxEventDeliveryTime, DefaultBackoffProvider);
}
else if (ex == null)
{
deliveryTask = consumerData.StreamConsumer.CompleteStream(consumerData.SubscriptionId);
}
else
{
// If data is not avialable, bring cursor current
if (ex is DataNotAvailableException)
{
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId.Guid,
consumerData.StreamId.Namespace, null);
}
// Notify client of error.
deliveryTask = DeliverErrorToConsumer(consumerData, ex, null);
}
try
{
numSentMessagesCounter.Increment();
await deliveryTask;
}
catch (Exception exc)
{
var message = string.Format("Exception while trying to deliver msgs to stream {0} in PersistentStreamPullingAgentGrain.RunConsumerCursor", consumerData.StreamId);
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_14, message, exc);
deliveryFailed = true;
}
// if we failed to deliver a batch
if (deliveryFailed && batch != null)
{
// notify consumer of delivery error, if we can.
await OrleansTaskExtentions.ExecuteAndIgnoreException(() => DeliverErrorToConsumer(consumerData, new StreamEventDeliveryFailureException(consumerData.StreamId), batch));
// record that there was a delivery failure
await streamFailureHandler.OnDeliveryFailure(consumerData.SubscriptionId, streamProviderName,
consumerData.StreamId, batch.SequenceToken);
// if configured to fault on delivery failure and this is not an implicit subscription, fault and remove the subscription
if (streamFailureHandler.ShouldFaultSubsriptionOnError && !SubscriptionMarker.IsImplicitSubscription(consumerData.SubscriptionId.Guid))
{
try
{
// notify consumer of faulted subscription, if we can.
DeliverErrorToConsumer(consumerData,
new FaultedSubscriptionException(consumerData.SubscriptionId, consumerData.StreamId), batch)
.Ignore();
// mark subscription as faulted.
await pubSub.FaultSubscription(consumerData.StreamId, consumerData.SubscriptionId);
}
finally
{
// remove subscription
RemoveSubscriber_Impl(consumerData.SubscriptionId, consumerData.StreamId);
}
return;
}
}
}
consumerData.State = StreamConsumerDataState.Inactive;
}
catch (Exception exc)
{
// RunConsumerCursor is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_15, "Ignored RunConsumerCursor Error", exc);
throw;
}
}
