public static async Task<StateManagerLease> GetOrCreateAsync(
IReliableStateManager StateManager,
IReliableDictionary<string, string> StateDictionary,
string EntryName,
string partitionId)
{
using (ITransaction tx = StateManager.CreateTransaction())
{
StateManagerLease lease;
// if something has been saved before load it
ConditionalResult<string> cResults = await StateDictionary.TryGetValueAsync(tx, EntryName);
if (cResults.HasValue)
{
lease = FromJsonString(cResults.Value);
lease.m_EntryName = EntryName;
lease.m_StateDictionary = StateDictionary;
lease.m_StateManager = StateManager;
}
else
{
// if not create new
lease = new StateManagerLease(StateManager, StateDictionary, EntryName, partitionId);
}
await tx.CommitAsync();
return lease;
}
}
public async Task AddMessageAsync(Message message)
{
DateTime time = DateTime.Now.ToLocalTime();
IReliableDictionary <DateTime, Message> messagesDictionary =
await this.StateManager.GetOrAddAsync <IReliableDictionary <DateTime, Message> >("messages");
//dictionary for the scores
IReliableDictionary <string, int> scoresDictionary =
await StateManager.GetOrAddAsync <IReliableDictionary <string, int> >("scores");
var currentQuestion =
await StateManager.GetOrAddAsync <IReliableQueue <KeyValuePair <string, string> > >("currentQuestion");
using (ITransaction tx = StateManager.CreateTransaction())
{
var currentScoreConditional = await scoresDictionary.TryGetValueAsync(tx, message.Name);
if (!currentScoreConditional.HasValue)
{
await scoresDictionary.GetOrAddAsync(tx, message.Name, 0);
}
await tx.CommitAsync();
}
//checking for the right answer
using (ITransaction tx = this.StateManager.CreateTransaction())
{
if (message.MessageText.ToLowerInvariant().Trim() == "try me")
{
var q = TriviaDatabase.GetRandomQuestion();
await currentQuestion.TryDequeueAsync(tx);
await currentQuestion.EnqueueAsync(tx, q);
await tx.CommitAsync();
}
else
{
var question = await currentQuestion.TryPeekAsync(tx);
if (question.Value.Value.ToLowerInvariant().Trim()
== message.MessageText.ToLowerInvariant().Trim())
{
await messagesDictionary.AddAsync(tx, time,
new Message { Name = message.Name, MessageText = message.MessageText });
await messagesDictionary.AddAsync(tx, time.AddTicks(1),
new Message { Name = "Admin", MessageText = "You are correct " + message.Name + "! You get one point for this" });
//get and update the score
var currentScoreConditional = await scoresDictionary.TryGetValueAsync(tx, message.Name);
var currentScore = currentScoreConditional.HasValue ? currentScoreConditional.Value + 1 : 0;
await scoresDictionary.AddOrUpdateAsync(tx, message.Name, currentScore, (x, y) => ++ y);
await currentQuestion.TryDequeueAsync(tx);
await currentQuestion.EnqueueAsync(tx, TriviaDatabase.GetRandomQuestion());
await tx.CommitAsync();
}
else
{
await messagesDictionary.AddAsync(tx, time, message);
await tx.CommitAsync();
}
}
}
}
public async Task <string> OrchestrateWorker(WorkerDescription workerDescription)
{
if (_processorDictionary == null)
{
this._processorDictionary = this.StateManager
.GetOrAddAsync <IReliableDictionary <string, ProcessorInformation> >("orchestrator.ProcessorDictionary").Result;
}
ServiceEventSource.Current.ServiceMessage(this.Context, $"Orchestrate worker called for {workerDescription.Identifier}");
var address = String.Empty;
using (var tx = this.StateManager.CreateTransaction())
{
var result = await _processorDictionary.TryGetValueAsync(tx, workerDescription.Identifier);
if (result.HasValue)
{
var info = result.Value;
await _processorDictionary.TryUpdateAsync(tx, workerDescription.Identifier,
new ProcessorInformation()
{
Address = info.Address,
TicksLastUpdated = DateTime.UtcNow.Ticks
},
info);
await tx.CommitAsync();
address = info.Address;
}
else
{
// spin up the new service here
ServiceEventSource.Current.ServiceMessage(this.Context, $"Creating processor for {workerDescription.Identifier}");
var appName = Context.CodePackageActivationContext.ApplicationName;
var svcName = $"{appName}/{Names.ProcessorSuffix}/{workerDescription.Identifier}";
await _fabricClient.ServiceManager.CreateServiceAsync(new StatefulServiceDescription()
{
HasPersistedState = true,
PartitionSchemeDescription = new UniformInt64RangePartitionSchemeDescription(1),
ServiceTypeName = Names.ProcessorTypeName,
ApplicationName = new System.Uri(appName),
ServiceName = new System.Uri(svcName)
});
ServiceEventSource.Current.ServiceMessage(this.Context, $"Processor for {workerDescription.Identifier} running on {svcName}");
await _processorDictionary.AddAsync(tx, workerDescription.Identifier, new ProcessorInformation()
{
Address = svcName,
TicksLastUpdated = DateTime.UtcNow.Ticks
});
address = svcName;
await tx.CommitAsync();
}
}
return(address);
}
/// <summary>
/// Registers an observer. This methods is invoked by an observer.
/// </summary>
/// <param name="topic">The topic.</param>
/// <param name="filterExpressions">Specifies filter expressions.</param>
/// <param name="entityId">The entity id of the observable.</param>
/// This method is called by a management service or actor.
/// <returns>The asynchronous result of the operation.</returns>
public async Task RegisterObserverServiceAsync(string topic, IEnumerable <string> filterExpressions, EntityId entityId)
{
if (string.IsNullOrWhiteSpace(topic))
{
throw new ArgumentException($"The {nameof(topic)} parameter cannot be null.", nameof(topic));
}
if (entityId == null)
{
throw new ArgumentException($"The {nameof(entityId)} parameter cannot be null.", nameof(entityId));
}
IList <string> expressions = filterExpressions as IList <string> ?? filterExpressions.ToList();
for (int k = 1; k <= ConfigurationHelper.MaxQueryRetryCount; k++)
{
try
{
EntityId id = await this.GetEntityIdAsync();
if (entityId.Kind == EntityKind.Actor)
{
IServerObservableActor actorProxy = ActorProxy.Create <IServerObservableActor>(entityId.ActorId, entityId.ServiceUri);
await actorProxy.RegisterObserverAsync(topic, expressions, id);
}
else
{
IServerObservableService serviceProxy = entityId.PartitionKey.HasValue
? ServiceProxy.Create <IServerObservableService>(entityId.ServiceUri, new ServicePartitionKey(entityId.PartitionKey.Value))
: ServiceProxy.Create <IServerObservableService>(entityId.ServiceUri);
await serviceProxy.RegisterObserverAsync(topic, expressions, id);
}
IReliableDictionary <string, Dictionary <Uri, EntityId> > topicsDictionary =
await this.StateManager.GetOrAddAsync <IReliableDictionary <string, Dictionary <Uri, EntityId> > >(Constants.TopicDictionary);
using (ITransaction transaction = this.StateManager.CreateTransaction())
{
ConditionalValue <Dictionary <Uri, EntityId> > result = await topicsDictionary.TryGetValueAsync(transaction, topic);
Dictionary <Uri, EntityId> observables = result.HasValue ? result.Value : new Dictionary <Uri, EntityId>();
if (!observables.ContainsKey(entityId.EntityUri))
{
observables.Add(entityId.EntityUri, entityId);
}
await topicsDictionary.AddOrUpdateAsync(transaction, topic, e => observables, (e, s) => observables);
await transaction.CommitAsync();
}
StringBuilder stringBuilder =
new StringBuilder(
$"Observer successfully registered.\r\n[Observable]: {entityId}\r\n[Observer]: {id}\r\n[Subscription]: Topic=[{topic}]");
int i = 1;
foreach (string expression in expressions.Where(expression => !string.IsNullOrWhiteSpace(expression)))
{
stringBuilder.Append($" FilterExpression[{i++}]=[{expression}]");
}
ServiceEventSource.Current.Message(stringBuilder.ToString());
return;
}
catch (FabricTransientException ex)
{
ServiceEventSource.Current.Error(ex);
}
catch (AggregateException ex)
{
foreach (Exception e in ex.InnerExceptions)
{
ServiceEventSource.Current.Error(e);
}
throw;
}
catch (Exception ex)
{
ServiceEventSource.Current.Error(ex);
throw;
}
await Task.Delay(ConfigurationHelper.BackoffQueryDelay);
}
throw new TimeoutException(Constants.RetryTimeoutExhausted);
}
public async Task <IActionResult> EndGame(string playerid, string playerdata)
{
try
{
if (!PlayerManager.IsActive)
{
return new ContentResult {
StatusCode = 500, Content = "Service is still starting up. Please retry."
}
}
;
// Get newer game data from the active room
Player player = JsonConvert.DeserializeObject <Player>(playerdata);
IReliableDictionary <string, PlayerPackage> playdict =
await this.stateManager.GetOrAddAsync <IReliableDictionary <string, PlayerPackage> >(PlayersDictionaryName);
using (ITransaction tx = this.stateManager.CreateTransaction())
{
ConditionalValue <PlayerPackage> playerOption = await playdict.TryGetValueAsync(tx, playerid, LockMode.Update);
if (!playerOption.HasValue)
{
// Tried to end game for a player that isn't here. This is a fail state.
await tx.CommitAsync();
return(new ContentResult {
StatusCode = 500, Content = "Cannot log out a player not in this system. Check partition."
});
}
// Player says already logged out, this means the last log in attempt was successful, but the return message never got back to
// room manager or it failed to remove the player.
if (playerOption.Value.State == LogState.LoggedOut)
{
await tx.CommitAsync();
return(new ContentResult {
StatusCode = 200
});
}
//The normal functionality, update the player and return a success
if (playerOption.Value.State == LogState.LoggedIn)
{
PlayerPackage newPlayerPackage = playerOption.Value;
newPlayerPackage.Player = player;
newPlayerPackage.State = LogState.LoggedOut;
await playdict.SetAsync(tx, playerid, newPlayerPackage);
await tx.CommitAsync();
return(new ContentResult {
StatusCode = 200
});
}
await tx.CommitAsync();
Environment.FailFast("Player must have one of the above states: doesn't exist, loggedin, or loggedout.");
return(new ContentResult {
StatusCode = 500
});
}
}
catch (Exception e)
{
return(exceptionHandler(e));
}
}
/// <summary>
/// Removes the given quantity of stock from an in item in the inventory.
/// </summary>
/// <param name="request"></param>
/// <returns>int: Returns the quantity removed from stock.</returns>
public async Task <int> RemoveStockAsync(InventoryItemId itemId, int quantity, CustomerOrderActorMessageId amId)
{
ServiceEventSource.Current.ServiceMessage(this, "inside remove stock {0}|{1}", amId.GetHashCode(), amId.GetHashCode());
IReliableDictionary <InventoryItemId, InventoryItem> inventoryItems =
await this.StateManager.GetOrAddAsync <IReliableDictionary <InventoryItemId, InventoryItem> >(InventoryItemDictionaryName);
IReliableDictionary <CustomerOrderActorMessageId, DateTime> recentRequests =
await this.StateManager.GetOrAddAsync <IReliableDictionary <CustomerOrderActorMessageId, DateTime> >(ActorMessageDictionaryName);
IReliableDictionary <CustomerOrderActorMessageId, Tuple <InventoryItemId, int> > requestHistory =
await
this.StateManager.GetOrAddAsync <IReliableDictionary <CustomerOrderActorMessageId, Tuple <InventoryItemId, int> > >(RequestHistoryDictionaryName);
int removed = 0;
ServiceEventSource.Current.ServiceMessage(this, "Received remove stock request. Item: {0}. Quantity: {1}.", itemId, quantity);
using (ITransaction tx = this.StateManager.CreateTransaction())
{
//first let's see if this is a duplicate request
ConditionalValue <DateTime> previousRequest = await recentRequests.TryGetValueAsync(tx, amId);
if (!previousRequest.HasValue)
{
//first time we've seen the request or it was a dupe from so long ago we have forgotten
// Try to get the InventoryItem for the ID in the request.
ConditionalValue <InventoryItem> item = await inventoryItems.TryGetValueAsync(tx, itemId);
// We can only remove stock for InventoryItems in the system.
if (item.HasValue)
{
// Update the stock quantity of the item.
// This only updates the copy of the Inventory Item that's in local memory here;
// It's not yet saved in the dictionary.
removed = item.Value.RemoveStock(quantity);
// We have to store the item back in the dictionary in order to actually save it.
// This will then replicate the updated item
await inventoryItems.SetAsync(tx, itemId, item.Value);
//we also have to make a note that we have returned this result, so that we can protect
//ourselves from stale or duplicate requests that come back later
await requestHistory.SetAsync(tx, amId, new Tuple <InventoryItemId, int>(itemId, removed));
ServiceEventSource.Current.ServiceMessage(
this,
"Removed stock complete. Item: {0}. Removed: {1}. Remaining: {2}",
item.Value.Id,
removed,
item.Value.AvailableStock);
}
}
else
{
//this is a duplicate request. We need to send back the result we already came up with and hope they get it this time
//find the previous result and send it back
ConditionalValue <Tuple <InventoryItemId, int> > previousResponse = await requestHistory.TryGetValueAsync(tx, amId);
if (previousResponse.HasValue)
{
removed = previousResponse.Value.Item2;
ServiceEventSource.Current.ServiceMessage(
this,
"Retrieved previous response for request {0}, from {1}, for Item {2} and quantity {3}",
amId,
previousRequest.Value,
previousResponse.Value.Item1,
previousResponse.Value.Item2);
}
else
{
//we've seen the request before but we don't have a record for what we responded, inconsistent state
ServiceEventSource.Current.ServiceMessage(
this,
"Inconsistent State: recieved duplicate request {0} but don't have matching response in history",
amId);
this.Partition.ReportFault(System.Fabric.FaultType.Transient);
}
//note about duplicate Requests: technically if a duplicate request comes in and we have
//sufficient invintory to return more now that we did previously, we could return more of the order and decrement
//the difference to reduce the total number of round trips. This optimization is not currently implemented
}
//always update the datetime for the given request
await recentRequests.SetAsync(tx, amId, DateTime.UtcNow);
// nothing will happen unless we commit the transaction!
ServiceEventSource.Current.Message("Committing Changes in Inventory Service");
await tx.CommitAsync();
ServiceEventSource.Current.Message("Inventory Service Changes Committed");
}
ServiceEventSource.Current.Message("Removed {0} of item {1}", removed, itemId);
return(removed);
}
protected override async Task RunAsync(CancellationToken cancellationToken)
{
IReliableQueue <string> queue = await this.StateManager.GetOrAddAsync <IReliableQueue <string> >("jobQueue");
IReliableDictionary <string, Job> dictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <string, Job> >("jobs");
try
{
// need to restart any existing jobs after failover
using (ITransaction tx = this.StateManager.CreateTransaction())
{
var enumerable = await dictionary.CreateEnumerableAsync(tx);
var enumerator = enumerable.GetAsyncEnumerator();
while (await enumerator.MoveNextAsync(cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
Job job = enumerator.Current.Value;
this.runningJobs.Add(this.StartJob(job, cancellationToken));
}
}
// start processing new jobs from the queue.
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <string> dequeueResult = await queue.TryDequeueAsync(tx);
if (!dequeueResult.HasValue)
{
await Task.Delay(TimeSpan.FromMilliseconds(500), cancellationToken);
continue;
}
string jobName = dequeueResult.Value;
ConditionalValue <Job> getResult = await dictionary.TryGetValueAsync(tx, jobName, LockMode.Update);
if (getResult.HasValue)
{
Job job = getResult.Value;
this.runningJobs.Add(this.StartJob(job, cancellationToken));
await dictionary.SetAsync(tx, jobName, new Job(job.Name, job.Parameters, job.Running));
}
await tx.CommitAsync();
}
}
catch (FabricTransientException)
{
await Task.Delay(TimeSpan.FromMilliseconds(500), cancellationToken);
}
catch (TimeoutException)
{
await Task.Delay(TimeSpan.FromMilliseconds(500), cancellationToken);
}
}
}
catch (OperationCanceledException)
{
await Task.WhenAll(this.runningJobs);
throw;
}
}
/// <summary>
/// This is the main entry point for your service replica.
/// This method executes when this replica of your service becomes primary and has write status.
/// </summary>
/// <param name="cancellationToken">Canceled when Service Fabric needs to shut down this service replica.</param>
protected override async Task RunAsync(CancellationToken cancellationToken)
{
IReliableDictionary <string, long> myDictionary =
await this.StateManager.GetOrAddAsync <IReliableDictionary <string, long> >
(CountDictionaryName);
bool takeBackup = false;
bool takeFullBackup = false;
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
using (var tx = this.StateManager.CreateTransaction())
{
var result = await myDictionary.TryGetValueAsync(tx, "Counter");
if (result.HasValue)
{
ServiceEventSource.Current.ServiceMessage(this, "Current Counter Value: {0}", result.Value.ToString());
}
else
{
ServiceEventSource.Current.ServiceMessage(this, "Value does not exist and will be added to the reliable dictionary...");
}
// Setting a flag that will be true when the counter in the reliable dictionary
// hits a multiple of 100
long newCount = await myDictionary.AddOrUpdateAsync(tx, "Counter", 0,
(key, value) => ++ value);
takeBackup = newCount > 0 && newCount % backupCount == 0;
// If an exception is thrown before calling CommitAsync, the transaction aborts, all changes are
// discarded, and nothing is saved to the secondary replicas.
await tx.CommitAsync();
}
// If the backup flag was set, then take a backup of this service's state
if (takeBackup)
{
ServiceEventSource.Current.ServiceMessage(this, "Backup initiated...");
// NOTE
// Here you could have logic to change the type of backup, full or incremental
// Indicate that we want a full backup, and to call BackupCallbackAsync when the backup is complete
if (!takeFullBackup)
{
BackupDescription backupDescription = new BackupDescription(BackupOption.Full,
this.BackupCallbackAsync);
incrementalCount = 0;
await base.BackupAsync(backupDescription);
takeFullBackup = true;
}
else
{
try
{
BackupDescription backupDescription = new BackupDescription(BackupOption.Incremental,
this.BackupCallbackAsync);
// Call BackupAsync, which is implemented in StatefulServiceBase (not in this code).
// Calling it prompts Service Fabric to do the backup you requested.
// All reliable objects are collected
// The BackupDescription object created above tells it what kind of backup and where to call
// this code back with status
incrementalCount++;
await base.BackupAsync(backupDescription);
}
catch (System.Exception ee)
{
}
}
}
else
{
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
}
}
}
/// <summary>
/// Creates an EventHubReceiver from the given connection sting and partition key.
/// The Reliable Dictionaries are used to create a receiver from wherever the service last left off,
/// or from the current date/time if it's the first time the service is coming up.
/// </summary>
/// <param name="connectionString"></param>
/// <param name="servicePartitionKey"></param>
/// <param name="epochDictionary"></param>
/// <param name="offsetDictionary"></param>
/// <returns></returns>
private async Task <Tuple <EventHubReceiver, MessagingFactory> > ConnectToIoTHubAsync(
string connectionString,
long servicePartitionKey,
IReliableDictionary <string, long> epochDictionary,
IReliableDictionary <string, string> offsetDictionary)
{
// EventHubs doesn't support NetMessaging, so ensure the transport type is AMQP.
ServiceBusConnectionStringBuilder connectionStringBuilder = new ServiceBusConnectionStringBuilder(connectionString);
connectionStringBuilder.TransportType = TransportType.Amqp;
// A new MessagingFactory is created here so that each partition of this service will have its own MessagingFactory.
// This gives each partition its own dedicated TCP connection to IoT Hub.
MessagingFactory messagingFactory = MessagingFactory.CreateFromConnectionString(connectionStringBuilder.ToString());
EventHubClient eventHubClient = messagingFactory.CreateEventHubClient("messages/events");
EventHubRuntimeInformation eventHubRuntimeInfo = await eventHubClient.GetRuntimeInformationAsync();
EventHubReceiver eventHubReceiver;
// Get an IoT Hub partition ID that corresponds to this partition's low key.
// This assumes that this service has a partition count 'n' that is equal to the IoT Hub partition count and a partition range of 0..n-1.
// For example, given an IoT Hub with 32 partitions, this service should be created with:
// partition count = 32
// partition range = 0..31
string eventHubPartitionId = eventHubRuntimeInfo.PartitionIds[servicePartitionKey];
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <string> offsetResult = await offsetDictionary.TryGetValueAsync(tx, "offset", LockMode.Default);
ConditionalValue <long> epochResult = await epochDictionary.TryGetValueAsync(tx, "epoch", LockMode.Update);
long newEpoch = epochResult.HasValue
? epochResult.Value + 1
: 0;
if (offsetResult.HasValue)
{
// continue where the service left off before the last failover or restart.
ServiceEventSource.Current.ServiceMessage(
this.Context,
"Creating EventHub listener on partition {0} with offset {1}",
eventHubPartitionId,
offsetResult.Value);
eventHubReceiver = await eventHubClient.GetDefaultConsumerGroup().CreateReceiverAsync(eventHubPartitionId, offsetResult.Value, newEpoch);
}
else
{
// first time this service is running so there is no offset value yet.
// start with the current time.
ServiceEventSource.Current.ServiceMessage(
this.Context,
"Creating EventHub listener on partition {0} with offset {1}",
eventHubPartitionId,
DateTime.UtcNow);
eventHubReceiver =
await
eventHubClient.GetDefaultConsumerGroup()
.CreateReceiverAsync(eventHubPartitionId, DateTime.UtcNow, newEpoch);
}
// epoch is recorded each time the service fails over or restarts.
await epochDictionary.SetAsync(tx, "epoch", newEpoch);
await tx.CommitAsync();
}
return(new Tuple <EventHubReceiver, MessagingFactory>(eventHubReceiver, messagingFactory));
}
/// <summary>
/// Sends data to observers for a given topic.
/// </summary>
/// <param name="topic">The topic.</param>
/// <param name="message">The current notification information.</param>
/// <param name="useObserverAsProxy">Observable uses one observer for each cluster node as a proxy when true,
/// it directly sends the message to all observers otherwise.</param>
/// <returns>The asynchronous result of the operation.</returns>
public async Task NotifyObserversAsync(string topic, Message message, bool useObserverAsProxy)
{
if (string.IsNullOrWhiteSpace(topic))
{
throw new ArgumentException($"The {nameof(topic)} parameter cannot be null.", nameof(topic));
}
if (string.IsNullOrWhiteSpace(message?.Body))
{
throw new ArgumentException($"The {nameof(message)} parameter cannot be null.", nameof(message));
}
try
{
EntityId id = await this.GetEntityIdAsync();
IReliableDictionary <string, Dictionary <Uri, ObserverInfo> > topicsDictionary =
await this.StateManager.GetOrAddAsync <IReliableDictionary <string, Dictionary <Uri, ObserverInfo> > >(Constants.TopicDictionary);
using (ITransaction transaction = this.StateManager.CreateTransaction())
{
ConditionalValue <Dictionary <Uri, ObserverInfo> > result = await topicsDictionary.TryGetValueAsync(transaction, topic);
if (!result.HasValue)
{
throw new ArgumentException($"{id} is not an observable for Topic=[{topic}]");
}
await transaction.CommitAsync();
}
List <Task> taskList = new List <Task>();
using (ITransaction transaction = this.StateManager.CreateTransaction())
{
ConditionalValue <Dictionary <Uri, ObserverInfo> > result = await topicsDictionary.TryGetValueAsync(transaction, topic);
if (result.HasValue)
{
Dictionary <Uri, ObserverInfo> observers = result.Value;
if (useObserverAsProxy)
{
if (JsonSerializerHelper.IsJson(message.Body))
{
// Create the list of observers: an observer is added to the list only at least of of its
// filter predicates is satisified by the message.
JObject jObject = JsonSerializerHelper.Deserialize(message.Body);
IEnumerable <EntityId> observerEnumerable = (from subscriptionInfo in observers.
Where(kvp => kvp.Value.Predicates.Any()).
Select(observer => observer.Value)
let ok = subscriptionInfo.Predicates.Any(predicate => predicate(jObject))
where ok
select subscriptionInfo.EntityId);
// observers are grouped by NodeName
taskList.AddRange(
observerEnumerable.
GroupBy(e => e.NodeName).
Select(groupingByNodeName => ProcessingHelper.GetObserverProxyAndList(groupingByNodeName, true)).
Select(tuple => ProcessingHelper.NotifyObserverAsync(topic, message, tuple.Item1, id, tuple.Item2)));
}
else
{
// observers are grouped by NodeName
taskList.AddRange(
observers.
Select(kvp => kvp.Value.EntityId).
GroupBy(e => e.NodeName).
Select(groupingByNodeName => ProcessingHelper.GetObserverProxyAndList(groupingByNodeName, true)).
Select(tuple => ProcessingHelper.NotifyObserverAsync(topic, message, tuple.Item1, id, tuple.Item2)));
}
}
else
{
if (JsonSerializerHelper.IsJson(message.Body))
{
JObject jObject = JsonSerializerHelper.Deserialize(message.Body);
taskList.AddRange(
(from subscriptionInfo in
observers.Where(kvp => kvp.Value.Predicates.Any()).
Select(observer => observer.Value)
let ok = subscriptionInfo.Predicates.Any(predicate => predicate(jObject))
where ok
select subscriptionInfo.EntityId).
Select(entityId => ProcessingHelper.NotifyObserverAsync(topic, message, entityId, id, null)));
}
else
{
taskList.AddRange(
observers.Select(
observer => ProcessingHelper.NotifyObserverAsync(topic, message, observer.Value.EntityId, id, null)));
}
}
}
await Task.WhenAll(taskList.ToArray());
await transaction.CommitAsync();
}
}
catch (AggregateException ex)
{
foreach (Exception e in ex.InnerExceptions)
{
ServiceEventSource.Current.Error(e);
}
throw;
}
catch (Exception ex)
{
ServiceEventSource.Current.Error(ex);
throw;
}
}
/// <summary>
/// Unregisters an entity as observable for a given topic.
/// This method is called by a management service or actor.
/// </summary>
/// <param name="topic">The topic.</param>
/// <param name="useObserverAsProxy">Observable uses one observer for each cluster node as a proxy when true,
/// it directly sends the message to all observers otherwise.</param>
/// <returns>The asynchronous result of the operation.</returns>
public async Task UnregisterObservableServiceAsync(string topic, bool useObserverAsProxy)
{
if (string.IsNullOrWhiteSpace(topic))
{
throw new ArgumentException($"The {nameof(topic)} parameter cannot be null.", nameof(topic));
}
try
{
EntityId id = await this.GetEntityIdAsync();
if (id != null)
{
for (int k = 1; k <= ConfigurationHelper.MaxQueryRetryCount; k++)
{
try
{
IRegistryService registryService =
ServiceProxy.Create <IRegistryService>(ConfigurationHelper.RegistryServiceUri,
new ServicePartitionKey(PartitionResolver.Resolve(topic, ConfigurationHelper.RegistryServicePartitionCount)));
await registryService.UnregisterObservableAsync(topic, id);
break;
}
catch (FabricTransientException ex)
{
ServiceEventSource.Current.Error(ex);
if (k == ConfigurationHelper.MaxQueryRetryCount)
{
throw;
}
}
catch (AggregateException ex)
{
foreach (Exception innerException in ex.InnerExceptions)
{
ServiceEventSource.Current.Error(innerException);
}
if (k == ConfigurationHelper.MaxQueryRetryCount)
{
throw;
}
}
catch (Exception ex)
{
ServiceEventSource.Current.Error(ex);
if (k == ConfigurationHelper.MaxQueryRetryCount)
{
throw;
}
}
await Task.Delay(ConfigurationHelper.BackoffQueryDelay);
}
IReliableDictionary <string, Dictionary <Uri, ObserverInfo> > topicsDictionary =
await this.StateManager.GetOrAddAsync <IReliableDictionary <string, Dictionary <Uri, ObserverInfo> > >(Constants.TopicDictionary);
using (ITransaction transaction = this.StateManager.CreateTransaction())
{
List <Task> taskList = new List <Task>();
ConditionalValue <Dictionary <Uri, ObserverInfo> > result = await topicsDictionary.TryGetValueAsync(transaction, topic);
if (result.HasValue)
{
Dictionary <Uri, ObserverInfo> observers = result.Value;
if (useObserverAsProxy)
{
// observers are grouped by NodeName
taskList.AddRange(
observers.
Select(kvp => kvp.Value.EntityId).
GroupBy(e => e.NodeName).
Select(groupingByNodeName => ProcessingHelper.GetObserverProxyAndList(groupingByNodeName, true)).
Select(tuple => ProcessingHelper.UnregisterObservableAsync(topic, tuple.Item1, id, tuple.Item2)));
}
else
{
taskList.AddRange(
observers.Select(observer => ProcessingHelper.UnregisterObservableAsync(topic, observer.Value.EntityId, id, null)));
}
await Task.WhenAll(taskList.ToArray());
await transaction.CommitAsync();
}
}
}
ServiceEventSource.Current.Message($"Observable successfully unregistered.\r\n[Observable]: {id}\r\n[Publication]: Topic=[{topic}].");
}
catch (AggregateException ex)
{
foreach (Exception e in ex.InnerExceptions)
{
ServiceEventSource.Current.Error(e);
}
throw;
}
catch (Exception ex)
{
ServiceEventSource.Current.Error(ex);
throw;
}
}
/// <summary>
/// Unregisters an observer. This methods is invoked by an observer.
/// </summary>
/// <param name="topic">The topic.</param>
/// <param name="entityId">The entity id of the observer.</param>
/// <returns>The asynchronous result of the operation.</returns>
public async Task UnregisterObserverAsync(string topic, EntityId entityId)
{
if (string.IsNullOrWhiteSpace(topic))
{
throw new ArgumentException($"The {nameof(topic)} parameter cannot be null.", nameof(topic));
}
if (entityId == null)
{
throw new ArgumentException($"The {nameof(entityId)} parameter cannot be null.", nameof(entityId));
}
for (int k = 1; k <= ConfigurationHelper.MaxQueryRetryCount; k++)
{
try
{
EntityId id = await this.GetEntityIdAsync();
IReliableDictionary <string, Dictionary <Uri, ObserverInfo> > topicsDictionary =
await this.StateManager.GetOrAddAsync <IReliableDictionary <string, Dictionary <Uri, ObserverInfo> > >(Constants.TopicDictionary);
using (ITransaction transaction = this.StateManager.CreateTransaction())
{
ConditionalValue <Dictionary <Uri, ObserverInfo> > result = await topicsDictionary.TryGetValueAsync(transaction, topic);
if (!result.HasValue)
{
throw new ArgumentException($"{id} is not an observable for Topic=[{topic}]");
}
await transaction.CommitAsync();
}
using (ITransaction transaction = this.StateManager.CreateTransaction())
{
ConditionalValue <Dictionary <Uri, ObserverInfo> > result = await topicsDictionary.TryGetValueAsync(transaction, topic);
if (result.HasValue)
{
Dictionary <Uri, ObserverInfo> observers = result.Value;
if (observers.ContainsKey(entityId.EntityUri))
{
observers.Remove(entityId.EntityUri);
ServiceEventSource.Current.Message(
$"Observer successfully unregistered.\r\n[Observable]: {id}\r\n[Observer]: {entityId}\r\n[Subscription]: Topic=[{topic}]");
await topicsDictionary.AddOrUpdateAsync(transaction, topic, e => observers, (e, s) => observers);
}
}
await transaction.CommitAsync();
}
break;
}
catch (FabricTransientException ex)
{
ServiceEventSource.Current.Error(ex);
if (k == ConfigurationHelper.MaxQueryRetryCount)
{
throw;
}
}
catch (AggregateException ex)
{
foreach (Exception e in ex.InnerExceptions)
{
ServiceEventSource.Current.Error(e);
}
throw;
}
catch (Exception ex)
{
ServiceEventSource.Current.Error(ex);
throw;
}
await Task.Delay(ConfigurationHelper.BackoffQueryDelay);
}
if (this.ObserverUnregistered == null)
{
return;
}
try
{
Delegate[] invocationList = this.ObserverUnregistered.GetInvocationList();
Task[] handlerTasks = new Task[invocationList.Length];
SubscriptionEventArgs args = new SubscriptionEventArgs(topic, entityId);
for (int i = 0; i < invocationList.Length; i++)
{
handlerTasks[i] = ProcessingHelper.ExecuteEventHandlerAsync((Func <SubscriptionEventArgs, Task>)invocationList[i], args);
}
await Task.WhenAll(handlerTasks);
}
catch (AggregateException ex)
{
foreach (Exception e in ex.InnerExceptions)
{
ServiceEventSource.Current.Error(e);
}
}
catch (Exception ex)
{
ServiceEventSource.Current.Error(ex);
}
}
/// <summary>
/// Creates an EventHubReceiver from the given connection sting and partition key.
/// The Reliable Dictionaries are used to create a receiver from wherever the service last left off,
/// or from the current date/time if it's the first time the service is coming up.
/// </summary>
/// <param name="connectionString"></param>
/// <param name="servicePartitionKey"></param>
/// <param name="epochDictionary"></param>
/// <param name="offsetDictionary"></param>
/// <returns></returns>
private async Task<Tuple<EventHubReceiver, MessagingFactory>> ConnectToIoTHubAsync(
string connectionString,
long servicePartitionKey,
IReliableDictionary<string, long> epochDictionary,
IReliableDictionary<string, string> offsetDictionary)
{
// EventHubs doesn't support NetMessaging, so ensure the transport type is AMQP.
ServiceBusConnectionStringBuilder connectionStringBuilder = new ServiceBusConnectionStringBuilder(connectionString);
connectionStringBuilder.TransportType = TransportType.Amqp;
ServiceEventSource.Current.ServiceMessage(
this.Context,
"RouterService connecting to IoT Hub at {0}",
String.Join(",", connectionStringBuilder.Endpoints.Select(x => x.ToString())));
// A new MessagingFactory is created here so that each partition of this service will have its own MessagingFactory.
// This gives each partition its own dedicated TCP connection to IoT Hub.
MessagingFactory messagingFactory = MessagingFactory.CreateFromConnectionString(connectionStringBuilder.ToString());
EventHubClient eventHubClient = messagingFactory.CreateEventHubClient("messages/events");
EventHubRuntimeInformation eventHubRuntimeInfo = await eventHubClient.GetRuntimeInformationAsync();
EventHubReceiver eventHubReceiver;
// Get an IoT Hub partition ID that corresponds to this partition's low key.
// This assumes that this service has a partition count 'n' that is equal to the IoT Hub partition count and a partition range of 0..n-1.
// For example, given an IoT Hub with 32 partitions, this service should be created with:
// partition count = 32
// partition range = 0..31
string eventHubPartitionId = eventHubRuntimeInfo.PartitionIds[servicePartitionKey];
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue<string> offsetResult = await offsetDictionary.TryGetValueAsync(tx, "offset", LockMode.Default);
ConditionalValue<long> epochResult = await epochDictionary.TryGetValueAsync(tx, "epoch", LockMode.Update);
long newEpoch = epochResult.HasValue
? epochResult.Value + 1
: 0;
if (offsetResult.HasValue)
{
// continue where the service left off before the last failover or restart.
ServiceEventSource.Current.ServiceMessage(
this.Context,
"Creating EventHub listener on partition {0} with offset {1}",
eventHubPartitionId,
offsetResult.Value);
eventHubReceiver = await eventHubClient.GetDefaultConsumerGroup().CreateReceiverAsync(eventHubPartitionId, offsetResult.Value, newEpoch);
}
else
{
// first time this service is running so there is no offset value yet.
// start with the current time.
ServiceEventSource.Current.ServiceMessage(
this.Context,
"Creating EventHub listener on partition {0} with offset {1}",
eventHubPartitionId,
DateTime.UtcNow);
eventHubReceiver =
await
eventHubClient.GetDefaultConsumerGroup()
.CreateReceiverAsync(eventHubPartitionId, DateTime.UtcNow, newEpoch);
}
// epoch is recorded each time the service fails over or restarts.
await epochDictionary.SetAsync(tx, "epoch", newEpoch);
await tx.CommitAsync();
}
return new Tuple<EventHubReceiver, MessagingFactory>(eventHubReceiver, messagingFactory);
}
public async Task <IActionResult> CreateEntity(string name, string key, [FromBody] UserProfile userProfile)
{
bool bRet = false;
if (String.IsNullOrEmpty(name) || String.IsNullOrEmpty(key))
{
return(this.BadRequest());
}
if (userProfile != null)
{
Debug.WriteLine("On CreateEntity postContent=[" + userProfile.ToString() + "]");
}
else
{
Debug.WriteLine("On CreateEntity postContent=[ userProfile is null ]");
}
if (userProfile == null)
{
return(this.BadRequest());
}
string id = HashUtil.GetUniqueId();
User user = new User();
user.Id = id;
user.Username = userProfile.UserName;
user.FirstName = userProfile.FirstName;
user.LastName = userProfile.LastName;
user.Password = userProfile.Password;
user.PasswordCreated = true;
IReliableDictionary <string, string> identitiesDictionary = await this.stateManager.GetOrAddAsync <IReliableDictionary <string, string> >(Names.IdentitiesDictionaryName);
IReliableDictionary <string, User> entitiesDictionary = await this.stateManager.GetOrAddAsync <IReliableDictionary <string, User> >(Names.EntitiesDictionaryName);
using (ITransaction tx = this.stateManager.CreateTransaction())
{
int retryCount = 1;
while (retryCount > 0)
{
try
{
await identitiesDictionary.AddAsync(tx, userProfile.UserName, id);
await entitiesDictionary.AddAsync(tx, id, user);
// Commit
await tx.CommitAsync();
retryCount = 0;
}
catch (TimeoutException te)
{
// transient error. Could Retry if one desires .
ServiceEventSource.Current.ServiceMessage(this.context, $"DataService - CreateEntity(Save) - TimeoutException : Retry Count#{retryCount}: Message=[{te.ToString()}]");
if (global::Iot.Common.Names.TransactionsRetryCount > retryCount)
{
retryCount = 0;
}
else
{
retryCount++;
await Task.Delay(global::Iot.Common.Names.TransactionRetryWaitIntervalInMills *(int)Math.Pow(2, retryCount));
}
}
catch (Exception ex)
{
ServiceEventSource.Current.ServiceMessage(this.context, $"DataService - CreateEntity(Save) - General Exception - Message=[{0}]", ex);
retryCount = 0;
tx.Abort();
}
}
}
// now let's check if the commits have finished
using (ITransaction tx = this.stateManager.CreateTransaction())
{
try
{
bool keepReading = true;
while (keepReading)
{
var result = await identitiesDictionary.TryGetValueAsync(tx, userProfile.UserName);
if (result.Value.Equals(id))
{
await tx.CommitAsync();
bRet = true;
break;
}
Thread.Sleep(1000);
}
}
catch (TimeoutException te)
{
// transient error. Could Retry if one desires .
ServiceEventSource.Current.ServiceMessage(this.context, $"DataService - CreateEntity(Wait Save) - TimeoutException : Message=[{te.ToString()}]");
}
catch (Exception ex)
{
ServiceEventSource.Current.ServiceMessage(this.context, $"DataService - CreateEntity(Wait Save) - General Exception - Message=[{0}]", ex);
tx.Abort();
}
}
return(this.Ok(bRet));
}
private async Task <PartitionReceiver> ConnectToEventHubAsync(string eventHubConnectionString, string hubName,
IReliableDictionary <string, string> streamOffsetDictionary)
{
var eventHubHelper = new EventHubHelper();
var eventHubClient = eventHubHelper.CreatEventHubClientIfExist(eventHubConnectionString, hubName);
EventHubRuntimeInformation eventHubRuntimeInfo = await eventHubClient.GetRuntimeInformationAsync();
PartitionReceiver partitionReceiver = null;
string[] partitionIds = eventHubRuntimeInfo.PartitionIds;
_eventHubPartitionId = await GetMatchingEventHubPartitionId(partitionIds);
try
{
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <string> offsetResult = await streamOffsetDictionary.TryGetValueAsync(tx, Names.HubStreamOffSetKey);
EventPosition eventPosition;
if (offsetResult.HasValue)
{
// continue where the service left off before the last failover or restart.
eventPosition = EventPosition.FromSequenceNumber(long.Parse(offsetResult.Value));
}
else
{
// first time this service is running so there is no offset value yet. start with the current time.
// Load from database sequence number
eventPosition = await LoadEventPositionFromDatabaseAsync() ??
EventPosition.FromEnqueuedTime(DateTime.UtcNow);
//EventPosition.FromEnqueuedTime(DateTime.UtcNow.Subtract(TimeSpan.FromHours(5)));//EventPosition.FromEnqueuedTime(DateTime.UtcNow);
if (eventPosition.SequenceNumber != null)
{
_latestSequenceNumber = eventPosition.SequenceNumber.Value;
await streamOffsetDictionary.SetAsync(tx, Names.HubStreamOffSetKey, eventPosition.SequenceNumber.ToString());
await tx.CommitAsync();
}
}
ServiceEventSource.Current.ServiceMessage(this.Context, "Creating EventHub listener on partition {0} with SequenceNumber {1}",
_eventHubPartitionId, eventPosition.SequenceNumber);
partitionReceiver = eventHubClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, _eventHubPartitionId, eventPosition);
}
}
catch (Exception e)
{
//ServiceEventSource.Current.ServiceMessage(this.Context, $"RouterService ConnectToEventHubAsync met exception= ( {e} )");
string err = $"RouterService ConnectToEventHubAsync met exception, exception type={e.GetType().Name}, exception= {e.Message}, at partition ={Context.PartitionId} .";
//ServiceEventSource.Current.CriticalError("RouterService", err);
throw;
}
return(partitionReceiver);
}
/// <summary>
/// Creates an EventHubReceiver from the given connection sting and partition key.
/// The Reliable Dictionaries are used to create a receiver from wherever the service last left off,
/// or from the current date/time if it's the first time the service is coming up.
/// </summary>
/// <param name="connectionString"></param>
/// <param name="servicePartitionKey"></param>
/// <param name="epochDictionary"></param>
/// <param name="offsetDictionary"></param>
/// <returns></returns>
private async Task <Tuple <EventHubReceiver, MessagingFactory> > ConnectToIoTHubAsync(
string connectionString,
long servicePartitionKey,
IReliableDictionary <string, long> epochDictionary,
IReliableDictionary <string, string> offsetDictionary, string processOnlyFutureEvents)
{
// EventHubs doesn't support NetMessaging, so ensure the transport type is AMQP.
ServiceBusConnectionStringBuilder connectionStringBuilder = new ServiceBusConnectionStringBuilder(connectionString);
connectionStringBuilder.TransportType = TransportType.Amqp;
ServiceEventSource.Current.ServiceMessage(
this.Context,
$"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync - connecting to IoT Hub at {0}",
String.Join(",", connectionStringBuilder.Endpoints.Select(x => x.ToString())));
// A new MessagingFactory is created here so that each partition of this service will have its own MessagingFactory.
// This gives each partition its own dedicated TCP connection to IoT Hub.
MessagingFactory messagingFactory = MessagingFactory.CreateFromConnectionString(connectionStringBuilder.ToString());
EventHubClient eventHubClient = messagingFactory.CreateEventHubClient("messages/events");
EventHubRuntimeInformation eventHubRuntimeInfo = await eventHubClient.GetRuntimeInformationAsync();
EventHubReceiver eventHubReceiver = null;
// Get an IoT Hub partition ID that corresponds to this partition's low key.
// This assumes that this service has a partition count 'n' that is equal to the IoT Hub partition count and a partition range of 0..n-1.
// For example, given an IoT Hub with 32 partitions, this service should be created with:
// partition count = 32
// partition range = 0..31
string eventHubPartitionId = eventHubRuntimeInfo.PartitionIds[servicePartitionKey];
int retryCount = 1;
while (retryCount > 0)
{
try
{
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <string> offsetResult = await offsetDictionary.TryGetValueAsync(tx, "offset", LockMode.Default);
ConditionalValue <long> epochResult = await epochDictionary.TryGetValueAsync(tx, "epoch", LockMode.Update);
long newEpoch = epochResult.HasValue
? epochResult.Value + 1
: 0;
if (offsetResult.HasValue)
{
// continue where the service left off before the last failover or restart.
ServiceEventSource.Current.ServiceMessage(
this.Context,
$"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync -Creating EventHub listener on partition {eventHubPartitionId} with offset {offsetResult.Value}");
eventHubReceiver = await eventHubClient.GetDefaultConsumerGroup().CreateReceiverAsync(eventHubPartitionId, offsetResult.Value, newEpoch);
}
else
{
// first time this service is running so there is no offset value yet.
// start with the current time.
ServiceEventSource.Current.ServiceMessage(
this.Context,
$"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync - Creating EventHub listener on partition {eventHubPartitionId} with offset time now{DateTime.UtcNow} - Starting service");
if (processOnlyFutureEvents.Equals("yes"))
{
eventHubReceiver =
await
eventHubClient.GetDefaultConsumerGroup()
.CreateReceiverAsync(eventHubPartitionId, DateTime.UtcNow, newEpoch);
}
else
{
eventHubReceiver =
await
eventHubClient.GetDefaultConsumerGroup()
.CreateReceiverAsync(eventHubPartitionId, newEpoch);
}
}
// epoch is recorded each time the service fails over or restarts.
await epochDictionary.SetAsync(tx, "epoch", newEpoch);
await tx.CommitAsync();
retryCount = 0;
}
}
catch (TimeoutException te)
{
// transient error. Retry.
ServiceEventSource.Current.ServiceMessage(this.Context, $"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync - TimeoutException Retry Count#{retryCount} : Message=[{te.ToString()}]");
retryCount++;
await Task.Delay(global::Iot.Common.Names.IoTHubRetryWaitIntervalsInMills);
}
catch (FabricTransientException fte)
{
// transient error. Retry.
ServiceEventSource.Current.ServiceMessage(this.Context, $"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync - FabricTransientException : Message=[{fte.ToString()}]");
retryCount++;
await Task.Delay(global::Iot.Common.Names.IoTHubRetryWaitIntervalsInMills);
}
catch (FabricNotPrimaryException fnpe)
{
ServiceEventSource.Current.ServiceMessage(this.Context, $"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync - FabricNotPrimaryException Exception - Message=[{fnpe}]");
retryCount = 0;
}
catch (Exception ex)
{
ServiceEventSource.Current.ServiceMessage(this.Context, $"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync - General Exception - Message=[{ex}]");
retryCount = 0;
}
}
return(new Tuple <EventHubReceiver, MessagingFactory>(eventHubReceiver, messagingFactory));
}
protected override async Task RunAsync(CancellationToken runAsyncCancellationToken)
{
// This is to keep track of exceptions in the validation step at the end of
// each iteration of the ChaosTestScenario that is being used under the cover
//
bool validationExceptionCaught = false;
IReliableDictionary <string, CurrentState> chaosServiceState =
await this.StateManager.GetOrAddAsync <IReliableDictionary <string, CurrentState> >(StringResource.ChaosServiceStateKey);
using (ITransaction tx = this.StateManager.CreateTransaction())
{
if (!await chaosServiceState.ContainsKeyAsync(tx, StringResource.ChaosServiceStateKey, LockMode.Update))
{
await chaosServiceState.AddAsync(tx, StringResource.ChaosServiceStateKey, CurrentState.Stopped);
}
await tx.CommitAsync();
}
while (!runAsyncCancellationToken.IsCancellationRequested)
{
try
{
// check to see if we're in a "stop" or "start" state.
// this continues to poll until we're in a "start" state.
// a ReliableDictionary is used to store this information so that if the service
// fails over to another node, the state is preserved and the chaos test will continue to execute.
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <CurrentState> currentStateResult =
await chaosServiceState.TryGetValueAsync(tx, StringResource.ChaosServiceStateKey);
if (currentStateResult.HasValue &&
(currentStateResult.Value == CurrentState.Stopped ||
currentStateResult.Value == CurrentState.None))
{
await Task.Delay(Constants.IntervalBetweenLoopIteration, runAsyncCancellationToken);
continue;
}
}
// this section runs the actual chaos test.
// the cancellation token source is linked to the token provided to RunAsync so that we
// can stop the test if the service needs to shut down.
using (FabricClient fabricClient = new FabricClient())
{
using (this.stopEventTokenSource = CancellationTokenSource.CreateLinkedTokenSource(runAsyncCancellationToken))
{
// when a validation exception is caught, this waits for a while to let the cluster stabilize before continuing.
if (validationExceptionCaught)
{
await Task.Delay(ChaosTestConfigSettings.MaxClusterStabilizationTimeout, this.stopEventTokenSource.Token);
validationExceptionCaught = false;
}
ChaosTestScenarioParameters chaosScenarioParameters =
new ChaosTestScenarioParameters(
ChaosTestConfigSettings.MaxClusterStabilizationTimeout,
ChaosTestConfigSettings.MaxConcurrentFaults,
ChaosTestConfigSettings.EnableMoveReplicaFaults,
TimeSpan.MaxValue)
{
WaitTimeBetweenFaults =
ChaosTestConfigSettings.WaitTimeBetweenFaults,
WaitTimeBetweenIterations =
ChaosTestConfigSettings.WaitTimeBetweenIterations
};
ChaosTestScenario chaosTestScenario = new ChaosTestScenario(fabricClient, chaosScenarioParameters);
// capture progress events so we can report them back
chaosTestScenario.ProgressChanged += this.TestScenarioProgressChanged;
// this continuously runs the chaos test until the CancellationToken is signaled.
await chaosTestScenario.ExecuteAsync(this.stopEventTokenSource.Token);
}
}
}
catch (TimeoutException e)
{
string message = $"Caught TimeoutException '{e.Message}'. Will wait for cluster to stabilize before continuing test";
ServiceEventSource.Current.ServiceMessage(this, message);
validationExceptionCaught = true;
await this.StoreEventAsync(message);
}
catch (FabricValidationException e)
{
string message = $"Caught FabricValidationException '{e.Message}'. Will wait for cluster to stabilize before continuing test";
ServiceEventSource.Current.ServiceMessage(this, message);
validationExceptionCaught = true;
await this.StoreEventAsync(message);
}
catch (OperationCanceledException)
{
if (runAsyncCancellationToken.IsCancellationRequested)
{
// if RunAsync is canceled then we need to quit.
throw;
}
ServiceEventSource.Current.ServiceMessage(
this,
"Caught OperationCanceledException Exception during test execution. This is expected if test was stopped");
}
catch (AggregateException e)
{
if (e.InnerException is OperationCanceledException)
{
if (runAsyncCancellationToken.IsCancellationRequested)
{
// if RunAsync is canceled then we need to quit.
throw;
}
ServiceEventSource.Current.ServiceMessage(
this,
"Caught OperationCanceledException Exception during test execution. This is expected if test was stopped");
}
else
{
string message = $"Caught unexpected Exception during test excecution {e.InnerException}";
ServiceEventSource.Current.ServiceMessage(this, message);
await this.StoreEventAsync(message);
}
}
catch (Exception e)
{
string message = $"Caught unexpected Exception during test excecution {e}";
ServiceEventSource.Current.ServiceMessage(this, message);
await this.StoreEventAsync(message);
}
}
}
protected override async Task RunAsync(CancellationToken cancellationToken)
{
IReliableConcurrentQueue <ReportProcessingStep> processQueue
= await this.StateManager.GetOrAddAsync <IReliableConcurrentQueue <ReportProcessingStep> >(ProcessingQueueName);
IReliableDictionary <string, ReportStatus> statusDictionary
= await this.StateManager.GetOrAddAsync <IReliableDictionary <string, ReportStatus> >(StatusDictionaryName);
// queue up all the processing steps and create an initial processing status if one doesn't exist already
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <ReportStatus> tryGetResult
= await statusDictionary.TryGetValueAsync(tx, this.reportContext.Name, LockMode.Update);
if (!tryGetResult.HasValue)
{
foreach (string processingStep in processingSteps)
{
cancellationToken.ThrowIfCancellationRequested();
await processQueue.EnqueueAsync(tx, new ReportProcessingStep(processingStep));
}
await statusDictionary.AddAsync(tx, this.reportContext.Name, new ReportStatus(0, "Not started."));
}
await tx.CommitAsync();
}
// start processing and checkpoint between each step so we don't lose any progress in the event of a fail-over
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <ReportProcessingStep> dequeueResult = await processQueue.TryDequeueAsync(tx, cancellationToken);
if (!dequeueResult.HasValue)
{
// all done!
break;
}
ReportProcessingStep currentProcessingStep = dequeueResult.Value;
ServiceEventSource.Current.ServiceMessage(
this.Context,
$"Processing step: {currentProcessingStep.Name}");
// This takes a shared lock rather than an update lock
// because this is the only place the row is written to.
// If there were other writers, then this should be an update lock.
ConditionalValue <ReportStatus> dictionaryGetResult =
await statusDictionary.TryGetValueAsync(tx, this.reportContext.Name, LockMode.Default);
ReportStatus currentStatus = dictionaryGetResult.Value;
ReportStatus newStatus = await this.ProcessReport(currentStatus, currentProcessingStep, cancellationToken);
await statusDictionary.SetAsync(tx, this.reportContext.Name, newStatus);
await tx.CommitAsync();
}
}
catch (TimeoutException)
{
// transient error. Retry.
ServiceEventSource.Current.ServiceMessage(this.Context, "TimeoutException in RunAsync.");
}
catch (FabricTransientException fte)
{
// transient error. Retry.
ServiceEventSource.Current.ServiceMessage(this.Context, "FabricTransientException in RunAsync: {0}", fte.Message);
}
catch (FabricNotPrimaryException)
{
// not primary any more, time to quit.
return;
}
catch (FabricNotReadableException)
{
// retry or wait until not primary
ServiceEventSource.Current.ServiceMessage(this.Context, "FabricNotReadableException in RunAsync.");
}
catch (Exception ex)
{
// all other exceptions: log and re-throw.
ServiceEventSource.Current.ServiceMessage(this.Context, "Exception in RunAsync: {0}", ex.Message);
throw;
}
// delay between each to step to prevent starving other processing service instances.
await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken);
}
ServiceEventSource.Current.ServiceMessage(
this.Context,
$"Processing complete!");
}
/// <summary>
/// This is the main entry point for your service replica.
/// This method executes when this replica of your service becomes primary and has write status.
/// </summary>
/// <param name="cancellationToken">Canceled when Service Fabric needs to shut down this service replica.</param>
protected override async Task RunAsync(CancellationToken cancellationToken)
{
try
{
await _receiver.StartMessagePumpAsync(OnNewMessages, 1, cancellationToken);
//IEnumerable<QueueMessage> qm = await _receiver.ReceiveMessagesAsync(100);
await _publisher.PutMessagesAsync(new[] { QueueMessage.FromText("content at " + DateTime.UtcNow) });
//qm = await _receiver.ReceiveMessagesAsync(100);
//separate writes
await _blobs.WriteTextAsync("one", "test text 1");
await _blobs.WriteTextAsync("two", "test text 2");
//with transaction object
using (ITransaction tx = await _blobs.OpenTransactionAsync())
{
await _blobs.WriteTextAsync("three", "test text 1");
await _blobs.WriteTextAsync("four", "test text 2");
await tx.CommitAsync();
}
IEnumerable <BlobId> keys = await _blobs.ListAsync(null);
string textBack = await _blobs.ReadTextAsync("one");
textBack = await _blobs.ReadTextAsync("two");
}
catch (Exception ex)
{
throw;
}
IReliableDictionary <string, long> myDictionary = await StateManager.GetOrAddAsync <IReliableDictionary <string, long> >("myDictionary");
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
using (SFT tx = this.StateManager.CreateTransaction())
{
Microsoft.ServiceFabric.Data.ConditionalValue <long> result = await myDictionary.TryGetValueAsync(tx, "Counter");
ServiceEventSource.Current.ServiceMessage(this.Context, "Current Counter Value: {0}",
result.HasValue ? result.Value.ToString() : "Value does not exist.");
await myDictionary.AddOrUpdateAsync(tx, "Counter", 0, (key, value) => ++ value);
// If an exception is thrown before calling CommitAsync, the transaction aborts, all changes are
// discarded, and nothing is saved to the secondary replicas.
await tx.CommitAsync();
}
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
}
}
public async Task <IActionResult> Post(string deviceId)
{
IActionResult resultRet = this.Ok();
DateTime durationCounter = DateTime.UtcNow;
TimeSpan duration;
string traceId = FnvHash.GetUniqueId();
Stream req = Request.Body;
string eventsArray = new StreamReader(req).ReadToEnd();
if (String.IsNullOrEmpty(deviceId))
{
ServiceEventSource.Current.ServiceMessage(
this.context,
"Data Service - Received a Really Bad Request - device id not defined");
return(this.BadRequest());
}
if (eventsArray == null)
{
ServiceEventSource.Current.ServiceMessage(this.context, $"Data Service - Received Bad Request from device {deviceId}");
return(this.BadRequest());
}
DeviceMessage deviceMessage = EventRegistry.DeserializeEvents(deviceId, eventsArray, this.context, ServiceEventSource.Current);
if (deviceMessage == null)
{
ServiceEventSource.Current.ServiceMessage(this.context, $"Data Service - Received Bad Request from device {deviceId} - Error parsing message body [{eventsArray}]");
return(this.BadRequest());
}
ServiceEventSource.Current.ServiceMessage(
this.context,
$"Data Service - Received event from device {deviceId} for message type [{deviceMessage.MessageType}] timestamp [{deviceMessage.Timestamp}]- Traceid[{traceId}]");
IReliableDictionary <string, DeviceMessage> storeLatestMessage = await this.stateManager.GetOrAddAsync <IReliableDictionary <string, DeviceMessage> >(TargetSolution.Names.EventLatestDictionaryName);
IReliableDictionary <DateTimeOffset, DeviceMessage> storeCompletedMessages = await this.stateManager.GetOrAddAsync <IReliableDictionary <DateTimeOffset, DeviceMessage> >(TargetSolution.Names.EventHistoryDictionaryName);
string transactionType = "";
DeviceMessage completedMessage = null;
DateTimeOffset messageTimestamp = DateTimeOffset.UtcNow;
int retryCounter = 1;
MessageConfiguration messageConfiguration = EventRegistry.GetMessageConfiguration(deviceMessage.MessageType);
try
{
while (retryCounter > 0)
{
transactionType = "";
using (ITransaction tx = this.stateManager.CreateTransaction())
{
try
{
transactionType = "In Progress Message";
await storeLatestMessage.AddOrUpdateAsync(
tx,
deviceId,
deviceMessage,
(key, currentValue) =>
{
return(messageConfiguration.ManageDeviceEventSeriesContent(currentValue, deviceMessage, out completedMessage));
});
duration = DateTime.UtcNow.Subtract(durationCounter);
ServiceEventSource.Current.ServiceMessage(
this.context,
$"Data Service Received event from device {deviceId} - Finished [{transactionType}] - Duration [{duration.TotalMilliseconds}] mills - Traceid[{traceId}]");
await tx.CommitAsync();
retryCounter = 0;
duration = DateTime.UtcNow.Subtract(durationCounter);
ServiceEventSource.Current.ServiceMessage(
this.context,
$"Data Service - Finish commits to message with timestamp [{completedMessage.Timestamp.ToString()}] from device {deviceId} - Duration [{duration.TotalMilliseconds}] mills - Traceid[{traceId}]");
}
catch (TimeoutException tex)
{
if (global::Iot.Common.Names.TransactionsRetryCount > retryCounter)
{
ServiceEventSource.Current.ServiceMessage(
this.context,
$"Data Service Timeout Exception when saving [{transactionType}] data from device {deviceId} - Iteration #{retryCounter} - Message-[{tex}] - Traceid[{traceId}]");
await Task.Delay(global::Iot.Common.Names.TransactionRetryWaitIntervalInMills *(int)Math.Pow(2, retryCounter));
retryCounter++;
}
else
{
ServiceEventSource.Current.ServiceMessage(
this.context,
$"Data Service Timeout Exception when saving [{transactionType}] data from device {deviceId} - Iteration #{retryCounter} - Transaction Aborted - Message-[{tex}] - Traceid[{traceId}]");
resultRet = this.BadRequest();
retryCounter = 0;
}
}
}
}
}
catch (Exception ex)
{
ServiceEventSource.Current.ServiceMessage(
this.context,
$"Data Service Exception when saving [{transactionType}] data from device {deviceId} - Message-[{ex}] - - Traceid[{traceId}]");
}
if (completedMessage != null)
{
transactionType = "Check Message Timestamp";
retryCounter = 1;
while (retryCounter > 0)
{
try
{
using (ITransaction tx = this.stateManager.CreateTransaction())
{
bool tryAgain = true;
while (tryAgain)
{
ConditionalValue <DeviceMessage> storedCompletedMessageValue = await storeCompletedMessages.TryGetValueAsync(tx, messageTimestamp, LockMode.Default);
duration = DateTime.UtcNow.Subtract(durationCounter);
ServiceEventSource.Current.ServiceMessage(
this.context,
$"Message Completed (Look for duplication - result [{storedCompletedMessageValue.HasValue}] from device {deviceId} - Starting [{transactionType}] - Duration [{duration.TotalMilliseconds}] mills - Traceid[{traceId}]");
if (storedCompletedMessageValue.HasValue)
{
DeviceMessage storedCompletedMessage = storedCompletedMessageValue.Value;
if (completedMessage.DeviceId.Equals(storedCompletedMessage.DeviceId))
{
tryAgain = false; // this means this record was already saved before - no duplication necessary
ServiceEventSource.Current.ServiceMessage(
this.context,
$"Data Service - Message with timestamp {completedMessage.Timestamp.ToString()} from device {deviceId} already present in the store - (Ignore this duplicated record) - Traceid[{traceId}]");
completedMessage = null;
}
else
{
// this is a true collision between information from different devices
messageTimestamp = messageTimestamp.AddMilliseconds(10);
ServiceEventSource.Current.ServiceMessage(
this.context,
$"Data Service - Message with timestamp {completedMessage.Timestamp.ToString()} from device {deviceId} already present in the store - (Adjusted the timestamp) - Traceid[{traceId}]");
}
}
else
{
tryAgain = false;
}
}
await tx.CommitAsync();
retryCounter = 0;
}
}
catch (TimeoutException tex)
{
if (global::Iot.Common.Names.TransactionsRetryCount > retryCounter)
{
ServiceEventSource.Current.ServiceMessage(
this.context,
$"Data Service Timeout Exception when saving [{transactionType}] data from device {deviceId} - Iteration #{retryCounter} - Message-[{tex}] - Traceid[{traceId}]");
await Task.Delay(global::Iot.Common.Names.TransactionRetryWaitIntervalInMills *(int)Math.Pow(2, retryCounter));
retryCounter++;
}
else
{
ServiceEventSource.Current.ServiceMessage(
this.context,
$"Data Service Timeout Exception when saving [{transactionType}] data from device {deviceId} - Iteration #{retryCounter} - Transaction Aborted - Message-[{tex}] - Traceid[{traceId}]");
resultRet = this.BadRequest();
retryCounter = 0;
}
}
}
completedMessage.Timestamp = messageTimestamp;
transactionType = "Save Completed Message";
retryCounter = 1;
while (retryCounter > 0)
{
try
{
using (ITransaction tx = this.stateManager.CreateTransaction())
{
await storeCompletedMessages.AddOrUpdateAsync(
tx,
completedMessage.Timestamp,
completedMessage,
(key, currentValue) =>
{
return(completedMessage);
}
);
duration = DateTime.UtcNow.Subtract(durationCounter);
ServiceEventSource.Current.ServiceMessage(
this.context,
$"Completed message saved message to Completed Messages Store - Duration [{duration.TotalMilliseconds}] mills - Traceid[{traceId}]");
await tx.CommitAsync();
retryCounter = 0;
}
}
catch (TimeoutException tex)
{
if (global::Iot.Common.Names.TransactionsRetryCount > retryCounter)
{
ServiceEventSource.Current.ServiceMessage(
this.context,
$"Data Service Timeout Exception when saving [{transactionType}] data from device {deviceId} - Iteration #{retryCounter} - Message-[{tex}] - Traceid[{traceId}]");
await Task.Delay(global::Iot.Common.Names.TransactionRetryWaitIntervalInMills *(int)Math.Pow(2, retryCounter));
retryCounter++;
}
else
{
ServiceEventSource.Current.ServiceMessage(
this.context,
$"Data Service Timeout Exception when saving [{transactionType}] data from device {deviceId} - Iteration #{retryCounter} - Transaction Aborted - Message-[{tex}] - Traceid[{traceId}]");
resultRet = this.BadRequest();
retryCounter = 0;
}
}
}
duration = DateTime.UtcNow.Subtract(durationCounter);
ServiceEventSource.Current.ServiceMessage(
this.context,
$"Data Service - Saved Message to Complete Message Store with timestamp [{completedMessage.Timestamp.ToString()}] indexed by timestamp[{messageTimestamp}] from device {deviceId} - Duration [{duration.TotalMilliseconds}] mills - Traceid[{traceId}]");
}
duration = DateTime.UtcNow.Subtract(durationCounter);
ServiceEventSource.Current.ServiceMessage(
this.context,
$"Data Service Received event from device {deviceId} - Message completed Duration [{duration.TotalMilliseconds}] mills - Traceid[{traceId}]");
return(resultRet);
}
/// <summary>
/// Performs a HealthCheck for a scheduled item.
/// </summary>
/// <param name="item">WatchdogScheduledItem instance.</param>
internal async Task PerformItemHealthCheck(WatchdogScheduledItem item)
{
// Get the health check dictionaries.
IReliableDictionary <string, HealthCheck> dict = await this.GetHealthCheckDictionaryAsync();
IReliableDictionary <long, WatchdogScheduledItem> scheduleDict = await this.GetHealthCheckScheduleDictionaryAsync();
// Create a transaction.
using (ITransaction tx = this._service.StateManager.CreateTransaction())
{
// Attempt to get the HealthCheck instance for the key. If not return.
ConditionalValue <HealthCheck> cv = await dict.TryGetValueAsync(tx, item.Key, LockMode.Update);
if (cv.HasValue)
{
HealthCheck hc = cv.Value;
try
{
// Find the partition information that matches the partition identifier.
// If the partition isn't found, remove the health check item.
Partition partition = await this.FindMatchingPartitionAsync(hc.Partition);
if (null == partition)
{
await dict.TryRemoveAsync(tx, hc.Key, this._timeout, this._token);
}
else
{
// Execute the check and evaluate the results returned in the new HealthCheck instance.
hc = await this.ExecuteHealthCheckAsync(hc, partition);
// Update the value of the HealthCheck to store the results of the test.
await dict.TryUpdateAsync(tx, item.Key, hc, cv.Value);
// Remove the current scheduled item.
await scheduleDict.TryRemoveAsync(tx, item.ExecutionTicks);
// Add the new scheduled item.
WatchdogScheduledItem newItem = new WatchdogScheduledItem(hc.LastAttempt.Add(hc.Frequency), hc.Key);
await(scheduleDict.TryAddAsync(tx, newItem.ExecutionTicks, newItem));
}
// Commit the transaction.
await tx.CommitAsync();
}
catch (TimeoutException ex)
{
ServiceEventSource.Current.ServiceMessage(this._service.Context, ex.Message);
}
catch (FabricNotPrimaryException ex)
{
ServiceEventSource.Current.ServiceMessage(this._service.Context, ex.Message);
return;
}
catch (Exception ex)
{
ServiceEventSource.Current.ServiceMessage(this._service.Context, ex.Message);
throw;
}
}
}
}
public async Task <IActionResult> NewGame(string playerid, string roomid, string roomtype)
{
try
{
if (!PlayerManager.IsActive)
{
return new ContentResult {
StatusCode = 500, Content = "Service is still starting up. Please retry."
}
}
;
IReliableDictionary <string, PlayerPackage> playdict =
await this.stateManager.GetOrAddAsync <IReliableDictionary <string, PlayerPackage> >(PlayersDictionaryName);
PlayerPackage playerPackage; //for handing up player information if login is needed in scenario 2
using (ITransaction tx = this.stateManager.CreateTransaction())
{
ConditionalValue <PlayerPackage> playerOption = await playdict.TryGetValueAsync(tx, playerid, LockMode.Update);
/////////////////////////////////////////////////
// SCENARIO 1: PLAYER DOES NOT HAVE AN ACCOUNT //
/////////////////////////////////////////////////
if (!playerOption.HasValue)
{
//State: Player does not exist / Cannot be in a game
Random rand = new Random(Environment.TickCount);
//Generate a new player with a random position
Player newPlayer = new Player(
rand.Next() % 100 - 6,
rand.Next() % 96 - 6,
this.startingColors[rand.Next() % this.startingColors.Length]);
//Package the new player with its baseline statistics
PlayerPackage newPlayerPackage = new PlayerPackage(newPlayer, LogState.LoggedIn, 1, DateTime.UtcNow, roomid);
await playdict.AddAsync(tx, playerid, newPlayerPackage);
await tx.CommitAsync();
return(await this.NewGameRequestHelper(roomid, playerid, roomtype, newPlayer));
}
//////////////////////////////////////////////////////
// SCENARIO 2: PLAYER HAS ACCOUNT AND IS LOGGED OUT //
//////////////////////////////////////////////////////
if (playerOption.Value.State == LogState.LoggedOut)
{
/*
* Scenario: We think player is logged out (LO-N), in which case this is normal functionality.
* The state could also be (LO-LI), which could happen if an EndGame failed halfway through.
* If this is the case, there are two scenarios: The first is that the room we are about to log into was
* the room that failed to log out, in which case we will override that data since we have the most updated
* data and the situation is resolved. The second case is that we are trying to log into a different room.
* In this case we trust that the protocol has removed that clients access to the player, which means the
* player will eventually be cleaned up by the timeout, keeping the game consistent.
*/
//Grab our player data and update the package
PlayerPackage updatedPlayerPackage = playerOption.Value;
updatedPlayerPackage.State = LogState.LoggedIn;
updatedPlayerPackage.RoomId = roomid;
updatedPlayerPackage.NumLogins++;
await playdict.SetAsync(tx, playerid, updatedPlayerPackage);
//finish our transaction
await tx.CommitAsync();
// Request a newgame in the room we want to join
return(await this.NewGameRequestHelper(roomid, playerid, roomtype, playerOption.Value.Player));
}
await tx.CommitAsync();
playerPackage = playerOption.Value;
} // end of tx
/////////////////////////////////////////////////////
// SCENARIO 3: PLAYER HAS ACCOUNT AND IS LOGGED IN //
/////////////////////////////////////////////////////
if (playerPackage.State == LogState.LoggedIn)
{
// Scenario: This state will generally be the success state, where the player thinks they are logged in and the
// appropriate room has the game. However, during login, it is possible that the process crashed between the time
// that the login transaction marked the data as logged in and that data being put in the room. We must check to
// verify that this is not the state we are in.
int key = Partitioners.GetRoomPartition(playerPackage.RoomId);
// We first ask if the room has the data to determine which of the above states we are in.
string url = this.proxy + $"Exists/?playerid={playerid}&roomid={playerPackage.RoomId}&PartitionKind=Int64Range&PartitionKey={key}";
HttpResponseMessage response = await this.httpClient.GetAsync(url);
if ((int)response.StatusCode == 404)
{
this.RenewProxy();
url = this.proxy + $"Exists/?playerid={playerid}&roomid={playerPackage.RoomId}&PartitionKind=Int64Range&PartitionKey={key}";
response = await this.httpClient.GetAsync(url);
}
string responseMessage = await response.Content.ReadAsStringAsync();
if ((int)response.StatusCode == 200)
{
//Player is logged in, so we must deny this request
if (responseMessage == "true")
{
return new ContentResult {
StatusCode = 400, Content = "This player is already logged in"
}
}
;
//Player is not logged in, so we can log into whichever room we want
if (responseMessage == "false")
{
using (ITransaction tx1 = this.stateManager.CreateTransaction())
{
playerPackage.RoomId = roomid;
playerPackage.NumLogins++;
await playdict.SetAsync(tx1, playerid, playerPackage);
await tx1.CommitAsync();
}
return(await this.NewGameRequestHelper(roomid, playerid, roomtype, playerPackage.Player));
}
Environment.FailFast("If returning a success code, the message must be either true or false.");
}
else
{
return(new ContentResult {
StatusCode = 500, Content = "Something went wrong, please retry"
});
}
}
Environment.FailFast("Players must exist with a valid state attached to them");
return(new ContentResult {
StatusCode = 500
});
}
catch (Exception e)
{
return(exceptionHandler(e));
}
}
/// <summary>
/// Creates/updates the github issue.
/// </summary>
/// <param name="updateHistoryError">Error info for which github issue has to be created</param>
/// <param name="issueRepo">Repository where the github issue is created</param>
/// <param name="shouldReplaceDescription">Func that carries info the description has to be replaced </param>
/// <param name="description">Description for the issue body / comment body</param>
/// <returns></returns>
private async Task CreateOrUpdateGithubIssueAsync(
UpdateHistoryEntry updateHistoryError,
string issueRepo,
Func <string, string, bool> shouldReplaceDescription,
string description)
{
var parsedRepoUri = ParseRepoUri(issueRepo);
IGitHubClient client = await _authenticateGitHubApplicationClient.CreateGitHubClientAsync(parsedRepoUri.owner, parsedRepoUri.repo);
Repository repo = await client.Repository.Get(
parsedRepoUri.owner,
parsedRepoUri.repo);
var issueNumber = new ConditionalValue <int>();
switch (updateHistoryError)
{
case RepositoryBranchUpdateHistoryEntry repoBranchUpdateHistoryError:
{
_logger.LogInformation($"Error Message : '{repoBranchUpdateHistoryError.ErrorMessage}' in repository : '{repoBranchUpdateHistoryError.Repository}'");
IReliableDictionary <(string repository, string branch), int> gitHubIssueEvaluator =
await _stateManager.GetOrAddAsync <IReliableDictionary <(string repository, string branch), int> >("gitHubIssueEvaluator");
using (ITransaction tx = _stateManager.CreateTransaction())
{
issueNumber = await gitHubIssueEvaluator.TryGetValueAsync(
tx,
(repoBranchUpdateHistoryError.Repository,
repoBranchUpdateHistoryError.Branch));
await tx.CommitAsync();
}
if (issueNumber.HasValue)
{
// Found an existing issue, fall through to update.
break;
}
// Create a new issue for the error if the issue is already closed or the issue does not exist.
_logger.LogInformation($@"Creating a new gitHub issue for dependency Update Error, for the error message : '{repoBranchUpdateHistoryError.ErrorMessage} for the repository : '{repoBranchUpdateHistoryError.Repository}'");
await CreateDependencyUpdateErrorIssueAsync(
client,
repoBranchUpdateHistoryError,
gitHubIssueEvaluator,
description,
repo.Id,
issueRepo);
break;
}
case SubscriptionUpdateHistoryEntry subscriptionUpdateHistoryError:
{
_logger.LogInformation($"Error Message : '{subscriptionUpdateHistoryError.ErrorMessage}' in subscription : '{subscriptionUpdateHistoryError.SubscriptionId}'");
IReliableDictionary <Guid, int> gitHubIssueEvaluator =
await _stateManager.GetOrAddAsync <IReliableDictionary <Guid, int> >("gitHubSubscriptionIssueEvaluator");
using (ITransaction tx = _stateManager.CreateTransaction())
{
issueNumber = await gitHubIssueEvaluator.TryGetValueAsync(
tx,
subscriptionUpdateHistoryError.SubscriptionId);
await tx.CommitAsync();
}
if (issueNumber.HasValue)
{
// Found an existing issue, fall through to update.
break;
}
// Create a new issue for the error if the issue is already closed or the issue does not exist.
_logger.LogInformation($@"Creating a new gitHub issue for Subscription Update Error, for the error message : '{subscriptionUpdateHistoryError.ErrorMessage} for subscription : '{subscriptionUpdateHistoryError.SubscriptionId}'");
await CreateSubscriptionUpdateErrorIssueAsync(
client,
subscriptionUpdateHistoryError,
gitHubIssueEvaluator,
description,
repo.Id,
issueRepo);
break;
}
default:
throw new InvalidOperationException($"Unknown update history entry type: {updateHistoryError.GetType()}");
}
// Updating an existing issue; can use same codepath.
if (issueNumber.HasValue)
{
Issue issue = await client.Issue.Get(repo.Id, issueNumber.Value);
// check if the issue is open only then update it else create a new issue and update the dictionary.
if (issue.State.Equals("Open"))
{
_logger.LogInformation($@"Updating a gitHub issue number : '{issueNumber}' for the error : '{updateHistoryError.ErrorMessage}' for {GetPrintableDescription(updateHistoryError)}");
await UpdateIssueAsync(client, updateHistoryError, shouldReplaceDescription, description, issue, repo.Id);
return;
}
}
}
public async Task <ConditionalValue <ReceivedProviderEarningsEvent> > TryGet(string key, CancellationToken cancellationToken = default(CancellationToken))
{
var value = await state.TryGetValueAsync(transactionProvider.Current, key, TimeSpan.FromSeconds(2), cancellationToken).ConfigureAwait(false);
return(new ConditionalValue <ReceivedProviderEarningsEvent>(value.HasValue, value.Value));
}
// An interface method which is for disconfiguring the appliations thereby deleting their entries in reliable dictionary.
public async Task <string> DisconfigureApplication(string applicationName, string primaryCluster, string secondaryCluster)
{
List <String> keysToRemove = new List <String>();
IReliableDictionary <String, PartitionWrapper> myDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <String, PartitionWrapper> >("partitionDictionary");
using (ITransaction tx = this.StateManager.CreateTransaction())
{
IAsyncEnumerable <KeyValuePair <String, PartitionWrapper> > enumerable = await myDictionary.CreateEnumerableAsync(tx);
IAsyncEnumerator <KeyValuePair <String, PartitionWrapper> > asyncEnumerator = enumerable.GetAsyncEnumerator();
while (await asyncEnumerator.MoveNextAsync(CancellationToken.None))
{
PartitionWrapper secondaryPartition = asyncEnumerator.Current.Value;
String partitionAccessKey = asyncEnumerator.Current.Key;
if (Utility.isPartitionFromPrimarySecondaryCombination(partitionAccessKey, primaryCluster, secondaryCluster))
{
if (secondaryPartition.applicationName.ToString().Equals(applicationName))
{
keysToRemove.Add(asyncEnumerator.Current.Key);
}
}
}
await tx.CommitAsync();
}
bool allPartitionsRemoved = true;
using (ITransaction tx = this.StateManager.CreateTransaction())
{
foreach (String key in keysToRemove)
{
ConditionalValue <PartitionWrapper> value = myDictionary.TryRemoveAsync(tx, key).Result;
if (!value.HasValue)
{
allPartitionsRemoved = false;
}
}
await tx.CommitAsync();
}
IReliableDictionary <String, List <String> > configuredApplicationsDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <String, List <String> > >("configuredApplicationsDictionary");
using (var tx = this.StateManager.CreateTransaction())
{
String primarySecondaryJoin = Utility.getPrimarySecondaryClusterJoin(primaryCluster, secondaryCluster);
ConditionalValue <List <String> > applicationsList = await configuredApplicationsDictionary.TryGetValueAsync(tx, primarySecondaryJoin);
if (applicationsList.HasValue)
{
List <String> configuredApplicationsList = applicationsList.Value;
configuredApplicationsList.Remove(applicationName);
var result = await configuredApplicationsDictionary.TryAddAsync(tx, primarySecondaryJoin, configuredApplicationsList);
}
await tx.CommitAsync();
}
if (allPartitionsRemoved)
{
return(applicationName);
}
return(null);
}
public async Task OnTimerTick()
{
// On every timer tick calls this method which goes through the workflowsInProgress dictionary
// and removes the completed tasks and updates the partition metadata.
// For every partition mapping in the reliable dictionary if the corresponding task is not present in the
// workflowsInProgress dictionary it will create a task and puts in the dictionary
IReliableDictionary <String, PartitionWrapper> partitionDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <String, PartitionWrapper> >("partitionDictionary");
List <String> keysToRemove = new List <String>();
if (workFlowsInProgress.Count != 0)
{
try
{
foreach (KeyValuePair <String, Task <RestoreResult> > workFlow in workFlowsInProgress)
{
Task <RestoreResult> task = workFlow.Value;
if (task.IsCompleted)
{
RestoreResult restoreResult = task.Result;
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <PartitionWrapper> partitionWrapper = await partitionDictionary.TryGetValueAsync(tx, workFlow.Key);
if (partitionWrapper.HasValue)
{
if (restoreResult == null)
{
PartitionWrapper updatedPartitionWrapper = ObjectExtensions.Copy(partitionWrapper.Value);
updatedPartitionWrapper.CurrentlyUnderRestore = null;
await partitionDictionary.SetAsync(tx, workFlow.Key, updatedPartitionWrapper);
ServiceEventSource.Current.ServiceMessage(this.Context, "Restore Task returned null!!! ");
}
else if (restoreResult.restoreState.Equals("Success"))
{
PartitionWrapper updatedPartitionWrapper = ObjectExtensions.Copy(partitionWrapper.Value);
updatedPartitionWrapper.LastBackupRestored = restoreResult.restoreInfo;
updatedPartitionWrapper.CurrentlyUnderRestore = null;
await partitionDictionary.SetAsync(tx, workFlow.Key, updatedPartitionWrapper);
ServiceEventSource.Current.ServiceMessage(this.Context, "Restored succcessfully!!! ");
}
}
await tx.CommitAsync();
}
keysToRemove.Add(workFlow.Key);
}
}
foreach (var key in keysToRemove)
{
workFlowsInProgress.Remove(key);
}
}
catch (Exception ex)
{
ServiceEventSource.Current.Message("exception caught : {0}", ex);
}
}
using (ITransaction tx = this.StateManager.CreateTransaction())
{
IAsyncEnumerable <KeyValuePair <String, PartitionWrapper> > enumerable = await partitionDictionary.CreateEnumerableAsync(tx);
IAsyncEnumerator <KeyValuePair <String, PartitionWrapper> > asyncEnumerator = enumerable.GetAsyncEnumerator();
while (await asyncEnumerator.MoveNextAsync(CancellationToken.None))
{
String primaryPartition = asyncEnumerator.Current.Key;
PartitionWrapper secondaryPartition = asyncEnumerator.Current.Value;
if (secondaryPartition == null)
{
continue;
}
JToken backupInfoToken = await GetLatestBackupAvailable(secondaryPartition.primaryPartitionId, secondaryPartition.primaryCluster.httpEndpoint, secondaryPartition.primaryCluster.certificateThumbprint);
if (backupInfoToken == null)
{
continue;
}
BackupInfo backupInfo = new BackupInfo(backupInfoToken["BackupId"].ToString(), backupInfoToken["BackupLocation"].ToString(), (DateTime)backupInfoToken["CreationTimeUtc"]);
string backupPolicy = await GetPolicy(secondaryPartition.primaryCluster.httpEndpoint, secondaryPartition.primaryCluster.certificateThumbprint, secondaryPartition.primaryPartitionId);
if (backupPolicy == null)
{
continue;
}
Task <RestoreResult> task = workFlowsInProgress.TryGetValue(primaryPartition, out Task <RestoreResult> value) ? value : null;
if (task == null)
{
if (secondaryPartition.LastBackupRestored == null || DateTime.Compare(backupInfo.backupTime, secondaryPartition.LastBackupRestored.backupTime) > 0)
{
Task <RestoreResult> restoreTask = Task <RestoreResult> .Run(() => RestoreWorkFlow(backupInfoToken, backupPolicy, secondaryPartition, secondaryPartition.secondaryCluster.httpEndpoint, secondaryPartition.secondaryCluster.certificateThumbprint));
workFlowsInProgress.Add(asyncEnumerator.Current.Key, restoreTask);
PartitionWrapper updatedPartitionWrapper = ObjectExtensions.Copy(secondaryPartition);
updatedPartitionWrapper.LatestBackupAvailable = backupInfo;
updatedPartitionWrapper.CurrentlyUnderRestore = backupInfo;
await partitionDictionary.SetAsync(tx, primaryPartition, updatedPartitionWrapper);
}
else
{
continue;
}
}
else if (task.IsCompleted)
{
RestoreResult restoreResult = task.Result;
if (restoreResult.restoreState.Equals("Success"))
{
PartitionWrapper updatedPartitionWrapper = ObjectExtensions.Copy(secondaryPartition);
updatedPartitionWrapper.LastBackupRestored = restoreResult.restoreInfo;
updatedPartitionWrapper.CurrentlyUnderRestore = null;
await partitionDictionary.SetAsync(tx, primaryPartition, updatedPartitionWrapper);
ServiceEventSource.Current.ServiceMessage(this.Context, "Successfully Restored!!! ");
}
workFlowsInProgress.Remove(primaryPartition);
if (secondaryPartition.LastBackupRestored == null || DateTime.Compare(backupInfo.backupTime, secondaryPartition.LastBackupRestored.backupTime) > 0)
{
Task <RestoreResult> restoreTask = Task <string> .Run(() => RestoreWorkFlow(backupInfoToken, backupPolicy, secondaryPartition, secondaryPartition.secondaryCluster.httpEndpoint, secondaryPartition.secondaryCluster.certificateThumbprint));
workFlowsInProgress.Add(primaryPartition, restoreTask);
PartitionWrapper updatedPartitionWrapper = ObjectExtensions.Copy(secondaryPartition);
updatedPartitionWrapper.LatestBackupAvailable = backupInfo;
updatedPartitionWrapper.CurrentlyUnderRestore = backupInfo;
await partitionDictionary.SetAsync(tx, primaryPartition, updatedPartitionWrapper);
}
else
{
continue;
}
}
else
{
PartitionWrapper updatedPartitionWrapper = ObjectExtensions.Copy(secondaryPartition);
updatedPartitionWrapper.LatestBackupAvailable = backupInfo;
await partitionDictionary.SetAsync(tx, primaryPartition, updatedPartitionWrapper);
}
}
await tx.CommitAsync();
}
}
/// <summary>
/// Processes a request to join a cluster.
/// </summary>
/// <param name="clusterId"></param>
/// <param name="user"></param>
/// <returns></returns>
public async Task JoinClusterAsync(int clusterId, string userEmail)
{
if (String.IsNullOrWhiteSpace(userEmail))
{
throw new ArgumentNullException("userEmail");
}
ServiceEventSource.Current.ServiceMessage(this, "Join cluster request. Cluster: {0}.", clusterId);
IReliableDictionary <int, Cluster> clusterDictionary =
await this.StateManager.GetOrAddAsync <IReliableDictionary <int, Cluster> >(ClusterDictionaryName);
using (ITransaction tx = this.StateManager.CreateTransaction())
{
IAsyncEnumerable <KeyValuePair <int, Cluster> > clusterAsyncEnumerable = await clusterDictionary.CreateEnumerableAsync(tx);
await clusterAsyncEnumerable.ForeachAsync(
CancellationToken.None,
item =>
{
if (item.Value.Users.Any(x => String.Equals(x.Email, userEmail, StringComparison.OrdinalIgnoreCase)))
{
ServiceEventSource.Current.ServiceMessage(
this,
"Join cluster request failed. User already exists on cluster: {0}.",
item.Key);
throw new JoinClusterFailedException(JoinClusterFailedReason.UserAlreadyJoined);
}
});
ConditionalValue <Cluster> result = await clusterDictionary.TryGetValueAsync(tx, clusterId, LockMode.Update);
if (!result.HasValue)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Join cluster request failed. Cluster does not exist. Cluster ID: {0}.",
clusterId);
throw new JoinClusterFailedException(JoinClusterFailedReason.ClusterDoesNotExist);
}
Cluster cluster = result.Value;
// make sure the cluster isn't about to be deleted.
if ((DateTimeOffset.UtcNow - cluster.CreatedOn.ToUniversalTime()) > (this.config.MaximumClusterUptime))
{
ServiceEventSource.Current.ServiceMessage(
this,
"Join cluster request failed. Cluster has expired. Cluster: {0}. Cluster creation time: {1}",
clusterId,
cluster.CreatedOn.ToUniversalTime());
throw new JoinClusterFailedException(JoinClusterFailedReason.ClusterExpired);
}
// make sure the cluster is ready
if (cluster.Status != ClusterStatus.Ready)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Join cluster request failed. Cluster is not ready. Cluster: {0}. Status: {1}",
clusterId,
cluster.Status);
throw new JoinClusterFailedException(JoinClusterFailedReason.ClusterNotReady);
}
if (cluster.Users.Count() >= this.config.MaximumUsersPerCluster)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Join cluster request failed. Cluster is full. Cluster: {0}. Users: {1}",
clusterId,
cluster.Users.Count());
throw new JoinClusterFailedException(JoinClusterFailedReason.ClusterFull);
}
int userPort;
string clusterAddress = cluster.Address;
TimeSpan clusterTimeRemaining = this.config.MaximumClusterUptime - (DateTimeOffset.UtcNow - cluster.CreatedOn);
DateTimeOffset clusterExpiration = cluster.CreatedOn + this.config.MaximumClusterUptime;
try
{
userPort = cluster.Ports.First(port => !cluster.Users.Any(x => x.Port == port));
}
catch (InvalidOperationException)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Join cluster request failed. No available ports. Cluster: {0}. Users: {1}. Ports: {2}",
clusterId,
cluster.Users.Count(),
cluster.Ports.Count());
throw new JoinClusterFailedException(JoinClusterFailedReason.NoPortsAvailable);
}
try
{
ServiceEventSource.Current.ServiceMessage(this, "Sending join mail. Cluster: {0}.", clusterId);
List <HyperlinkView> links = new List <HyperlinkView>();
links.Add(
new HyperlinkView(
"http://" + clusterAddress + ":" + ClusterHttpGatewayPort + "/Explorer/index.html",
"Service Fabric Explorer",
"explore what's on the cluster with the built-in Service Fabric Explorer."));
try
{
IEnumerable <ApplicationView> applications =
await this.applicationDeployService.GetApplicationDeploymentsAsync(cluster.Address, ClusterConnectionPort);
links.AddRange(applications.Select(x => x.EntryServiceInfo));
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(this, "Failed to get application deployment info. {0}.", e.GetActualMessage());
}
await this.mailer.SendJoinMail(
userEmail,
clusterAddress + ":" + ClusterConnectionPort,
userPort,
clusterTimeRemaining,
clusterExpiration,
links);
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(this, "Failed to send join mail. {0}.", e.GetActualMessage());
throw new JoinClusterFailedException(JoinClusterFailedReason.SendMailFailed);
}
List <ClusterUser> newUserList = new List <ClusterUser>(cluster.Users);
newUserList.Add(new ClusterUser(userEmail, userPort));
Cluster updatedCluster = new Cluster(
cluster.InternalName,
cluster.Status,
cluster.AppCount,
cluster.ServiceCount,
cluster.Address,
cluster.Ports,
newUserList,
cluster.CreatedOn);
await clusterDictionary.SetAsync(tx, clusterId, updatedCluster);
await tx.CommitAsync();
}
ServiceEventSource.Current.ServiceMessage(this, "Join cluster request completed. Cluster: {0}.", clusterId);
}