private void AssignPartyAsRequeued(GatewayInternalService.GatewayInternalServiceClient gatewayClient,
WaitingParty party)
{
var assignRequest = new AssignDeploymentsRequest();
assignRequest.Assignments.Add(new Assignment
{
Result = Assignment.Types.Result.Requeued,
Party = party.Party
});
gatewayClient.AssignDeployments(assignRequest);
}
protected override void DoMatch(GatewayInternalService.GatewayInternalServiceClient gatewayClient,
DeploymentServiceClient deploymentServiceClient)
{
try
{
var resp = gatewayClient.PopWaitingParties(new PopWaitingPartiesRequest
{
Type = _tag,
NumParties = 1
});
Console.WriteLine($"Fetched {resp.Parties.Count} from gateway");
foreach (var party in resp.Parties)
{
Console.WriteLine("Attempting to match a retrieved party.");
var deployment = GetDeploymentWithTag(deploymentServiceClient, _tag);
if (deployment != null)
{
var assignRequest = new AssignDeploymentsRequest();
Console.WriteLine("Found a deployment, assigning it to the party.");
assignRequest.Assignments.Add(new Assignment
{
DeploymentId = deployment.Id,
DeploymentName = deployment.Name,
Result = Assignment.Types.Result.Matched,
Party = party.Party
});
MarkDeploymentAsInUse(deploymentServiceClient, deployment);
gatewayClient.AssignDeployments(assignRequest);
}
else
{
Console.WriteLine(
$"Unable to find a deployment with tag {_tag} in project {_project}");
Console.WriteLine("Requeueing the party");
AssignPartyAsRequeued(gatewayClient, party);
}
}
}
catch (RpcException e)
{
if (e.StatusCode != StatusCode.ResourceExhausted && e.StatusCode != StatusCode.Unavailable)
{
throw;
}
/* Unable to get the requested number of parties - ignore. */
Console.WriteLine("No parties available.");
Thread.Sleep(TickMs);
}
}
public void ContinueWithoutAssignmentIfJoinRequestHasBeenDeleted()
{
_memoryStoreClient
.Setup(client => client.GetAsync <PlayerJoinRequest>(It.IsAny <string>()))
.ReturnsAsync((PlayerJoinRequest)null);
_memoryStoreClient
.Setup(client => client.GetAsync <PartyJoinRequest>(_partyMatched.Id))
.ReturnsAsync((PartyJoinRequest)null);
var updated = new List <Entry>();
var requeued = new List <QueuedEntry>();
var deleted = new List <Entry>();
_transaction.Setup(tx => tx.UpdateAll(It.IsAny <IEnumerable <Entry> >()))
.Callback <IEnumerable <Entry> >(reqs => updated.AddRange(reqs));
_transaction.Setup(tx => tx.EnqueueAll(It.IsAny <IEnumerable <QueuedEntry> >()))
.Callback <IEnumerable <QueuedEntry> >(reqs => requeued.AddRange(reqs));
_transaction.Setup(tx => tx.DeleteAll(It.IsAny <IEnumerable <Entry> >()))
.Callback <IEnumerable <Entry> >(reqs => deleted.AddRange(reqs));
var ctx = Util.CreateFakeCallContext();
var req = new AssignDeploymentsRequest
{
Assignments =
{
new Assignment
{
Party = ConvertToProto(_partyMatched),
Result = Assignment.Types.Result.Matched
}
}
};
var task = _service.AssignDeployments(req, ctx);
Assert.That(task.IsCompleted);
Assert.AreEqual(StatusCode.OK, ctx.Status.StatusCode);
Assert.IsEmpty(updated);
Assert.IsEmpty(deleted);
Assert.IsEmpty(requeued);
}
private AssignDeploymentsRequest ConstructAssignRequest(IEnumerable <WaitingParty> waitingParties,
string deployment)
{
var req = new AssignDeploymentsRequest();
foreach (var waitingParty in waitingParties)
{
var assignment = new Assignment
{
DeploymentId = deployment,
DeploymentName = $"test_deployment_{deployment}",
Result = Assignment.Types.Result.Matched,
Party = waitingParty.Party
};
if (deployment == "requeue" && !_requeued.Contains(waitingParty.Party.Id))
{
assignment.Result = Assignment.Types.Result.Requeued;
_requeued.Add(waitingParty.Party.Id);
}
req.Assignments.Add(assignment);
}
return(req);
}
public void CallAppropriateMemoryStoreMethods()
{
var startTime = (DateTime.UtcNow - DateTime.UnixEpoch).TotalMilliseconds;
var updated = new List <Entry>();
var requeued = new List <QueuedEntry>();
var deleted = new List <Entry>();
_memoryStoreClient
.Setup(client => client.GetAsync <PartyJoinRequest>(_partyMatched.Id)).ReturnsAsync(_matchedPartyJoinRequest);
_memoryStoreClient
.Setup(client => client.GetAsync <PartyJoinRequest>(_partyError.Id)).ReturnsAsync(_errorPartyJoinRequest);
_memoryStoreClient
.Setup(client => client.GetAsync <PartyJoinRequest>(_requeuePartyJoinRequest.Id))
.ReturnsAsync(_requeuePartyJoinRequest);
_memoryStoreClient
.Setup(client => client.GetAsync <PlayerJoinRequest>(It.IsAny <string>()))
.ReturnsAsync((string id) => new PlayerJoinRequest(id, "", "", null)
{
State = MatchState.Matching
});
_transaction.Setup(tx => tx.UpdateAll(It.IsAny <IEnumerable <Entry> >()))
.Callback <IEnumerable <Entry> >(reqs => updated.AddRange(reqs));
_transaction.Setup(tx => tx.EnqueueAll(It.IsAny <IEnumerable <QueuedEntry> >()))
.Callback <IEnumerable <QueuedEntry> >(reqs => requeued.AddRange(reqs));
_transaction.Setup(tx => tx.DeleteAll(It.IsAny <IEnumerable <Entry> >()))
.Callback <IEnumerable <Entry> >(reqs => deleted.AddRange(reqs));
var ctx = Util.CreateFakeCallContext();
var req = new AssignDeploymentsRequest
{
Assignments =
{
new Assignment
{
Party = ConvertToProto(_partyMatched),
Result = Assignment.Types.Result.Matched,
DeploymentId = "1",
DeploymentName = "deployment1"
},
new Assignment
{
Party = ConvertToProto(_partyError),
Result = Assignment.Types.Result.Error
},
new Assignment
{
Party = ConvertToProto(_partyRequeue),
Result = Assignment.Types.Result.Requeued
}
}
};
_service.AssignDeployments(req, ctx);
Assert.AreEqual(StatusCode.OK, ctx.Status.StatusCode);
_memoryStoreClient.Verify(client => client.GetAsync <PlayerJoinRequest>(It.IsAny <string>()), Times.Exactly(4));
// We expect 6 entities to be updates. First 5 should be PlayerJoinRequests. The last should be the
// PartyJoinRequest of the requeued party.
Assert.AreEqual(5, updated.Count);
var playerJoinRequests = new List <PlayerJoinRequest>();
for (var i = 0; i < 4; i++)
{
Assert.IsInstanceOf <PlayerJoinRequest>(updated[i]);
playerJoinRequests.Add((PlayerJoinRequest)updated[i]);
}
Assert.AreEqual(LeaderPartyMatched, playerJoinRequests[0].Id);
Assert.AreEqual("1", playerJoinRequests[0].DeploymentId);
Assert.AreEqual("deployment1", playerJoinRequests[0].DeploymentName);
Assert.AreEqual(MatchState.Matched, playerJoinRequests[0].State);
Assert.AreEqual(PlayerPartyMatched, playerJoinRequests[1].PlayerIdentity);
Assert.AreEqual("1", playerJoinRequests[1].DeploymentId);
Assert.AreEqual("deployment1", playerJoinRequests[1].DeploymentName);
Assert.AreEqual(MatchState.Matched, playerJoinRequests[1].State);
Assert.AreEqual(LeaderPartyError, playerJoinRequests[2].PlayerIdentity);
Assert.AreEqual(MatchState.Error, playerJoinRequests[2].State);
Assert.AreEqual(LeaderPartyRequeue, playerJoinRequests[3].PlayerIdentity);
Assert.AreEqual(MatchState.Requested, playerJoinRequests[3].State);
Assert.IsInstanceOf <PartyJoinRequest>(updated[4]);
var updatedPartyRequest = (PartyJoinRequest)updated[4];
Assert.AreEqual(_partyRequeue.Id, updatedPartyRequest.Id);
Assert.AreEqual("type-requeue", updatedPartyRequest.Type);
Assert.AreEqual("type-requeue", updatedPartyRequest.QueueName);
Assert.AreEqual("type-requeue", updatedPartyRequest.QueueName);
Assert.Greater((DateTime.UtcNow - DateTime.UnixEpoch).TotalMilliseconds, updatedPartyRequest.Score);
Assert.Greater(updatedPartyRequest.Score, startTime);
// We expect for the requeued PartyJoinRequest to be exactly the same as the updated one.
Assert.AreEqual(1, requeued.Count);
Assert.AreEqual(updatedPartyRequest, requeued[0]);
// We expect for 2 PartyJoinRequests to have been deleted, one per each party which has reached a final
// state.
Assert.AreEqual(2, deleted.Count);
Assert.AreEqual(_partyMatched.Id, deleted[0].Id);
Assert.AreEqual(_partyError.Id, deleted[1].Id);
_logger.Verify(logger => logger.Warning(It.IsAny <string>()), Times.Never);
}
protected override void DoMatch(GatewayInternalService.GatewayInternalServiceClient gatewayClient,
DeploymentServiceClient deploymentServiceClient)
{
try
{
var resp = gatewayClient.PopWaitingParties(new PopWaitingPartiesRequest
{
Type = _tag,
NumParties = 1
});
Console.WriteLine($"Fetched {resp.Parties.Count} from gateway");
foreach (var party in resp.Parties)
{
Console.WriteLine("Attempting to match a retrieved party.");
_analytics.Send("match", "party_matching", new Dictionary <string, string>
{
{ "partyId", party.Party.Id },
{ "queueType", _tag },
{ "partyPhase", party.Party.CurrentPhase.ToString() },
{ "matchRequestId", party.MatchRequestId }
}, party.Party.LeaderPlayerId);
foreach (var memberId in party.Party.MemberIds)
{
_analytics.Send("match", "player_matching", new Dictionary <string, string>
{
{ "partyId", party.Party.Id },
{ "queueType", _tag },
{ "playerJoinRequestState", "Matching" },
{ "matchRequestId", party.MatchRequestId }
}, memberId);
}
var deployment = GetDeploymentWithTag(deploymentServiceClient, _tag);
if (deployment != null)
{
var assignRequest = new AssignDeploymentsRequest();
Console.WriteLine("Found a deployment, assigning it to the party.");
assignRequest.Assignments.Add(new Assignment
{
DeploymentId = deployment.Id,
DeploymentName = deployment.Name,
Result = Assignment.Types.Result.Matched,
Party = party.Party
});
MarkDeploymentAsInUse(deploymentServiceClient, deployment);
gatewayClient.AssignDeployments(assignRequest);
_analytics.Send("deployment", "deployment_in_use", new Dictionary <string, string>
{
{ "spatialProjectId", _project },
{ "deploymentName", deployment.Name },
{ "deploymentId", deployment.Id }
});
}
else
{
Console.WriteLine(
$"Unable to find a deployment with tag {_tag} in project {_project}");
Console.WriteLine("Requeueing the party");
AssignPartyAsRequeued(gatewayClient, party);
}
}
}
catch (RpcException e)
{
if (e.StatusCode != StatusCode.ResourceExhausted && e.StatusCode != StatusCode.Unavailable)
{
throw;
}
/* Unable to get the requested number of parties - ignore. */
Console.WriteLine("No parties available.");
Thread.Sleep(TickMs);
}
}
public override async Task <AssignDeploymentsResponse> AssignDeployments(AssignDeploymentsRequest request,
ServerCallContext context)
{
try
{
using (var memClient = _matchmakingMemoryStoreClientManager.GetClient())
{
var toUpdate = new List <Entry>();
foreach (var assignment in request.Assignments)
{
Reporter.AssignDeploymentInc(assignment.DeploymentId, assignment.Result);
foreach (var memberId in assignment.Party.MemberIds)
{
var playerJoinRequest = await memClient.GetAsync <PlayerJoinRequest>(memberId);
if (playerJoinRequest == null)
{
continue;
}
switch (assignment.Result)
{
case Assignment.Types.Result.Error:
playerJoinRequest.State = MatchState.Error;
break;
case Assignment.Types.Result.Matched:
playerJoinRequest.AssignMatch(assignment.DeploymentId, assignment.DeploymentName);
break;
case Assignment.Types.Result.Requeued:
playerJoinRequest.State = MatchState.Requested;
break;
}
toUpdate.Add(playerJoinRequest);
}
}
var toRequeue = new List <PartyJoinRequest>();
var toDelete = new List <PartyJoinRequest>();
foreach (var assignment in request.Assignments)
{
var party = assignment.Party;
var partyJoinRequest = await memClient.GetAsync <PartyJoinRequest>(party.Id);
if (partyJoinRequest == null)
{
// Party join request has been cancelled.
continue;
}
if (assignment.Result == Assignment.Types.Result.Requeued)
{
partyJoinRequest.RefreshQueueData();
toRequeue.Add(partyJoinRequest);
toUpdate.Add(partyJoinRequest);
}
else
{
// If the matchmaking process for this party has reached a final state, we should delete the
// PartyJoinRequest associated to it.
toDelete.Add(partyJoinRequest);
}
}
using (var tx = memClient.CreateTransaction())
{
tx.UpdateAll(toUpdate);
tx.EnqueueAll(toRequeue);
tx.DeleteAll(toDelete);
}
}
}
catch (EntryNotFoundException e)
{
Reporter.AssignDeploymentNotFoundInc(e.Id);
Log.Warning($"Attempted to assign deployment to nonexistent join request {e.Id}.");
throw new RpcException(new Status(StatusCode.NotFound, "Join request does not exist"));
}
catch (TransactionAbortedException)
{
Reporter.TransactionAbortedInc("AssignDeployments");
Log.Warning("Transaction aborted during deployment assignment.");
throw new RpcException(new Status(StatusCode.Unavailable,
"assignment aborted due to concurrent modification; safe to retry"));
}
return(new AssignDeploymentsResponse());
}
public override async Task <AssignDeploymentsResponse> AssignDeployments(AssignDeploymentsRequest request,
ServerCallContext context)
{
try
{
using (var memClient = _matchmakingMemoryStoreClientManager.GetClient())
{
var toUpdate = new List <Entry>();
foreach (var assignment in request.Assignments)
{
Reporter.AssignDeploymentInc(assignment.DeploymentId, assignment.Result);
foreach (var memberId in assignment.Party.MemberIds)
{
var playerJoinRequest = await memClient.GetAsync <PlayerJoinRequest>(memberId);
if (playerJoinRequest == null)
{
continue;
}
switch (assignment.Result)
{
case Assignment.Types.Result.Error:
playerJoinRequest.State = MatchState.Error;
break;
case Assignment.Types.Result.Matched:
playerJoinRequest.AssignMatch(assignment.DeploymentId, assignment.DeploymentName);
break;
case Assignment.Types.Result.Requeued:
playerJoinRequest.State = MatchState.Requested;
break;
}
toUpdate.Add(playerJoinRequest);
}
}
var toRequeue = new List <PartyJoinRequest>();
var toDelete = new List <PartyJoinRequest>();
foreach (var assignment in request.Assignments)
{
var party = assignment.Party;
var partyJoinRequest = await memClient.GetAsync <PartyJoinRequest>(party.Id);
if (partyJoinRequest == null)
{
// Party join request has been cancelled.
continue;
}
var eventAttributes = new Dictionary <string, string>
{
{ "partyId", partyJoinRequest.Id },
{ "matchRequestId", partyJoinRequest.MatchRequestId },
{ "queueType", partyJoinRequest.Type },
{ "partyPhase", partyJoinRequest.Party.CurrentPhase.ToString() }
};
if (assignment.Result == Assignment.Types.Result.Matched)
{
toDelete.Add(partyJoinRequest);
eventAttributes.Add("spatialProjectId", _project);
eventAttributes.Add("deploymentName", assignment.DeploymentName);
eventAttributes.Add("deploymentId", assignment.DeploymentId);
_analytics.Send("party_matched", eventAttributes, partyJoinRequest.Party.LeaderPlayerId);
}
else if (assignment.Result == Assignment.Types.Result.Requeued)
{
partyJoinRequest.RefreshQueueData();
toRequeue.Add(partyJoinRequest);
toUpdate.Add(partyJoinRequest);
_analytics.Send("party_requeued", eventAttributes, partyJoinRequest.Party.LeaderPlayerId);
}
else if (assignment.Result == Assignment.Types.Result.Error)
{
toDelete.Add(partyJoinRequest);
_analytics.Send("party_error", eventAttributes, partyJoinRequest.Party.LeaderPlayerId);
}
else
{
toDelete.Add(partyJoinRequest);
}
}
using (var tx = memClient.CreateTransaction())
{
tx.UpdateAll(toUpdate);
tx.EnqueueAll(toRequeue);
tx.DeleteAll(toDelete);
}
foreach (var playerJoinRequest in toUpdate.OfType <PlayerJoinRequest>())
{
var eventAttributes = new Dictionary <string, string>
{
{ "partyId", playerJoinRequest.PartyId },
{ "matchRequestId", playerJoinRequest.MatchRequestId },
{ "queueType", playerJoinRequest.Type },
{ "playerJoinRequestState", playerJoinRequest.State.ToString() }
};
switch (playerJoinRequest.State)
{
case MatchState.Matched:
eventAttributes.Add("spatialProjectId", _project);
eventAttributes.Add("deploymentName", playerJoinRequest.DeploymentName);
eventAttributes.Add("deploymentId", playerJoinRequest.DeploymentId);
_analytics.Send("player_matched", eventAttributes, playerJoinRequest.Id);
break;
case MatchState.Requested:
_analytics.Send("player_requeued", eventAttributes, playerJoinRequest.Id);
break;
case MatchState.Error:
_analytics.Send("player_error", eventAttributes, playerJoinRequest.Id);
break;
}
}
}
}
catch (EntryNotFoundException e)
{
Reporter.AssignDeploymentNotFoundInc(e.Id);
Log.Warning($"Attempted to assign deployment to nonexistent join request {e.Id}.");
throw new RpcException(new Status(StatusCode.NotFound, "Join request does not exist"));
}
catch (TransactionAbortedException)
{
Reporter.TransactionAbortedInc("AssignDeployments");
Log.Warning("Transaction aborted during deployment assignment.");
throw new RpcException(new Status(StatusCode.Unavailable,
"assignment aborted due to concurrent modification; safe to retry"));
}
return(new AssignDeploymentsResponse());
}
protected override void DoMatch(GatewayInternalService.GatewayInternalServiceClient gatewayClient,
DeploymentServiceClient deploymentServiceClient)
{
try
{
var resp = gatewayClient.PopWaitingParties(new PopWaitingPartiesRequest
{
Type = _tag,
NumParties = 1
});
Console.WriteLine($"Fetched {resp.Parties.Count} from gateway");
foreach (var party in resp.Parties)
{
Console.WriteLine($"Attempting to match a retrieved party of {party.Party.MemberIds.Count} players.");
bool requeue = false;
var deployment = GetDeploymentWithTag(deploymentServiceClient, _tag);
if (deployment != null)
{
var assignRequest = new AssignDeploymentsRequest();
Console.WriteLine("Found a deployment.");
int numPlayers = 0;
foreach (string tag in deployment.Tag)
{
if (tag.StartsWith(PlayersTag))
{
int.TryParse(tag.Split("_")[1], out numPlayers);
Console.WriteLine($"Deployment already has {numPlayers} in.");
break;
}
}
int totalPlayers = numPlayers + party.Party.MemberIds.Count;
if (totalPlayers <= PlayersPerDeployment)
{
if (numPlayers > 0)
{
Console.WriteLine($"Removing tag {PlayersTag}_{numPlayers}");
deployment.Tag.Remove($"{PlayersTag}_{numPlayers}");
}
if (totalPlayers == PlayersPerDeployment)
{
Console.WriteLine("Deployment is full, marking as in_use");
MarkDeploymentAsInUse(deploymentServiceClient, deployment);
}
else
{
Console.WriteLine($"Deployment isn't full, adding tag {PlayersTag}_{totalPlayers}");
deployment.Tag.Add($"{PlayersTag}_{totalPlayers}");
var req = new UpdateDeploymentRequest {
Deployment = deployment
};
deploymentServiceClient.UpdateDeployment(req);
}
assignRequest.Assignments.Add(new Assignment
{
DeploymentId = deployment.Id,
DeploymentName = deployment.Name,
Result = Assignment.Types.Result.Matched,
Party = party.Party
});
gatewayClient.AssignDeployments(assignRequest);
}
else
{
Console.WriteLine("Deployment does not have space for a party of this size.");
requeue = true;
}
}
else
{
Console.WriteLine($"Unable to find a deployment with tag {_tag} in project {_project}");
requeue = true;
}
if (requeue)
{
Console.WriteLine("Requeueing the party");
AssignPartyAsRequeued(gatewayClient, party);
}
}
}
catch (RpcException e)
{
if (e.StatusCode != StatusCode.ResourceExhausted && e.StatusCode != StatusCode.Unavailable)
{
throw;
}
/* Unable to get the requested number of parties - ignore. */
Thread.Sleep(TickMs);
}
}
