public static SnapshotServiceClient CreateSnapshotClient(Options.Common options)
{
return(string.IsNullOrEmpty(options.Environment)
? SnapshotServiceClient.Create()
: SnapshotServiceClient.Create(GetEndpoint(options.Environment),
GetTokenCredential(options.Environment)));
}
static void Main(string[] args)
{
Log.Logger = new LoggerConfiguration()
.WriteTo.Console(new RenderedCompactJsonFormatter())
.Enrich.FromLogContext()
.CreateLogger();
// See https://support.microsoft.com/en-gb/help/821268/contention-poor-performance-and-deadlocks-when-you-make-calls-to-web-s
ThreadPool.SetMaxThreads(100, 100);
ThreadPool.SetMinThreads(50, 50);
Parser.Default.ParseArguments <DeploymentPoolArgs>(args)
.WithParsed(parsedArgs =>
{
parsedArgs.Validate();
var spatialRefreshToken = Environment.GetEnvironmentVariable(SpatialRefreshTokenEnvironmentVariable) ??
throw new Exception(
$"{SpatialRefreshTokenEnvironmentVariable} environment variable is required.");
if (spatialRefreshToken == "")
{
throw new ArgumentException("Refresh token should not be empty");
}
IAnalyticsSender analyticsSender = new AnalyticsSenderBuilder("deployment_pool")
.WithCommandLineArgs(parsedArgs)
.With(new LogExceptionStrategy(Log.Logger))
.Build();
var spatialDeploymentClient =
DeploymentServiceClient.Create(credentials: new PlatformRefreshTokenCredential(spatialRefreshToken));
var spatialSnapshotClient =
SnapshotServiceClient.Create(credentials: new PlatformRefreshTokenCredential(spatialRefreshToken));
var platformInvoker = new PlatformInvoker(parsedArgs, spatialDeploymentClient, spatialSnapshotClient, analyticsSender);
var cancelTokenSource = new CancellationTokenSource();
var cancelToken = cancelTokenSource.Token;
var metricsServer = new MetricServer(parsedArgs.MetricsPort).Start();
var dplPool = new DeploymentPool(parsedArgs, spatialDeploymentClient, platformInvoker, cancelToken, analyticsSender);
var dplPoolTask = Task.Run(() => dplPool.Start());
var unixSignalTask = Task.Run(() => UnixSignal.WaitAny(new[] { new UnixSignal(Signum.SIGINT), new UnixSignal(Signum.SIGTERM) }));
Task.WaitAny(dplPoolTask, unixSignalTask);
if (unixSignalTask.IsCompleted)
{
Log.Information($"Received UNIX signal {unixSignalTask.Result}");
Log.Information("Server shutting down...");
cancelTokenSource.Cancel();
metricsServer.StopAsync();
dplPoolTask.Wait();
Log.Information("Server stopped cleanly");
}
else
{
/* The server task has completed; we can just exit. */
Log.Information($"The deployment pool has stopped itself or encountered an unhandled exception {dplPoolTask.Exception}");
}
});
}
public static void Init(string serviceAccountToken)
{
var credentials = new PlatformRefreshTokenCredential(serviceAccountToken);
deploymentServiceClient = DeploymentServiceClient.Create(credentials: credentials);
snapshotServiceClient = SnapshotServiceClient.Create(credentials: credentials);
}
private static string UploadSnapshot(SnapshotServiceClient client, string snapshotPath, string projectName,
string deploymentName)
{
Console.WriteLine($"Uploading {snapshotPath} to project {projectName}");
// Read snapshot.
var bytes = File.ReadAllBytes(snapshotPath);
if (bytes.Length == 0)
{
Console.Error.WriteLine($"Unable to load {snapshotPath}. Does the file exist?");
return(string.Empty);
}
// Create HTTP endpoint to upload to.
var snapshotToUpload = new Snapshot
{
ProjectName = projectName,
DeploymentName = deploymentName
};
using (var md5 = MD5.Create())
{
snapshotToUpload.Checksum = Convert.ToBase64String(md5.ComputeHash(bytes));
snapshotToUpload.Size = bytes.Length;
}
var uploadSnapshotResponse =
client.UploadSnapshot(new UploadSnapshotRequest {
Snapshot = snapshotToUpload
});
snapshotToUpload = uploadSnapshotResponse.Snapshot;
// Upload content.
var httpRequest = WebRequest.Create(uploadSnapshotResponse.UploadUrl) as HttpWebRequest;
httpRequest.Method = "PUT";
httpRequest.ContentLength = snapshotToUpload.Size;
httpRequest.Headers.Set("Content-MD5", snapshotToUpload.Checksum);
using (var dataStream = httpRequest.GetRequestStream())
{
dataStream.Write(bytes, 0, bytes.Length);
}
// Block until we have a response.
httpRequest.GetResponse();
// Confirm that the snapshot was uploaded successfully.
var confirmUploadResponse = client.ConfirmUpload(new ConfirmUploadRequest
{
DeploymentName = snapshotToUpload.DeploymentName,
Id = snapshotToUpload.Id,
ProjectName = snapshotToUpload.ProjectName
});
return(confirmUploadResponse.Snapshot.Id);
}
private static Operation <Deployment, CreateDeploymentMetadata> CreateMainDeploymentAsync(DeploymentServiceClient deploymentServiceClient,
bool launchSimPlayerDeployment, string projectName, string assemblyName, string runtimeVersion, string mainDeploymentName, string mainDeploymentJsonPath, string mainDeploymentSnapshotPath, string regionCode, string clusterCode, string deploymentTags)
{
var snapshotServiceClient = SnapshotServiceClient.Create(GetApiEndpoint(regionCode), GetPlatformRefreshTokenCredential(regionCode));
// Upload snapshots.
var mainSnapshotId = UploadSnapshot(snapshotServiceClient, mainDeploymentSnapshotPath, projectName,
mainDeploymentName, regionCode);
if (mainSnapshotId.Length == 0)
{
throw new Exception("Error while uploading snapshot.");
}
// Create main deployment.
var mainDeploymentConfig = new Deployment
{
AssemblyId = assemblyName,
LaunchConfig = new LaunchConfig
{
ConfigJson = File.ReadAllText(mainDeploymentJsonPath)
},
Name = mainDeploymentName,
ProjectName = projectName,
StartingSnapshotId = mainSnapshotId,
RegionCode = regionCode,
ClusterCode = clusterCode,
RuntimeVersion = runtimeVersion
};
mainDeploymentConfig.Tag.Add(DEPLOYMENT_LAUNCHED_BY_LAUNCHER_TAG);
foreach (String tag in deploymentTags.Split(' '))
{
if (tag.Length > 0)
{
mainDeploymentConfig.Tag.Add(tag);
}
}
if (launchSimPlayerDeployment)
{
// This tag needs to be added to allow simulated players to connect using login
// tokens generated with anonymous auth.
mainDeploymentConfig.Tag.Add("dev_login");
}
Console.WriteLine(
$"Creating the main deployment {mainDeploymentName} in project {projectName} with snapshot ID {mainSnapshotId}. Link: https://console.improbable.io/projects/{projectName}/deployments/{mainDeploymentName}/overview");
var mainDeploymentCreateOp = deploymentServiceClient.CreateDeployment(new CreateDeploymentRequest
{
Deployment = mainDeploymentConfig
});
return(mainDeploymentCreateOp);
}
public PlatformInvoker(DeploymentPoolArgs args,
DeploymentServiceClient deploymentServiceClient,
SnapshotServiceClient snapshotServiceClient)
{
deploymentNamePrefix = args.DeploymentNamePrefix + HumanNamer.GetRandomName(2, "_") + "_";
launchConfigFilePath = args.LaunchConfigFilePath;
snapshotFilePath = args.SnapshotFilePath;
assemblyName = args.AssemblyName;
spatialProject = args.SpatialProject;
matchType = args.MatchType;
this.deploymentServiceClient = deploymentServiceClient;
this.snapshotServiceClient = snapshotServiceClient;
}
public PlatformInvoker(DeploymentPoolArgs args,
DeploymentServiceClient deploymentServiceClient,
SnapshotServiceClient snapshotServiceClient,
IAnalyticsSender analytics = null)
{
deploymentNamePrefix = args.DeploymentNamePrefix + HumanNamer.GetRandomName(2, "_") + "_";
launchConfigFilePath = args.LaunchConfigFilePath;
snapshotFilePath = args.SnapshotFilePath;
assemblyName = args.AssemblyName;
spatialProject = args.SpatialProject;
matchType = args.MatchType;
this.deploymentServiceClient = deploymentServiceClient;
this.snapshotServiceClient = snapshotServiceClient;
_analytics = (analytics ?? new NullAnalyticsSender()).WithEventClass("deployment");
}
public async Task <object> TakeDeploymentSnapshot(dynamic data)
{
return(await Task.Run(() => {
PlatformRefreshTokenCredential CredentialWithProvidedToken = new PlatformRefreshTokenCredential(data._RefreshToken);
SnapshotServiceClient _snapshotServiceClient = SnapshotServiceClient.Create(credentials: CredentialWithProvidedToken);
var latestSnapshot = _snapshotServiceClient.TakeSnapshot(new TakeSnapshotRequest
{
Snapshot = new Snapshot
{
ProjectName = data._ProjectName,
DeploymentName = data._DeploymentName
}
}).PollUntilCompleted()
.GetResultOrNull();
return latestSnapshot.Id;
}));
}
private static string UploadSnapshot(SnapshotServiceClient client, Options.Create options)
{
if (!File.Exists(options.SnapshotPath))
{
Ipc.WriteError(Ipc.ErrorCode.NotFound, $"Could not find snapshot file at: {options.SnapshotPath}");
return(null);
}
// Read snapshot.
var bytes = File.ReadAllBytes(options.SnapshotPath);
if (bytes.Length == 0)
{
Ipc.WriteError(Ipc.ErrorCode.Unknown, $"Snapshot file at {options.SnapshotPath} has zero bytes.");
return(null);
}
// Create HTTP endpoint to upload to.
var snapshotToUpload = new Snapshot
{
ProjectName = options.ProjectName,
DeploymentName = options.DeploymentName
};
using (var md5 = MD5.Create())
{
snapshotToUpload.Checksum = Convert.ToBase64String(md5.ComputeHash(bytes));
snapshotToUpload.Size = bytes.Length;
}
var uploadSnapshotResponse =
client.UploadSnapshot(new UploadSnapshotRequest {
Snapshot = snapshotToUpload
});
snapshotToUpload = uploadSnapshotResponse.Snapshot;
using (var httpClient = new HttpClient())
{
try
{
var content = new ByteArrayContent(bytes);
content.Headers.Add("Content-MD5", snapshotToUpload.Checksum);
if (options.Environment == "cn-production")
{
content.Headers.Add("x-amz-server-side-encryption", "AES256");
}
using (var response = httpClient.PutAsync(uploadSnapshotResponse.UploadUrl, content).Result)
{
if (response.StatusCode != HttpStatusCode.OK)
{
Ipc.WriteError(Ipc.ErrorCode.SnapshotUploadFailed, $"Snapshot upload returned non-OK error code: {response.StatusCode}");
return(null);
}
}
}
catch (HttpRequestException e)
{
Ipc.WriteError(Ipc.ErrorCode.SnapshotUploadFailed, $"Failed to upload snapshot with following exception: {e.Message}");
return(null);
}
}
// Confirm that the snapshot was uploaded successfully.
var confirmUploadResponse = client.ConfirmUpload(new ConfirmUploadRequest
{
DeploymentName = snapshotToUpload.DeploymentName,
Id = snapshotToUpload.Id,
ProjectName = snapshotToUpload.ProjectName
});
return(confirmUploadResponse.Snapshot.Id);
}
private static Operation <Deployment, CreateDeploymentMetadata> CreateSimPlayerDeploymentAsync(DeploymentServiceClient deploymentServiceClient,
string projectName, string assemblyName, string runtimeVersion, string mainDeploymentName, string simDeploymentName,
string simDeploymentJsonPath, string simDeploymentSnapshotPath, string regionCode, string clusterCode, int numSimPlayers, bool useChinaPlatform)
{
var snapshotServiceClient = SnapshotServiceClient.Create(GetApiEndpoint(useChinaPlatform), GetPlatformRefreshTokenCredential(useChinaPlatform));
// Upload snapshots.
var simDeploymentSnapshotId = UploadSnapshot(snapshotServiceClient, simDeploymentSnapshotPath, projectName,
simDeploymentName, useChinaPlatform);
if (simDeploymentSnapshotId.Length == 0)
{
throw new Exception("Error while uploading sim player snapshot.");
}
var playerAuthServiceClient = PlayerAuthServiceClient.Create(GetApiEndpoint(useChinaPlatform), GetPlatformRefreshTokenCredential(useChinaPlatform));
// Create development authentication token used by the simulated players.
var dat = playerAuthServiceClient.CreateDevelopmentAuthenticationToken(
new CreateDevelopmentAuthenticationTokenRequest
{
Description = "DAT for simulated player deployment.",
Lifetime = Duration.FromTimeSpan(new TimeSpan(7, 0, 0, 0)),
ProjectName = projectName
});
// Add worker flags to sim deployment JSON.
var devAuthTokenFlag = new JObject();
devAuthTokenFlag.Add("name", "simulated_players_dev_auth_token");
devAuthTokenFlag.Add("value", dat.TokenSecret);
var targetDeploymentFlag = new JObject();
targetDeploymentFlag.Add("name", "simulated_players_target_deployment");
targetDeploymentFlag.Add("value", mainDeploymentName);
var numSimulatedPlayersFlag = new JObject();
numSimulatedPlayersFlag.Add("name", "total_num_simulated_players");
numSimulatedPlayersFlag.Add("value", $"{numSimPlayers}");
var simDeploymentConfigJson = File.ReadAllText(simDeploymentJsonPath);
dynamic simDeploymentConfig = JObject.Parse(simDeploymentConfigJson);
if (simDeploymentJsonPath.EndsWith(".pb.json"))
{
for (var i = 0; i < simDeploymentConfig.worker_flagz.Count; ++i)
{
if (simDeploymentConfig.worker_flagz[i].worker_type == CoordinatorWorkerName)
{
simDeploymentConfig.worker_flagz[i].flagz.Add(devAuthTokenFlag);
simDeploymentConfig.worker_flagz[i].flagz.Add(targetDeploymentFlag);
simDeploymentConfig.worker_flagz[i].flagz.Add(numSimulatedPlayersFlag);
break;
}
}
for (var i = 0; i < simDeploymentConfig.flagz.Count; ++i)
{
if (simDeploymentConfig.flagz[i].name == "loadbalancer_v2_config_json")
{
string layerConfigJson = simDeploymentConfig.flagz[i].value;
dynamic loadBalanceConfig = JObject.Parse(layerConfigJson);
var lbLayerConfigurations = loadBalanceConfig.layerConfigurations;
for (var j = 0; j < lbLayerConfigurations.Count; ++j)
{
if (lbLayerConfigurations[j].layer == CoordinatorWorkerName)
{
var rectangleGrid = lbLayerConfigurations[j].rectangleGrid;
rectangleGrid.cols = numSimPlayers;
rectangleGrid.rows = 1;
break;
}
}
simDeploymentConfig.flagz[i].value = Newtonsoft.Json.JsonConvert.SerializeObject(loadBalanceConfig);
break;
}
}
}
else // regular non pb.json
{
for (var i = 0; i < simDeploymentConfig.workers.Count; ++i)
{
if (simDeploymentConfig.workers[i].worker_type == CoordinatorWorkerName)
{
simDeploymentConfig.workers[i].flags.Add(devAuthTokenFlag);
simDeploymentConfig.workers[i].flags.Add(targetDeploymentFlag);
simDeploymentConfig.workers[i].flags.Add(numSimulatedPlayersFlag);
}
}
// Specify the number of managed coordinator workers to start by editing
// the load balancing options in the launch config. It creates a rectangular
// launch config of N cols X 1 row, N being the number of coordinators
// to create.
// This assumes the launch config contains a rectangular load balancing
// layer configuration already for the coordinator worker.
var lbLayerConfigurations = simDeploymentConfig.load_balancing.layer_configurations;
for (var i = 0; i < lbLayerConfigurations.Count; ++i)
{
if (lbLayerConfigurations[i].layer == CoordinatorWorkerName)
{
var rectangleGrid = lbLayerConfigurations[i].rectangle_grid;
rectangleGrid.cols = numSimPlayers;
rectangleGrid.rows = 1;
}
}
}
// Create simulated player deployment.
var simDeployment = new Deployment
{
AssemblyId = assemblyName,
LaunchConfig = new LaunchConfig
{
ConfigJson = simDeploymentConfig.ToString()
},
Name = simDeploymentName,
ProjectName = projectName,
RuntimeVersion = runtimeVersion,
StartingSnapshotId = simDeploymentSnapshotId,
};
if (!String.IsNullOrEmpty(clusterCode))
{
simDeployment.ClusterCode = clusterCode;
}
else
{
simDeployment.RegionCode = regionCode;
}
simDeployment.Tag.Add(DEPLOYMENT_LAUNCHED_BY_LAUNCHER_TAG);
simDeployment.Tag.Add(SIM_PLAYER_DEPLOYMENT_TAG);
Console.WriteLine(
$"Creating the simulated player deployment {simDeploymentName} in project {projectName} with {numSimPlayers} simulated players. Link: https://{GetConsoleHost(useChinaPlatform)}/projects/{projectName}/deployments/{simDeploymentName}/overview");
var simDeploymentCreateOp = deploymentServiceClient.CreateDeployment(new CreateDeploymentRequest
{
Deployment = simDeployment
});
return(simDeploymentCreateOp);
}
private static int CreateDeploymentInternal <TOptions>(TOptions options, Func <TOptions, string> getLaunchConfigJson)
where TOptions : Options.Create
{
var snapshotServiceClient = SnapshotServiceClient.Create();
var deploymentServiceClient = DeploymentServiceClient.Create();
try
{
var deployment = new Deployment
{
AssemblyId = options.AssemblyName,
LaunchConfig = new LaunchConfig
{
ConfigJson = getLaunchConfigJson(options)
},
Name = options.DeploymentName,
ProjectName = options.ProjectName,
RegionCode = options.Region.ToString()
};
if (options.SnapshotPath != null)
{
var snapshotId = UploadSnapshot(snapshotServiceClient, options.SnapshotPath, options.ProjectName,
options.DeploymentName);
if (string.IsNullOrEmpty(snapshotId))
{
return(Program.ErrorExitCode);
}
deployment.StartingSnapshotId = snapshotId;
}
if (options.Tags != null)
{
foreach (var tag in options.Tags)
{
deployment.Tag.Add(tag);
}
}
var deploymentOp = deploymentServiceClient.CreateDeployment(new CreateDeploymentRequest
{
Deployment = deployment
}).PollUntilCompleted();
if (deploymentOp.Result.Status != Deployment.Types.Status.Running)
{
Ipc.WriteError(Ipc.ErrorCode.Unknown, "Deployment failed to start for an unknown reason.");
return(Program.ErrorExitCode);
}
}
catch (Grpc.Core.RpcException e)
{
if (e.Status.StatusCode == Grpc.Core.StatusCode.NotFound)
{
Ipc.WriteError(Ipc.ErrorCode.NotFound, e.Status.Detail);
return(Program.ErrorExitCode);
}
throw;
}
return(Program.SuccessExitCode);
}
private static int CreateDeployment(string[] args)
{
bool launchSimPlayerDeployment = args.Length == 9;
var projectName = args[1];
var assemblyName = args[2];
var mainDeploymentName = args[3];
var mainDeploymentJson = args[4];
var mainDeploymentSnapshotFilePath = args[5];
var deploymentRegionCode = args[6];
var simDeploymentName = string.Empty;
var simDeploymentJson = string.Empty;
if (launchSimPlayerDeployment)
{
simDeploymentName = args[7];
simDeploymentJson = args[8];
}
// Create service clients.
var playerAuthServiceClient = PlayerAuthServiceClient.Create();
var snapshotServiceClient = SnapshotServiceClient.Create();
var deploymentServiceClient = DeploymentServiceClient.Create();
try
{
// Upload snapshots.
var mainSnapshotId = UploadSnapshot(snapshotServiceClient, mainDeploymentSnapshotFilePath, projectName,
mainDeploymentName);
if (mainSnapshotId.Length == 0)
{
return(1);
}
// Create main deployment.
var mainDeploymentConfig = new Deployment
{
AssemblyId = assemblyName,
LaunchConfig = new LaunchConfig
{
ConfigJson = File.ReadAllText(mainDeploymentJson)
},
Name = mainDeploymentName,
ProjectName = projectName,
StartingSnapshotId = mainSnapshotId,
RegionCode = deploymentRegionCode
};
if (launchSimPlayerDeployment)
{
// This tag needs to be added to allow simulated clients to connect using login
// tokens generated with anonymous auth.
mainDeploymentConfig.Tag.Add("dev_login");
}
Console.WriteLine(
$"Creating the main deployment {mainDeploymentName} in project {projectName} with snapshot ID {mainSnapshotId}.");
var mainDeploymentCreateOp = deploymentServiceClient.CreateDeployment(new CreateDeploymentRequest
{
Deployment = mainDeploymentConfig
}).PollUntilCompleted();
Console.WriteLine("Successfully created the main deployment.");
if (launchSimPlayerDeployment)
{
// Create development authentication token used by the simulated players.
var dat = playerAuthServiceClient.CreateDevelopmentAuthenticationToken(
new CreateDevelopmentAuthenticationTokenRequest
{
Description = "DAT for sim worker deployment.",
Lifetime = Duration.FromTimeSpan(new TimeSpan(7, 0, 0, 0)),
ProjectName = projectName
});
// Add worker flags to sim deployment JSON.
var devAuthTokenIdFlag = new JObject();
devAuthTokenIdFlag.Add("name", "fps_simulated_players_dev_auth_token_id");
devAuthTokenIdFlag.Add("value", dat.DevelopmentAuthenticationToken.Id);
var targetDeploymentFlag = new JObject();
targetDeploymentFlag.Add("name", "fps_simulated_players_target_deployment");
targetDeploymentFlag.Add("value", mainDeploymentName);
var simWorkerConfigJson = File.ReadAllText(simDeploymentJson);
dynamic simWorkerConfig = JObject.Parse(simWorkerConfigJson);
for (var i = 0; i < simWorkerConfig.workers.Count; ++i)
{
if (simWorkerConfig.workers[i].worker_type == "SimulatedPlayerCoordinator")
{
simWorkerConfig.workers[i].flags.Add(devAuthTokenIdFlag);
simWorkerConfig.workers[i].flags.Add(targetDeploymentFlag);
}
}
simWorkerConfigJson = simWorkerConfig.ToString();
// Create simulated player deployment.
var simDeploymentConfig = new Deployment
{
AssemblyId = assemblyName,
LaunchConfig = new LaunchConfig
{
ConfigJson = simWorkerConfigJson
},
Name = simDeploymentName,
ProjectName = projectName,
RegionCode = deploymentRegionCode
};
simDeploymentConfig.Tag.Add("simulated_clients");
Console.WriteLine($"Creating the simulated player deployment {simDeploymentName} in project {projectName}.");
var simDeploymentCreateOp = deploymentServiceClient.CreateDeployment(new CreateDeploymentRequest
{
Deployment = simDeploymentConfig
}).PollUntilCompleted();
Console.WriteLine("Successfully created the simulated player deployment.");
}
}
catch (Grpc.Core.RpcException e)
{
if (e.Status.StatusCode == Grpc.Core.StatusCode.NotFound)
{
Console.WriteLine(
$"Unable to launch the deployment(s). This is likely because the project '{projectName}' or assembly '{assemblyName}' doesn't exist.");
}
else
{
throw;
}
}
return(0);
}
protected static SnapshotServiceClient GetSnapshotServiceClient(string host, int port)
{
return(SnapshotServiceClient.Create(new PlatformApiEndpoint(host, port, true)));
}
