//todo: should we add a AddIdSyncComponents method?
private async Task SyncComponentsToGraphReplicaSet()
{
var commands = MoreEnumerable
.TraverseBreadthFirst((IGraphMergeContext)_graphMergeContext !, ctx => ctx !.ChildContexts)
.SelectMany(ctx =>
{
var nodeCommands = new List <ICommand>();
if (ctx.ReplaceRelationshipsCommand.Relationships.Any())
{
nodeCommands.Add(ctx.ReplaceRelationshipsCommand);
}
if (ctx.ExtraCommands.Any())
{
nodeCommands.AddRange(ctx.ExtraCommands);
}
if (!ctx.SyncNameProvider.GraphSyncPartSettings.PreexistingNode)
{
nodeCommands.Add(ctx.MergeNodeCommand);
}
return(nodeCommands);
})
.Reverse()
.ToArray();
await _graphMergeContext !.GraphReplicaSet.Run(commands);
}
public async Task Run_Command_LimitedToOneInstance_CommandIsRunOnCorrectInstance()
{
const int limitedToInstance = 2;
GraphReplicaSet = new GraphReplicaSet(ReplicaSetName, GraphInstances, Logger, limitedToInstance);
await GraphReplicaSet.Run(Command);
int index = 0;
foreach (var graph in GraphInstances)
{
var calls = Fake.GetCalls(graph).ToList();
TestOutputHelper.WriteLine($"Graph #{index}: Run() called x{calls.Count}.");
++index;
}
index = 0;
foreach (var graph in GraphInstances)
{
A.CallTo(() => graph.Run(A <ICommand> ._))
.MustHaveHappened(index == limitedToInstance ? 1 : 0, Times.Exactly);
++index;
}
}
public async Task <List <GraphPredictedYearModel>?> GetPredictedYearAsync(GraphReplicaSet graphReplicaSet, int soc)
{
logger.LogInformation($"Fetching Soc ({soc}) {graphPredictedYearNodeName} from {graphReplicaSet}");
var query = $"MATCH({graphSocDatasetNodeName} {{Soc: {soc}}}) -[{graphJobGrowthRelationshipName}]-> ({graphPredictedNodeName}) -[{graphPredictedEmploymentRelationshipName}]-> (n:{graphPredictedYearNodeName}) RETURN n";
var models = await genericGraphQueryService.ExecuteCypherQuery <GraphPredictedYearModel>(graphReplicaSet, query).ConfigureAwait(false);
return(models);
}
public async Task <List <GraphJobProfileModel>?> GetJobProfilesAsync(GraphReplicaSet graphReplicaSet, int soc)
{
logger.LogInformation($"Fetching Soc ({soc}) {graphJobProfileNodeName} from {graphReplicaSet}");
var query = $"MATCH({graphSocDatasetNodeName} {{Soc: {0}}}) -[r:{graphJobProfileRelationshipName}]-> (n:{graphJobProfileNodeName}) RETURN n";
var models = await genericGraphQueryService.ExecuteCypherQuery <GraphJobProfileModel>(graphReplicaSet, query).ConfigureAwait(false);
return(models);
}
public async Task <List <SocModel> > GetSummaryAsync(GraphReplicaSet graphReplicaSet)
{
logger.LogInformation($"Fetching Soc summary from {graphReplicaSet}");
var query = $"MATCH(s:{graphSocDatasetNodeName}) RETURN s";
var models = await genericGraphQueryService.ExecuteCypherQuery <SocModel>(graphReplicaSet, query).ConfigureAwait(false);
return(models);
}
public async Task PurgeSocAsync(int soc, GraphReplicaSet graphReplicaSet)
{
logger.LogInformation($"Purging Graph of LMI data for SOC {soc}");
var commands = graphConnector.BuildPurgeCommandsForInitialKey(soc.ToString(CultureInfo.InvariantCulture));
logger.LogInformation($"Purging Graph of LMI data for SOC {soc}: executing commands");
await graphConnector.RunAsync(commands, graphReplicaSet).ConfigureAwait(false);
logger.LogInformation($"Purged Graph of LMI data for SOC {soc}");
}
public async Task PurgeAsync(GraphReplicaSet graphReplicaSet)
{
logger.LogInformation("Purging Graph of LMI data");
var commands = graphConnector.BuildPurgeCommands();
logger.LogInformation("Purging Graph of LMI data: executing commands");
await graphConnector.RunAsync(commands, graphReplicaSet).ConfigureAwait(false);
logger.LogInformation("Purged Graph of LMI data");
}
public async Task <List <TModel> > ExecuteCypherQuery <TModel>(GraphReplicaSet graphReplicaSet, string query)
where TModel : class, new()
{
string replicaSetName = graphReplicaSet switch
{
GraphReplicaSet.Published => graphOptions.PublishedReplicaSetName,
GraphReplicaSet.Draft => graphOptions.DraftReplicaSetName,
_ => throw new NotImplementedException(),
};
var result = await graphCluster.Run(replicaSetName, new GenericCypherQueryModel <TModel>(query)).ConfigureAwait(false);
return(result);
}
public async Task <GraphPredictedModel?> GetPredictedAsync(GraphReplicaSet graphReplicaSet, int soc)
{
logger.LogInformation($"Fetching Soc ({soc}) {graphPredictedNodeName} from {graphReplicaSet}");
var query = $"MATCH({graphSocDatasetNodeName} {{Soc: {soc}}}) -[r:{graphJobGrowthRelationshipName}]-> (n:{graphPredictedNodeName}) RETURN n";
var models = await genericGraphQueryService.ExecuteCypherQuery <GraphPredictedModel>(graphReplicaSet, query).ConfigureAwait(false);
var model = models?.FirstOrDefault();
if (model != null)
{
model.PredictedEmployment = await GetPredictedYearAsync(graphReplicaSet, soc).ConfigureAwait(false);
}
return(model);
}
public async Task <List <GraphBreakdownYearModel>?> GetBreakdownYearAsync(GraphReplicaSet graphReplicaSet, int soc, string graphRelationshipName)
{
logger.LogInformation($"Fetching Soc ({soc}) {graphRelationshipName}/{graphBreakdownRelationshipName} from {graphReplicaSet}");
var query = $"MATCH({graphSocDatasetNodeName} {{Soc: {soc}}}) -[r:{graphRelationshipName}]-> (n:{graphBreakdownNodeName}) -[r2:{graphBreakdownRelationshipName}]-> (n2:{graphBreakdownYearNodeName}) RETURN n2";
var models = await genericGraphQueryService.ExecuteCypherQuery <GraphBreakdownYearModel>(graphReplicaSet, query).ConfigureAwait(false);
if (models != null && models.Any())
{
foreach (var model in models)
{
model.Breakdown = await GetBreakdownYearValueAsync(graphReplicaSet, soc, graphRelationshipName).ConfigureAwait(false);
}
}
return(models);
}
public async Task GraphConnectorRunReturnsSuccess(GraphReplicaSet graphReplicaSet)
{
// arrange
var commands = new List <string>
{
"command one",
"command two",
};
A.CallTo(() => fakeServiceProvider.GetService(typeof(ICustomCommand))).Returns(new CustomCommand());
// act
await graphConnector.RunAsync(commands, graphReplicaSet).ConfigureAwait(false);
// assert
A.CallTo(() => fakeServiceProvider.GetService(typeof(ICustomCommand))).MustHaveHappened(commands.Count, Times.Exactly);
A.CallTo(() => fakeGraphCluster.Run(A <string> .Ignored, A <ICommand[]> .Ignored)).MustHaveHappenedOnceExactly();
Assert.True(true);
}
public async Task RunAsync(IList <string>?commands, GraphReplicaSet graphReplicaSet)
{
_ = commands ?? throw new ArgumentNullException(nameof(commands));
string replicaSetName = graphReplicaSet switch
{
GraphReplicaSet.Published => graphOptions.PublishedReplicaSetName,
GraphReplicaSet.Draft => graphOptions.DraftReplicaSetName,
_ => throw new NotImplementedException(),
};
var customCommands = new List <ICustomCommand>();
foreach (var command in commands)
{
var customCommand = serviceProvider.GetRequiredService <ICustomCommand>();
customCommand.Command = command;
customCommands.Add(customCommand);
}
await graphCluster.Run(replicaSetName, customCommands.ToArray()).ConfigureAwait(false);
}
public async Task PublishAsync(GraphReplicaSet fromGraphReplicaSet, GraphReplicaSet toGraphReplicaSet)
{
logger.LogInformation($"Publishing from {fromGraphReplicaSet} LMI data to {toGraphReplicaSet} graph");
var socModels = await socGraphQueryService.GetSummaryAsync(fromGraphReplicaSet).ConfigureAwait(false);
if (socModels != null && socModels.Any())
{
foreach (var socModel in socModels)
{
var graphSocDataset = await socGraphQueryService.GetDetailAsync(fromGraphReplicaSet, socModel.Soc).ConfigureAwait(false);
if (graphSocDataset != null)
{
await ImportAsync(graphSocDataset, toGraphReplicaSet).ConfigureAwait(false);
}
}
}
logger.LogInformation($"Published from {fromGraphReplicaSet} LMI data to {toGraphReplicaSet} graph");
}
public async Task <GraphSocDatasetModel?> GetDetailAsync(GraphReplicaSet graphReplicaSet, int soc)
{
logger.LogInformation($"Fetching Soc ({soc}) details from {graphReplicaSet}");
var query = $"MATCH(s:{graphSocDatasetNodeName} {{Soc: {soc}}}) RETURN s";
var graphSocDatasets = await genericGraphQueryService.ExecuteCypherQuery <GraphSocDatasetModel>(graphReplicaSet, query).ConfigureAwait(false);
var graphSocDataset = graphSocDatasets?.FirstOrDefault();
if (graphSocDataset != null)
{
graphSocDataset.JobProfiles = await GetJobProfilesAsync(graphReplicaSet, soc).ConfigureAwait(false);
graphSocDataset.JobGrowth = await GetPredictedAsync(graphReplicaSet, soc).ConfigureAwait(false);
graphSocDataset.QualificationLevel = await GetBreakdownAsync(graphReplicaSet, soc, graphQualificationLevelRelationshipName).ConfigureAwait(false);
graphSocDataset.EmploymentByRegion = await GetBreakdownAsync(graphReplicaSet, soc, graphEmploymentByRegionRelationshipName).ConfigureAwait(false);
graphSocDataset.TopIndustriesInJobGroup = await GetBreakdownAsync(graphReplicaSet, soc, graphTopIndustriesInJobGroupRelationshipName).ConfigureAwait(false);
}
return(graphSocDataset);
}
public async Task <bool> ImportAsync(GraphSocDatasetModel?graphSocDataset, GraphReplicaSet graphReplicaSet)
{
_ = graphSocDataset ?? throw new ArgumentNullException(nameof(graphSocDataset));
try
{
logger.LogInformation($"Importing SOC dataset to Graph: {graphSocDataset.Soc}");
var commands = graphConnector.BuildImportCommands(graphSocDataset);
logger.LogInformation($"Importing SOC dataset to Graph: {graphSocDataset.Soc}: executing commands");
await graphConnector.RunAsync(commands, graphReplicaSet).ConfigureAwait(false);
logger.LogInformation($"Imported SOC dataset to Graph: {graphSocDataset.Soc}");
return(true);
}
catch (Exception ex)
{
logger.LogError(ex, $"Error received importing data to graphs database for LMI SOC: {graphSocDataset.Soc}");
return(false);
}
}
public GraphReplicaSetTests(ITestOutputHelper testOutputHelper)
: base(testOutputHelper)
{
GraphReplicaSet = new GraphReplicaSet(ReplicaSetName, GraphInstances, Logger);
}