public ClusterMonitorService(
IServiceProvider sp,
IClusterRequestHandler _node,
IEntitiesRepository entitiesRepository,
MetricManagementService metricManagementService,
IMetricTicksRepository metricTicksRepository,
IDatabaseMetricsCollector databaseMetricsCollector,
NodeStateService nodeStateService,
IOptions <ClusterOptions> clusterOptions)
{
_metricManagementService = metricManagementService;
// var sp = serviceProvider.CreateScope().ServiceProvider;
_mediator = sp.GetService <IMediator>();
_logger = sp.GetService <ILogger <ClusterMonitorService> >();
_state = sp.GetService <IClusterStateService>();
_logger.LogInformation("Starting clean up service...");
node = _node;
_entitiesRepository = entitiesRepository;
_metricTicksRepository = metricTicksRepository;
_databaseMetricsCollector = databaseMetricsCollector;
monitoringTimer = new System.Threading.Timer(CollectMetricsEventHandler);
node.MetricGenerated += metricGenerated;
_nodeStateService = nodeStateService;
_clusterOptions = clusterOptions;
Start();
}
public BaseService(ILogger logger, ClusterOptions clusterOptions, NodeOptions nodeOptions, IStateMachine <State> stateMachine, NodeStateService nodeState)
{
_logger = logger;
ClusterOptions = clusterOptions;
NodeOptions = nodeOptions;
StateMachine = stateMachine;
NodeStateService = nodeState;
}
public ClusterClient(IClusterConnectionPool connectionPool,
IServiceProvider serviceProvider,
NodeStateService nodeStateService)
{
_clusterConnectionPool = connectionPool;
_nodeStateService = nodeStateService;
_serviceProvider = serviceProvider;
}
public Monitor(
IStateMachine <State> stateMachine,
ILogger <Monitor <State> > logger,
NodeStateService nodeStateService)
{
_logger = logger;
_stateMachine = stateMachine;
_nodeStateService = nodeStateService;
}
public ValuesController(
IDataRouter router,
ClusterClient clusterClient,
NodeStateService node,
IStateMachine <TestState> stateMachine)
{
_clusterClient = clusterClient;
_router = (TestDataRouter)router;
_node = node;
_stateMachine = stateMachine;
}
public MetricManagementService(
ILogger <MetricManagementService> logger,
IClusterRequestHandler node,
NodeStateService nodeStateService,
IEntitiesRepository entitiesRepository,
IConfiguration configuration
)
{
_logger = logger;
_logger.LogInformation("Populating Metrics...");
_metricLibrary = new MetricLibrary();
_node = node;
_nodeStateService = nodeStateService;
_entitiesRepository = entitiesRepository;
_configuration = configuration;
EnableMetrics = _configuration.GetValue <bool>("EnableMonitoring");
if (EnableMetrics)
{
writeThread = new Task(async() =>
{
MetricTick tick;
while (true)
{
// Console.WriteLine("Number of tasks " + _ticks.Count());
if (_nodeStateService.InCluster)
{
if (_ticks.TryDequeue(out tick))
{
tick.Date = tick.Date.ToUniversalTime();
tick.Id = Guid.NewGuid();
var startTime = DateTime.Now;
await _node.Handle(new AddShardWriteOperation()
{
WaitForSafeWrite = true,
Operation = ConsensusCore.Domain.Enums.ShardOperationOptions.Create,
Data = tick,
Metric = false // Do not metric the metric write operations
});
_logger.LogDebug("Total write time took " + (DateTime.Now - startTime).TotalMilliseconds + " total ticks left in queue " + _ticks.Count());
if (_ticks.Count > 100)
{
_logger.LogWarning("Tick count is greater then 100...");
}
}
}
await Task.Delay(10);
}
});
writeThread.Start();
}
}
public Writer(
ILogger <Writer <State> > logger,
IShardRepository shardRepository,
IDataRouter dataRouter,
IStateMachine <State> stateMachine,
NodeStateService nodeStateService,
ClusterClient clusterClient)
{
_logger = logger;
_dataRouter = dataRouter;
_shardRepository = shardRepository;
_stateMachine = stateMachine;
_nodeStateService = nodeStateService;
_clusterClient = clusterClient;
}
public ClusterRequestHandler(
IOptions <ClusterOptions> clusterOptions,
ILogger <ClusterRequestHandler <State> > logger,
IRaftService raftService,
NodeStateService nodeStateService,
ClusterClient clusterClient,
IDataService dataService)
{
_clusterClient = clusterClient;
_clusterOptions = clusterOptions.Value;
_nodeStateService = nodeStateService;
_logger = logger;
_raftService = raftService;
_dataService = dataService;
}
public Syncer(IShardRepository shardRepository,
ILogger <Syncer <State> > logger,
IStateMachine <State> stateMachine,
ClusterClient clusterClient,
NodeStateService nodeStateService,
Writer <State> writer)
{
_logger = logger;
_shardRepository = shardRepository;
_stateMachine = stateMachine;
_nodeStateService = nodeStateService;
_clusterClient = clusterClient;
_writer = writer;
ResyncShardWriteOperations();
}
public NodeController(IClusterRequestHandler handler,
ILogger <NodeController <State> > logger,
NodeStateService nodeStateService,
IStateMachine <State> stateMachine,
INodeStorage <State> nodeStorage,
IClusterConnectionPool clusterConnectionPool,
IShardRepository shardRepository)
{
_handler = handler;
Logger = logger;
_nodeStateService = nodeStateService;
_stateMachine = stateMachine;
_nodeStorage = nodeStorage;
_clusterConnectionPool = clusterConnectionPool;
_shardRepository = shardRepository;
}
public CompleteStepCommandHandler(IEntitiesRepository entitiesRepository,
IClusterStateService clusterStateService,
ILogger <CompleteStepCommandHandler> logger,
CindiClusterOptions options,
IMediator mediator,
IClusterRequestHandler node,
NodeStateService nodeStateService
)
{
_entitiesRepository = entitiesRepository;
_clusterStateService = clusterStateService;
Logger = logger;
_option = options;
_node = node;
_mediator = mediator;
}
public TaskService(
ILoggerFactory loggerFactory,
IStateMachine <State> stateMachine,
NodeStateService nodeStateService,
ClusterClient clusterClient,
IOptions <ClusterOptions> clusterOptions,
IDataService dataService)
{
_logger = loggerFactory.CreateLogger <TaskService <State> >();
_stateMachine = stateMachine;
_clusterOptions = clusterOptions.Value;
_clusterClient = clusterClient;
_nodeStateService = nodeStateService;
_dataService = dataService;
_monitor = new Monitor <State>(stateMachine, loggerFactory.CreateLogger <Monitor <State> >(), nodeStateService);
_scanTasks = new Task(async() => await ScanTasks());
_scanTasks.Start();
}
public DataService(
ILoggerFactory loggerFactory,
IShardRepository shardRepository,
IDataRouter dataRouter,
IStateMachine <State> stateMachine,
NodeStateService nodeStateService,
ClusterClient clusterClient,
IOptions <ClusterOptions> clusterOptions,
IOperationCacheRepository transactionCacheRepository,
IOptions <NodeOptions> nodeOptions)
{
_nodeStateService = nodeStateService;
_nodeOptions = nodeOptions.Value;
_clusterOptions = clusterOptions.Value;
_stateMachine = stateMachine;
_writeCache = new WriteCache(transactionCacheRepository, loggerFactory.CreateLogger <WriteCache>(), _nodeOptions.PersistWriteQueue);
_logger = loggerFactory.CreateLogger <DataService <State> >();
_shardRepository = shardRepository;
_clusterClient = clusterClient;
Reader = new Reader <State>(
loggerFactory.CreateLogger <Reader <State> >(),
shardRepository,
dataRouter,
stateMachine,
nodeStateService,
clusterClient);;
Allocator = new Allocator <State>(
loggerFactory.CreateLogger <Allocator <State> >(),
shardRepository,
dataRouter,
stateMachine,
nodeStateService,
clusterClient);
Writer = new Writer <State>(loggerFactory.CreateLogger <Writer <State> >(),
shardRepository,
dataRouter,
stateMachine,
nodeStateService,
clusterClient
);
Syncer = new Syncer <State>(shardRepository, loggerFactory.CreateLogger <Syncer <State> >(), stateMachine, clusterClient, nodeStateService, Writer);
_writeTask = new Task(async() =>
{
//Before you write you should first dequeue all transactions
if (_writeCache.TransitQueue.Count() > 0)
{
_logger.LogInformation("Found transactions in transit, attempting to reapply them...");
foreach (var operationKV in _writeCache.TransitQueue.ToDictionary(entry => entry.Key, entry => entry.Value))
{
var operation = operationKV.Value;
try
{
var result = await Writer.WriteShardData(operation.Data, operation.Operation, operation.Id, operation.TransactionDate);
}
catch (Exception e)
{
_logger.LogError("Failed to apply operation " + operation.Id + " with exception " + e.Message + Environment.NewLine + e.StackTrace);
try
{
await _writeCache.CompleteOperation(operation.Id);
}
catch (Exception completionError)
{
_logger.LogError("Error removing operation from transit queue with error " + completionError.Message + Environment.NewLine
+ completionError.StackTrace + Environment.NewLine + JsonConvert.SerializeObject(operation, Formatting.Indented));
}
}
}
}
while (true)
{
try
{
var operation = await _writeCache.DequeueOperation();
if (operation != null)
{
try
{
var result = await Writer.WriteShardData(operation.Data, operation.Operation, operation.Id, operation.TransactionDate);
}
catch (Exception e)
{
_logger.LogError("Failed to write operation with exception " + e.Message + Environment.NewLine + JsonConvert.SerializeObject(operation, Formatting.Indented));
}
await _writeCache.CompleteOperation(operation.Id);
}
else
{
//Release write thread for a small amount of time
await Task.Delay(100);
}
}
catch (Exception e)
{
_logger.LogError("Encountered critical write error " + e.Message + Environment.NewLine + e.StackTrace);
}
}
});
_writeTask.Start();
TaskUtility.RestartTask(ref _indexCreationTask, async() => await CreateIndexLoop());
_allocationTask = new Task(async() => await AllocateShards());
_allocationTask.Start();
_replicaValidationChecksTask = Task.Run(async() => await CheckAllReplicas());
_objectLockWatcher = new Task(async() => await CheckLocks());
_objectLockWatcher.Start();
if (_nodeOptions.EnablePerformanceLogging)
{
var performancePrinting = new Task(() =>
{
while (true)
{
Console.Clear();
Console.WriteLine("Performance Report...");
foreach (var value in totals)
{
Console.WriteLine(value.Key + ":" + (value.Value / totalRequests));
}
Console.WriteLine("Queue:" + _writeCache.OperationsInQueue);
Task.Delay(1000);
}
});
performancePrinting.Start();
}
}
/* public static RaftService<TestState> GetTestConsensusCoreNode()
* {
* var moqClusterOptions = new Mock<IOptions<ClusterOptions>>();
* moqClusterOptions.Setup(mqo => mqo.Value).Returns(new ClusterOptions()
* {
* NodeUrls = "localhost:5022",
* TestMode = true,
* NumberOfShards = 1,
* DataTransferTimeoutMs = 1000,
* ElectionTimeoutMs = 1000,
* LatencyToleranceMs = 1000,
* MinimumNodes = 1
* });
*
* var moqNodeOptions = new Mock<IOptions<NodeOptions>>();
* moqNodeOptions.Setup(mqo => mqo.Value).Returns(new NodeOptions() { });
*
* var serviceProvider = new ServiceCollection()
* .AddLogging()
* .BuildServiceProvider();
*
* var factory = serviceProvider.GetService<ILoggerFactory>();
*
* var logger = factory.CreateLogger<RaftService<TestState>>();
*
* NodeInMemoryRepository<TestState> inMemoryRepository = new NodeInMemoryRepository<TestState>();
* var NodeStorage = new NodeStorage<TestState>(inMemoryRepository) { };
* var _dataRouter = new TestDataRouter();
* var _stateMachine = new StateMachine<TestState>();
* var _connector = new ClusterClient(TimeSpan.FromMilliseconds(1000), TimeSpan.FromMilliseconds(1000));
*
* return new ConsensusCoreNode<TestState>(moqClusterOptions.Object,
* moqNodeOptions.Object,
* logger,
* _stateMachine,
* inMemoryRepository,
* _connector,
* _dataRouter,
* new ShardManager<TestState, IShardRepository>(_stateMachine,
* factory.CreateLogger<ShardManager<TestState, IShardRepository>>(),
* _connector,
* _dataRouter,
* moqClusterOptions.Object,
* inMemoryRepository),
* NodeStorage
* );
* }*/
public static DataService <TestState> GetTestShardManager()
{
var serviceProvider = new ServiceCollection()
.AddLogging()
.BuildServiceProvider();
var _stateMachine = new StateMachine <TestState>()
{
};
var factory = serviceProvider.GetService <ILoggerFactory>();
NodeInMemoryRepository <TestState> inMemoryRepository = new NodeInMemoryRepository <TestState>();
var _dataRouter = new TestDataRouter();
var moqClusterOptions = new Mock <IOptions <ClusterOptions> >();
var moqNodeOptions = new Mock <IOptions <NodeOptions> >();
moqNodeOptions.Setup(mqo => mqo.Value).Returns(new NodeOptions()
{
});
Guid nodeStorageId = Guid.NewGuid();
var NodeStorage = new NodeStorage <TestState>(factory.CreateLogger <NodeStorage <TestState> >(), inMemoryRepository)
{
Id = nodeStorageId
};
var nodeStateService = new NodeStateService()
{
Id = nodeStorageId
};
var services = new ServiceCollection();
var provider = services.BuildServiceProvider();
var _connector = new ClusterClient(new ClusterConnectionPool <TestState>(_stateMachine, TimeSpan.FromMilliseconds(10000), TimeSpan.FromMilliseconds(1000)), provider, nodeStateService);
_stateMachine.ApplyLogsToStateMachine(new List <ConsensusCore.Domain.Models.LogEntry>()
{
new ConsensusCore.Domain.Models.LogEntry()
{
Commands = new List <BaseCommand>()
{
new CreateIndex()
{
Type = "number",
Shards = new List <ShardAllocationMetadata>()
{
new ShardAllocationMetadata()
{
Id = DefaultShardId,
InsyncAllocations = new HashSet <Guid>()
{
nodeStorageId
},
PrimaryAllocation = nodeStorageId,
Type = "number",
StaleAllocations = new HashSet <Guid>()
}
}
}
}
}
});
moqClusterOptions.Setup(mqo => mqo.Value).Returns(new ClusterOptions()
{
NodeUrls = "localhost:5022",
TestMode = true,
NumberOfShards = 1,
DataTransferTimeoutMs = 1000,
ElectionTimeoutMs = 1000,
LatencyToleranceMs = 1000,
MinimumNodes = 1
});
var manager = new DataService <TestState>(
factory,
inMemoryRepository,
_dataRouter,
_stateMachine,
nodeStateService,
_connector,
moqClusterOptions.Object,
inMemoryRepository,
moqNodeOptions.Object
);
return(manager);
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app,
IHostingEnvironment env,
IClusterStateService service,
ILogger <Startup> logger,
IRaftService raftService,
IDataService dataService,
ITaskService taskService,
IStateMachine <CindiClusterState> stateMachine,
NodeStateService node,
IDatabaseMetricsCollector collector,
ClusterMonitorService monitor,
IMediator mediator,
IServiceProvider serviceProvider,
MetricManagementService metricManagementService,
InternalBotManager internalBotManager
)
{
BootstrapThread = new Task(() =>
{
while (node.Status != ConsensusCore.Domain.Models.NodeStatus.Yellow &&
node.Status != ConsensusCore.Domain.Models.NodeStatus.Green
)
{
logger.LogInformation("Waiting for cluster to establish a quorum");
Thread.Sleep(1000);
}
var med = (IMediator)app.ApplicationServices.CreateScope().ServiceProvider.GetService(typeof(IMediator));
ClusterStateService.Initialized = stateMachine.CurrentState.Initialized;
var key = Configuration.GetValue <string>("EncryptionKey");
if (key != null)
{
if (ClusterStateService.Initialized)
{
logger.LogWarning("Loading key in configuration file, this is not recommended for production.");
try
{
service.SetEncryptionKey(key);
logger.LogInformation("Successfully applied encryption key.");
}
catch (InvalidPrivateKeyException e)
{
logger.LogError("Failed to apply stored key. Key does not match registered encryption hash.");
}
}
}
if (!ClusterStateService.Initialized)
{
if (key != null)
{
logger.LogWarning("Initializing new node with key in configuration file, this is not recommended for production.");
service.SetEncryptionKey(key);
}
else
{
logger.LogWarning("No default key detected, post key to /api/cluster/encryption-key.");
}
var setPassword = Configuration.GetValue <string>("DefaultPassword");
if (node.Role == ConsensusCore.Domain.Enums.NodeState.Leader)
{
// Thread.Sleep(5000);
med.Send(new InitializeClusterCommand()
{
DefaultPassword = setPassword == null ? "PleaseChangeMe" : setPassword,
Name = Configuration.GetValue <string>("ClusterName")
}).GetAwaiter().GetResult();
}
}
if (node.Role == ConsensusCore.Domain.Enums.NodeState.Leader)
{
metricManagementService.InitializeMetricStore();
if (service.GetSettings == null)
{
logger.LogWarning("No setting detected, resetting settings to default.");
med.Send(new UpdateClusterStateCommand()
{
DefaultIfNull = true
}).GetAwaiter().GetResult();;
}
}
foreach (var template in InternalStepLibrary.All)
{
med.Send(new CreateStepTemplateCommand()
{
Name = template.Name,
InputDefinitions = template.InputDefinitions,
OutputDefinitions = template.OutputDefinitions,
CreatedBy = SystemUsers.SYSTEM_TEMPLATES_MANAGER,
Version = template.Version,
Description = template.Description
}).GetAwaiter().GetResult();
}
//internalBotManager.AddAdditionalBot();
});
BootstrapThread.Start();
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
}
app.UseHttpsRedirection();
app.UseSwagger();
app.UseSwaggerUI(c =>
{
c.SwaggerEndpoint("/swagger/v1/swagger.json", "My API V1");
//c.OAuthUseBasicAuthenticationWithAccessCodeGrant();
});
app.UseCindiClusterPipeline();
app.UseAuthentication();
app.UseCors("AllowAll");
app.UseMvc();
if (EnableUI)
{
OverrideUISettings();
app.UseSpaStaticFiles();
app.UseSpa(spa =>
{
// To learn more about options for serving an Angular SPA from ASP.NET Core,
// see https://go.microsoft.com/fwlink/?linkid=864501
spa.Options.SourcePath = "ClientApp";
if (env.IsDevelopment())
{
spa.UseAngularCliServer(npmScript: "start");
}
});
}
}
