/// <summary>
/// Creates or adds data to one or more time series. The response is empty if all time series in the request were written. If any time series could not be written, a corresponding failure message is included in the error response.
/// Documentation https://developers.google.com/monitoring/v3/reference/timeSeries/create
/// Generation Note: This does not always build corectly. Google needs to standardise things I need to figuer out which ones are wrong.
/// </summary>
/// <param name="service">Authenticated Monitoring service.</param>
/// <param name="name">The project on which to execute the request. The format is "projects/{project_id_or_number}".</param>
/// <param name="body">A valid Monitoring v3 body.</param>
/// <returns>EmptyResponse</returns>
public static Empty Create(MonitoringService service, string name, CreateTimeSeriesRequest body)
{
try
{
// Initial validation.
if (service == null)
{
throw new ArgumentNullException("service");
}
if (body == null)
{
throw new ArgumentNullException("body");
}
if (name == null)
{
throw new ArgumentNullException(name);
}
// Make the request.
return(service.TimeSeries.Create(body, name).Execute());
}
catch (Exception ex)
{
throw new Exception("Request TimeSeries.Create failed.", ex);
}
}
/// <summary>
/// Creates a new metric descriptor. User-created metric descriptors define custom metrics.
/// Documentation https://developers.google.com/monitoring/v3/reference/metricDescriptors/create
/// Generation Note: This does not always build corectly. Google needs to standardise things I need to figuer out which ones are wrong.
/// </summary>
/// <param name="service">Authenticated Monitoring service.</param>
/// <param name="name">The project on which to execute the request. The format is "projects/{project_id_or_number}".</param>
/// <param name="body">A valid Monitoring v3 body.</param>
/// <returns>MetricDescriptorResponse</returns>
public static MetricDescriptor Create(MonitoringService service, string name, MetricDescriptor body)
{
try
{
// Initial validation.
if (service == null)
{
throw new ArgumentNullException("service");
}
if (body == null)
{
throw new ArgumentNullException("body");
}
if (name == null)
{
throw new ArgumentNullException(name);
}
// Make the request.
return(service.MetricDescriptors.Create(body, name).Execute());
}
catch (Exception ex)
{
throw new Exception("Request MetricDescriptors.Create failed.", ex);
}
}
/// <summary>
/// Lists monitored resource descriptors that match a filter. This method does not require a Stackdriver account.
/// Documentation https://developers.google.com/monitoring/v3/reference/monitoredResourceDescriptors/list
/// Generation Note: This does not always build corectly. Google needs to standardise things I need to figuer out which ones are wrong.
/// </summary>
/// <param name="service">Authenticated Monitoring service.</param>
/// <param name="name">The project on which to execute the request. The format is "projects/{project_id_or_number}".</param>
/// <param name="optional">Optional paramaters.</param> /// <returns>ListMonitoredResourceDescriptorsResponseResponse</returns>
public static ListMonitoredResourceDescriptorsResponse List(MonitoringService service, string name, MonitoredResourceDescriptorsListOptionalParms optional = null)
{
try
{
// Initial validation.
if (service == null)
{
throw new ArgumentNullException("service");
}
if (name == null)
{
throw new ArgumentNullException(name);
}
// Building the initial request.
var request = service.MonitoredResourceDescriptors.List(name);
// Applying optional parameters to the request.
request = (MonitoredResourceDescriptorsResource.ListRequest)SampleHelpers.ApplyOptionalParms(request, optional);
// Requesting data.
return(request.Execute());
}
catch (Exception ex)
{
throw new Exception("Request MonitoredResourceDescriptors.List failed.", ex);
}
}
public MonitoringController(
MonitoringService monitoringService,
IDeploymentTargetReadService targetSource)
{
_monitoringService = monitoringService;
_targetSource = targetSource;
}
/// <summary>
///
/// </summary>
/// <returns></returns>
public ActionResult NodeSummary()
{
try
{
if (DateTime.Now - LastNodeSummaryUpdate > FwdingCacheTimeout)
{
bool useTestnet = GetUseTestnet();
LndRpcClient lndClient = GetLndClient(useTestnet);
var xfers = lndClient.GetForwardingEvents();
//Total amount transferred
nodeSummaryViewModel.TotalValueXfer = xfers.forwarding_events == null ? 0 : Convert.ToDouble(xfers.forwarding_events.Sum(f => Convert.ToInt64(f.amt_out))) / 100000000.0;
nodeSummaryViewModel.NumXfer = xfers.forwarding_events == null ? 0 : xfers.forwarding_events.Count;
nodeSummaryViewModel.TotalFees = xfers.forwarding_events == null ? 0.ToString("0.00000000") : (Convert.ToDouble(xfers.forwarding_events.Sum(f => Convert.ToInt64(f.fee))) / 100000000.0).ToString("0.00000000");
LastNodeSummaryUpdate = DateTime.Now;
}
}
catch (Exception e)
{
ViewBag.TotalValueXfer = "Unknown";
ViewBag.NumXfer = "Unknown";
ViewBag.TotalFees = "Unknown";
MonitoringService.SendMessage("Lightning Error", " Exception: " + e.Message + "\r\n Stack: " + e.StackTrace);
}
return(PartialView("NodeSummary", nodeSummaryViewModel));
}
public async Task TestMonitoringServiceSuccessAsync()
{
var provider = Substitute.For <ISiteAvailabilityProvider>();
var repository = Substitute.For <ISiteAvailabilityRepository>();
var service = new MonitoringService(provider, repository);
var cts = new CancellationTokenSource();
cts.CancelAfter(3000);
var sites = new List <Site>();
sites.Add(new Site {
Id = Guid.NewGuid(), Url = "url1", IsAvailable = false
});
sites.Add(new Site {
Id = Guid.NewGuid(), Url = "url2", IsAvailable = true
});
repository.GetAllSitesAsync().Returns(Task.FromResult <IEnumerable <Site> >(sites));
try
{
await service.StartAsync(cts.Token);
Assert.Fail(); //This line should be unreachable
}
catch (TaskCanceledException e)
{
Assert.AreEqual("A task was canceled.", e.Message);
}
}
private static string TransmitViaJSONRPC(string ClaimId, CoinpanicContext db, CoinClaim userclaim, string signedTransaction, string coin)
{
GetConnectionDetails(coin, out string host, out int port, out string user, out string pass);
var client = new RestClient("http://" + host + ":" + Convert.ToString(port));
client.Authenticator = new HttpBasicAuthenticator(user, pass);
var request = new RestRequest("/", Method.POST);
request.RequestFormat = DataFormat.Json;
request.AddBody(new
{
jsonrpc = "1.0",
id = "1",
method = "sendrawtransaction",
@params = new List <string>()
{
signedTransaction
},
});
var restResponse = client.Execute(request);
var content = restResponse.Content; // raw content as string
userclaim.TransactionHash = content;
userclaim.WasTransmitted = true;
userclaim.SubmitDate = DateTime.Now;
db.SaveChanges();
MonitoringService.SendMessage("New " + userclaim.CoinShortName + " broadcasting via explorer " + Convert.ToString(userclaim.TotalValue),
"Claim broadcast: https://www.coinpanic.com/Claim/ClaimConfirm?claimId=" + ClaimId + " " + " for " + userclaim.CoinShortName + "\r\n " + signedTransaction);
return(content);
}
public override bool PreExec(IWpfTextView textView, DeveroomEditorCommandTargetKey commandKey, IntPtr inArgs = default(IntPtr))
{
MonitoringService.MonitorCommandCommentUncomment();
var selectionSpan = GetSelectionSpan(textView);
var lines = GetSpanFullLines(selectionSpan).ToArray();
Debug.Assert(lines.Length > 0);
using (var textEdit = selectionSpan.Snapshot.TextBuffer.CreateEdit())
{
foreach (var line in lines)
{
int commentCharPosition = line.GetText().IndexOf('#');
if (commentCharPosition >= 0)
{
textEdit.Delete(line.Start.Position + commentCharPosition, 1);
}
}
textEdit.Apply();
}
SetSelectionToChangedLines(textView, lines);
return(true);
}
public ActionResult Details(int id, int id2)
{
ViewBag.Id2 = id2;
post = MonitoringService.GetMonitoringService().GetPost(id);
Post p = Newtonsoft.Json.JsonConvert.DeserializeObject <Post>(post.Content);
return(PartialView("Details", p));
}
public HookService(
IDeploymentTargetReadService targetSource,
DeploymentService deploymentService,
ILogger logger,
MonitoringService monitoringService)
{
_targetSource = targetSource;
_deploymentService = deploymentService;
_logger = logger;
_monitoringService = monitoringService;
}
public ActionResult PostDelete(int id, int id2)
{
int Id = id;
MonitoringService.GetMonitoringService().DeletePost(Id);
appealContext.Appeals.Find(id2).AppealStatus = true;
appealContext.SaveChanges();
return(View("HomePage", appealContext.Appeals.ToList().ToPagedList(pageNumber, pageSize)));
}
public async Task Test_ServiceIsDown_MailSent()
{
// Given
var srv = new MonitoringService();
var down = new DownService();
// When
await srv.Test(down);
// Then : check your email
}
public void SerializationFileConfig()
{
// Given
var srv = new MonitoringService();
// When
var cfg = srv.DeserializeConfiguration(getFilePath("config2.json");
// Then
Assert.IsNotNull(cfg);
}
public async Task StartMonitoring_AnyState_Ok()
{
MonitoringService svc = MonitoringService.Create();
bool res = false;
await Task.Run(() =>
{
svc.StartMonitoring(new CancellationTokenSource(), DateTime.Now, DateTime.Now + TimeSpan.FromMinutes(2), "love", () => { });
res = true;
});
Assert.IsTrue(res);
}
void OnDeserialized(StreamingContext context)
{
FiredDateTime = string.IsNullOrEmpty(firedDateTimeString) ? default(DateTime) : DateTime.Parse(firedDateTimeString);
Enum.TryParse(severityString, out Severity);
switch (monitoringServiceString)
{
case "Platform":
MonitoringService = MonitoringService.Platform;
break;
case "Log Analytics":
MonitoringService = MonitoringService.LogAnalytics;
break;
case "Application Insights":
MonitoringService = MonitoringService.ApplicationInsights;
break;
case "Activity Log - Administrative":
MonitoringService = MonitoringService.ActivityLog;
break;
case "ServiceHealth":
MonitoringService = MonitoringService.ServiceHealth;
break;
case "Resource Health":
MonitoringService = MonitoringService.ResourceHealth;
break;
default:
break;
}
switch (signalTypeString)
{
case "Metric":
SignalType = SignalType.Metric;
break;
case "Log":
SignalType = SignalType.Log;
break;
case "Activity Log":
SignalType = SignalType.ActivityLog;
break;
default:
break;
}
Enum.TryParse(monitorConditionString, out MonitorCondition);
}
public PackageWebHookAutoDeployHandler(
IDeploymentTargetReadService targetSource,
DeploymentWorkerService deploymentService,
ILogger logger,
MonitoringService monitoringService,
IApplicationSettingsStore applicationSettingsStore)
{
_targetSource = targetSource;
_deploymentService = deploymentService;
_logger = logger;
_monitoringService = monitoringService;
_applicationSettingsStore = applicationSettingsStore;
}
public async Task Execute(IJobExecutionContext context)
{
List <SiteInfo> list = new List <SiteInfo>();
using (var service = new MonitoringService()) {
list = await service.CheckSites();
}
//TO DO: some action
var hubContext = GlobalHost.ConnectionManager.GetHubContext <NotificationHub>();
// отправляем сообщение
hubContext.Clients.All.displaySiteList(list);
}
public bool TrySignIn(string userIdentificator, string password, bool createPersistentCookie, string ip)
{
if (string.IsNullOrWhiteSpace(userIdentificator) || string.IsNullOrWhiteSpace(password))
{
throw new ArgumentException();
}
if (ValidationHelper.IsCorrectEmail(userIdentificator))
{
var email = userIdentificator;
var user = SecurityService.EmailUserValidation(email, password);
if (user == null)
{
return(false);
}
if (user.BlockedUserRecord != null)
{
MonitoringService.AsynchLogUserAuthentification(user.Id.ToString(), ip, false);
throw new BusinessLogicException("Вы были забанены! Причина: " + user.BlockedUserRecord.Reason);
}
MonitoringService.AsynchLogUserAuthentification(user.Id.ToString(), ip, true);
AuthenticationService.SignIn(user.Id.ToString(), createPersistentCookie);
return(true);
}
if (ValidationHelper.IsCorrectLogin(userIdentificator))
{
var login = userIdentificator;
var user = SecurityService.LoginUserValidation(login, password);
if (user == null)
{
return(false);
}
MonitoringService.AsynchLogUserAuthentification(user.Id.ToString(), ip, true);
AuthenticationService.SignIn(user.Id.ToString(), createPersistentCookie);
return(true);
}
MonitoringService.AsynchLogUserAuthentification(null, ip, false);
throw new ArgumentException();
}
public async Task IsServicesOKAsync()
{
//Given
var srv = new MonitoringService();
var ServicesOk = new Service()
{
Uri = "http://www.sdpv.com/",
};
//When
await srv.Test(ServicesOk);
//Then
}
public async void ListAsync_Success()
{
// Arrange
var _monitoringRepositoryMock = MonitoringTestBase.ListAsync();
var _vehicleTypeRepositoryMock = VehicleTypeTestBase.ListAsync();
var _service = new MonitoringService(_mapper, _monitoringRepositoryMock.Object, _vehicleTypeRepositoryMock.Object);
// Act
var result = await _service.ListAsync();
//Assert
_monitoringRepositoryMock.Verify(_ => _.ListAsync(), Times.Once);
Assert.NotNull(result);
}
public override async Task <EmbededViewResult> Execute()
{
var logsService = new MonitoringService();
Model = new RequestModel();
if (this.HttpContext.Request.Query.ContainsKey("TraceIdentifier"))
{
var traceIdentifier = this.HttpContext.Request.Query["TraceIdentifier"].ToString().Trim();
Model.Request = new MonitoringService().AllRequests.FirstOrDefault(x => x.TraceIdentifier == traceIdentifier);
Model.TraceIdentifier = traceIdentifier;
}
return(await View());
}
public async Task TestGoogle_Ok()
{
// Given
var srv = new MonitoringService();
var google = new Service()
{
Uri = "https://www.google.com"
};
// When
await srv.Test(google);
// Then
}
public async Task ShouldPollProviders()
{
var statuses = new List <MachineStatus>();
var dataContext = new Data.UnitTestContext();
var configurationManager = new Mock <IConfigurationManager>();
var providerManager = new MachineProviderManager(machine =>
{
var mock = new Mock <IMachineProvider>();
mock.Setup(x => x.GetStatus(It.IsAny <CancellationToken>()))
.Returns(() =>
{
return(Task.FromResult(new MachineStatus {
MachineId = machine.Id
}));
});
return(mock.Object);
});
var machineCount = 10;
var machines = dataContext.GetRepository <Machine>();
for (int i = 0; i < machineCount; i++)
{
machines.Create(new OctoprintMachine
{
Name = "machine_" + i
});
}
var monitoringService = new MonitoringService(new MachineManager(dataContext, providerManager), configurationManager.Object, providerManager);
monitoringService.StatusUpdate += (sender, args) =>
{
statuses.Add(args.Data);
};
monitoringService.PollProviders();
//The poll providers method is configured to run the task in the back ground and then
//push the updates with an event. So wait a big to let the tasks complete.
await Task.Delay(machineCount * 10);
Assert.AreEqual(machineCount, statuses.Count);
for (var i = 1; i <= machineCount; i++)
{
Assert.True(statuses.Any(status => status.MachineId == i));
}
}
protected override void ConfigureApplicationContainer(TinyIoCContainer container)
{
base.ConfigureApplicationContainer(container);
container.Register((c, n) => _context);
container.Register <Func <Machine, IMachineProvider> >((c, n) => machine =>
{
var machineType = MachineProviderManager.GetProviderType(machine);
var provider = (IMachineProvider)Activator.CreateInstance(machineType, machine);
return(provider);
});
container.Register <IAuthenticationManager, AuthenticationManager>();
container.Register <IAuthorizationManager, AuthorizationManager>();
container.Register <IConfigurationManager, ConfigurationManager>();
container.Register <IUserManager, UserManager>();
container.Register <IMachineManager, MachineManager>();
container.Register <IControlManager, ControlManager>();
container.Register <IMonitoringService>((c, n) =>
{
if (_monitoringService == null)
{
_monitoringService = new MonitoringService(
c.Resolve <MachineManager>(),
c.Resolve <ConfigurationManager>(),
c.Resolve <MachineProviderManager>());
_monitoringService.StatusUpdate += (sender, args) =>
{
c.Resolve <Action <MachineStatus> >()?.Invoke(args.Data);
};
}
return(_monitoringService);
});
container.Register <StatusHubService>();
container.Register <AuthenticationModule>();
container.Register <AuthorizationModule>();
container.Register <ConfigurationModule>();
container.Register <ControlModule>();
container.Register <MachinesModule>();
container.Register <UsersModule>();
_certificateExceptionHandler = container.Resolve <CertificateExceptionHandler>();
_certificateExceptionHandler.Initialize();
}
private void PerformGoToDefinition(MatchResultItem match, ITextBuffer textBuffer)
{
MonitoringService.MonitorCommandGoToStepDefinition(match.Type == MatchResultType.Undefined);
switch (match.Type)
{
case MatchResultType.Undefined:
PerformOfferCopySnippet(match, textBuffer);
break;
case MatchResultType.Defined:
case MatchResultType.Ambiguous:
PerformJump(match);
break;
}
}
public SceneMonitor GetCurrentSceneMonitor(int productLineId, string sceneName)
{
IMonitoringService service = new MonitoringService();
IEnumerable <DataItem> dataItems = service.GetRealtimeDataItems(productLineId, sceneName);
SceneMonitor result = new SceneMonitor
{
Id = DateTime.Now,
Name = sceneName,
DataSet = dataItems
};
//return realtimeRepository.GetCurrent(sceneName);
return(result);
}
public async Task IsServerActive()
{
//Given
var srv = new MonitoringService();
var ServerActive = new Service()
{
Uri = "icmp://80.248.213.227"
};
//When
await srv.Test(ServerActive);
//Then
Assert.IsTrue(ServerActive.IsServerActive());
}
protected void StartService()
{
try
{
MonitoringService service = MonitoringService.Create();
Thread.Sleep(10000);
_tokenSource = new CancellationTokenSource();
service.StartMonitoring(_tokenSource, () => Stop());
EventLog.WriteEntry(MonitoringService.EventTarget, "Service start", EventLogEntryType.Information);
}
catch (Exception e)
{
EventLog.WriteEntry(MonitoringService.EventTarget, "Service start fail:" + e.Message + "\nStack trace:" + e.StackTrace, EventLogEntryType.Error);
//File.WriteAllText(path, e.Message);
}
}
public AutoDeployBackgroundService(
[NotNull] IDeploymentTargetReadService deploymentTargetReadService,
[NotNull] MonitoringService monitoringService,
[NotNull] DeploymentWorker deploymentWorker,
[NotNull] AutoDeployConfiguration autoDeployConfiguration,
[NotNull] ILogger logger,
[NotNull] PackageService packageService)
{
_deploymentTargetReadService = deploymentTargetReadService ??
throw new ArgumentNullException(nameof(deploymentTargetReadService));
_monitoringService = monitoringService ?? throw new ArgumentNullException(nameof(monitoringService));
_deploymentWorker = deploymentWorker ?? throw new ArgumentNullException(nameof(deploymentWorker));
_autoDeployConfiguration = autoDeployConfiguration ??
throw new ArgumentNullException(nameof(autoDeployConfiguration));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_packageService = packageService ?? throw new ArgumentNullException(nameof(packageService));
}
public OverseerBootstrapper(IDataContext context)
{
_context = context;
var applicationSettings = _context.GetApplicationSettings();
_monitoringService = new MonitoringService(applicationSettings.Interval, () =>
{
var providerManager = Container.Resolve <PrinterProviderManager>();
return(providerManager.GetPrinterProviders());
});
_monitoringService.StatusUpdate += (sender, args) =>
{
StatusHub.PushStatusUpdate(args.Status);
};
}
public SingleVNode(TFChunkDb db, SingleVNodeSettings vNodeSettings, SingleVNodeAppSettings appSettings)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
db.OpenVerifyAndClean();
_tcpEndPoint = vNodeSettings.ExternalTcpEndPoint;
_httpEndPoint = vNodeSettings.HttpEndPoint;
_outputBus = new InMemoryBus("OutputBus");
_controller = new SingleVNodeController(Bus, _httpEndPoint);
_mainQueue = new QueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(MainQueue);
//MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandler(monitoringInnerBus, "MonitoringQueue", watchSlowMsg: true, slowMsgThresholdMs: 100);
var monitoring = new MonitoringService(monitoringQueue, monitoringRequestBus, db.Config.WriterCheckpoint, appSettings.StatsPeriod);
Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>());
Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring);
//STORAGE SUBSYSTEM
var indexPath = Path.Combine(db.Config.Path, "index");
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(),
new InMemoryCheckpoint(),
maxSizeForMemory: 1000000,
maxTablesPerLevel: 2);
var readIndex = new ReadIndex(_mainQueue,
db,
() => new TFChunkReader(db, db.Config.WriterCheckpoint),
TFConsts.ReadIndexReaderCount,
db.Config.WriterCheckpoint,
tableIndex,
new XXHashUnsafe());
var writer = new TFChunkWriter(db);
var storageWriter = new StorageWriter(_mainQueue, _outputBus, writer, readIndex);
var storageReader = new StorageReader(_mainQueue, _outputBus, readIndex, TFConsts.StorageReaderHandlerCount);
monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader);
var chaser = new TFChunkChaser(db,
db.Config.WriterCheckpoint,
db.Config.GetNamedCheckpoint(Checkpoint.Chaser));
var storageChaser = new StorageChaser(_mainQueue, chaser);
_outputBus.Subscribe<SystemMessage.SystemInit>(storageChaser);
_outputBus.Subscribe<SystemMessage.SystemStart>(storageChaser);
_outputBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser);
var storageScavenger = new StorageScavenger(db, readIndex);
_outputBus.Subscribe<SystemMessage.ScavengeDatabase>(storageScavenger);
//TCP
var tcpService = new TcpService(MainQueue, _tcpEndPoint);
Bus.Subscribe<SystemMessage.SystemInit>(tcpService);
Bus.Subscribe<SystemMessage.SystemStart>(tcpService);
Bus.Subscribe<SystemMessage.BecomeShuttingDown>(tcpService);
//HTTP
HttpService = new HttpService(MainQueue, _httpEndPoint.ToHttpUrl());
Bus.Subscribe<SystemMessage.SystemInit>(HttpService);
Bus.Subscribe<SystemMessage.BecomeShuttingDown>(HttpService);
Bus.Subscribe<HttpMessage.SendOverHttp>(HttpService);
Bus.Subscribe<HttpMessage.UpdatePendingRequests>(HttpService);
HttpService.SetupController(new AdminController(MainQueue));
HttpService.SetupController(new PingController());
HttpService.SetupController(new StatController(monitoringQueue));
HttpService.SetupController(new ReadEventDataController(MainQueue));
HttpService.SetupController(new AtomController(MainQueue));
HttpService.SetupController(new WebSiteController(MainQueue));
//REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(MainQueue, 1, 1);
Bus.Subscribe<ReplicationMessage.EventCommited>(requestManagement);
Bus.Subscribe<ReplicationMessage.CreateStreamRequestCreated>(requestManagement);
Bus.Subscribe<ReplicationMessage.WriteRequestCreated>(requestManagement);
Bus.Subscribe<ReplicationMessage.TransactionStartRequestCreated>(requestManagement);
Bus.Subscribe<ReplicationMessage.TransactionWriteRequestCreated>(requestManagement);
Bus.Subscribe<ReplicationMessage.TransactionCommitRequestCreated>(requestManagement);
Bus.Subscribe<ReplicationMessage.DeleteStreamRequestCreated>(requestManagement);
Bus.Subscribe<ReplicationMessage.RequestCompleted>(requestManagement);
Bus.Subscribe<ReplicationMessage.CommitAck>(requestManagement);
Bus.Subscribe<ReplicationMessage.PrepareAck>(requestManagement);
Bus.Subscribe<ReplicationMessage.WrongExpectedVersion>(requestManagement);
Bus.Subscribe<ReplicationMessage.InvalidTransaction>(requestManagement);
Bus.Subscribe<ReplicationMessage.StreamDeleted>(requestManagement);
Bus.Subscribe<ReplicationMessage.PreparePhaseTimeout>(requestManagement);
Bus.Subscribe<ReplicationMessage.CommitPhaseTimeout>(requestManagement);
var clientService = new ClientService();
Bus.Subscribe<TcpMessage.ConnectionClosed>(clientService);
Bus.Subscribe<ClientMessage.SubscribeToStream>(clientService);
Bus.Subscribe<ClientMessage.UnsubscribeFromStream>(clientService);
Bus.Subscribe<ClientMessage.SubscribeToAllStreams>(clientService);
Bus.Subscribe<ClientMessage.UnsubscribeFromAllStreams>(clientService);
Bus.Subscribe<ReplicationMessage.EventCommited>(clientService);
//TIMER
//var timer = new TimerService(new TimerBasedScheduler(new RealTimer(), new RealTimeProvider()));
TimerService = new TimerService(new ThreadBasedScheduler(new RealTimeProvider()));
Bus.Subscribe<TimerMessage.Schedule>(TimerService);
MainQueue.Start();
monitoringQueue.Start();
}
public SingleVNode(TFChunkDb db,
SingleVNodeSettings vNodeSettings,
bool dbVerifyHashes,
int memTableEntryCount = ESConsts.MemTableEntryCount)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
db.OpenVerifyAndClean(dbVerifyHashes);
_tcpEndPoint = vNodeSettings.ExternalTcpEndPoint;
_httpEndPoint = vNodeSettings.ExternalHttpEndPoint;
_mainBus = new InMemoryBus("MainBus");
_controller = new SingleVNodeController(Bus, _httpEndPoint);
_mainQueue = new QueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(MainQueue);
//MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandler(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
var monitoring = new MonitoringService(monitoringQueue,
monitoringRequestBus,
MainQueue,
db.Config.WriterCheckpoint,
db.Config.Path,
vNodeSettings.StatsPeriod,
_httpEndPoint,
vNodeSettings.StatsStorage);
Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>());
Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.StateChangeMessage, Message>());
Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
Bus.Subscribe(monitoringQueue.WidenFrom<ClientMessage.CreateStreamCompleted, Message>());
monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.StateChangeMessage>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe<ClientMessage.CreateStreamCompleted>(monitoring);
monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring);
//STORAGE SUBSYSTEM
var indexPath = Path.Combine(db.Config.Path, "index");
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(maxSize: memTableEntryCount * 2),
maxSizeForMemory: memTableEntryCount,
maxTablesPerLevel: 2);
var readIndex = new ReadIndex(_mainQueue,
ESConsts.ReadIndexReaderCount,
() => new TFChunkSequentialReader(db, db.Config.WriterCheckpoint, 0),
() => new TFChunkReader(db, db.Config.WriterCheckpoint),
tableIndex,
new XXHashUnsafe(),
new LRUCache<string, StreamCacheInfo>(ESConsts.MetadataCacheCapacity));
var writer = new TFChunkWriter(db);
new StorageWriterService(_mainQueue, _mainBus, writer, readIndex); // subscribes internally
var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderHandlerCount, db.Config.WriterCheckpoint);
_mainBus.Subscribe<SystemMessage.SystemInit>(storageReader);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageReader);
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(storageReader);
monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader);
var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex, _tcpEndPoint);
_mainBus.Subscribe<SystemMessage.SystemInit>(storageChaser);
_mainBus.Subscribe<SystemMessage.SystemStart>(storageChaser);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser);
var storageScavenger = new StorageScavenger(db, readIndex);
_mainBus.Subscribe<SystemMessage.ScavengeDatabase>(storageScavenger);
// NETWORK SEND
_networkSendService = new NetworkSendService(tcpQueueCount: vNodeSettings.TcpSendingThreads, httpQueueCount: vNodeSettings.HttpSendingThreads);
//TCP
var tcpService = new TcpService(MainQueue, _tcpEndPoint, _networkSendService);
Bus.Subscribe<SystemMessage.SystemInit>(tcpService);
Bus.Subscribe<SystemMessage.SystemStart>(tcpService);
Bus.Subscribe<SystemMessage.BecomeShuttingDown>(tcpService);
//HTTP
_httpService = new HttpService(ServiceAccessibility.Private, MainQueue, vNodeSettings.HttpReceivingThreads, vNodeSettings.HttpPrefixes);
Bus.Subscribe<SystemMessage.SystemInit>(HttpService);
Bus.Subscribe<SystemMessage.BecomeShuttingDown>(HttpService);
Bus.Subscribe<HttpMessage.SendOverHttp>(HttpService);
Bus.Subscribe<HttpMessage.PurgeTimedOutRequests>(HttpService);
HttpService.SetupController(new AdminController(MainQueue));
HttpService.SetupController(new PingController());
HttpService.SetupController(new StatController(monitoringQueue, _networkSendService));
HttpService.SetupController(new ReadEventDataController(MainQueue, _networkSendService));
HttpService.SetupController(new AtomController(MainQueue, _networkSendService));
HttpService.SetupController(new WebSiteController(MainQueue));
//REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(MainQueue, 1, 1);
Bus.Subscribe<StorageMessage.CreateStreamRequestCreated>(requestManagement);
Bus.Subscribe<StorageMessage.WriteRequestCreated>(requestManagement);
Bus.Subscribe<StorageMessage.TransactionStartRequestCreated>(requestManagement);
Bus.Subscribe<StorageMessage.TransactionWriteRequestCreated>(requestManagement);
Bus.Subscribe<StorageMessage.TransactionCommitRequestCreated>(requestManagement);
Bus.Subscribe<StorageMessage.DeleteStreamRequestCreated>(requestManagement);
Bus.Subscribe<StorageMessage.RequestCompleted>(requestManagement);
Bus.Subscribe<StorageMessage.AlreadyCommitted>(requestManagement);
Bus.Subscribe<StorageMessage.CommitAck>(requestManagement);
Bus.Subscribe<StorageMessage.PrepareAck>(requestManagement);
Bus.Subscribe<StorageMessage.WrongExpectedVersion>(requestManagement);
Bus.Subscribe<StorageMessage.InvalidTransaction>(requestManagement);
Bus.Subscribe<StorageMessage.StreamDeleted>(requestManagement);
Bus.Subscribe<StorageMessage.PreparePhaseTimeout>(requestManagement);
Bus.Subscribe<StorageMessage.CommitPhaseTimeout>(requestManagement);
new SubscriptionsService(_mainBus, readIndex); // subcribes internally
//TIMER
_timerService = new TimerService(new ThreadBasedScheduler(new RealTimeProvider()));
Bus.Subscribe<TimerMessage.Schedule>(TimerService);
monitoringQueue.Start();
MainQueue.Start();
}
public SingleVNode(TFChunkDb db,
SingleVNodeSettings vNodeSettings,
bool dbVerifyHashes,
int memTableEntryCount,
params ISubsystem[] subsystems)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
_settings = vNodeSettings;
_mainBus = new InMemoryBus("MainBus");
_controller = new SingleVNodeController(_mainBus, _settings.ExternalHttpEndPoint, db, this);
_mainQueue = new QueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(_mainQueue);
_subsystems = subsystems;
// MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
var monitoring = new MonitoringService(monitoringQueue,
monitoringRequestBus,
_mainQueue,
db.Config.WriterCheckpoint,
db.Config.Path,
vNodeSettings.StatsPeriod,
_settings.ExternalHttpEndPoint,
vNodeSettings.StatsStorage);
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.StateChangeMessage, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShutdown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<ClientMessage.WriteEventsCompleted, Message>());
monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.StateChangeMessage>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.BecomeShutdown>(monitoring);
monitoringInnerBus.Subscribe<ClientMessage.WriteEventsCompleted>(monitoring);
monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring);
var truncPos = db.Config.TruncateCheckpoint.Read();
if (truncPos != -1)
{
Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
var truncator = new TFChunkDbTruncator(db.Config);
truncator.TruncateDb(truncPos);
}
db.Open(dbVerifyHashes);
// STORAGE SUBSYSTEM
var indexPath = Path.Combine(db.Config.Path, "index");
var readerPool = new ObjectPool<ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(db, db.Config.WriterCheckpoint));
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(maxSize: memTableEntryCount * 2),
() => new TFReaderLease(readerPool),
maxSizeForMemory: memTableEntryCount,
maxTablesPerLevel: 2,
inMem: db.Config.InMemDb);
var hasher = new XXHashUnsafe();
var readIndex = new ReadIndex(_mainQueue,
readerPool,
tableIndex,
hasher,
ESConsts.StreamInfoCacheCapacity,
Application.IsDefined(Application.AdditionalCommitChecks),
Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1);
var writer = new TFChunkWriter(db);
var epochManager = new EpochManager(ESConsts.CachedEpochCount,
db.Config.EpochCheckpoint,
writer,
initialReaderCount: 1,
maxReaderCount: 5,
readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint));
epochManager.Init();
var storageWriter = new StorageWriterService(_mainQueue, _mainBus, _settings.MinFlushDelay,
db, writer, readIndex.IndexWriter, epochManager); // subscribes internally
monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageWriter);
var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint);
_mainBus.Subscribe<SystemMessage.SystemInit>(storageReader);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageReader);
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(storageReader);
monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader);
var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex.IndexCommitter, epochManager);
_mainBus.Subscribe<SystemMessage.SystemInit>(storageChaser);
_mainBus.Subscribe<SystemMessage.SystemStart>(storageChaser);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser);
var storageScavenger = new StorageScavenger(db,
tableIndex,
hasher,
readIndex,
Application.IsDefined(Application.AlwaysKeepScavenged),
mergeChunks: false /*!Application.IsDefined(Application.DisableMergeChunks)*/);
// ReSharper disable RedundantTypeArgumentsOfMethod
_mainBus.Subscribe<ClientMessage.ScavengeDatabase>(storageScavenger);
// ReSharper restore RedundantTypeArgumentsOfMethod
// MISC WORKERS
_workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
_workersHandler = new MultiQueuedHandler(
vNodeSettings.WorkerThreads,
queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
string.Format("Worker #{0}", queueNum + 1),
groupName: "Workers",
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200)));
// AUTHENTICATION INFRASTRUCTURE
var passwordHashAlgorithm = new Rfc2898PasswordHashAlgorithm();
var dispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_workersHandler, crossThread: true));
var internalAuthenticationProvider = new InternalAuthenticationProvider(dispatcher, passwordHashAlgorithm, ESConsts.CachedPrincipalCount);
var passwordChangeNotificationReader = new PasswordChangeNotificationReader(_mainQueue, dispatcher);
_mainBus.Subscribe<SystemMessage.SystemStart>(passwordChangeNotificationReader);
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(passwordChangeNotificationReader);
_mainBus.Subscribe(internalAuthenticationProvider);
SubscribeWorkers(bus =>
{
bus.Subscribe(dispatcher.ForwardReader);
bus.Subscribe(dispatcher.BackwardReader);
bus.Subscribe(dispatcher.Writer);
bus.Subscribe(dispatcher.StreamDeleter);
bus.Subscribe(dispatcher);
});
// TCP
{
var tcpService = new TcpService(_mainQueue, _settings.ExternalTcpEndPoint, _workersHandler,
TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
internalAuthenticationProvider, null);
_mainBus.Subscribe<SystemMessage.SystemInit>(tcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(tcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(tcpService);
}
// SECURE TCP
if (vNodeSettings.ExternalSecureTcpEndPoint != null)
{
var secureTcpService = new TcpService(_mainQueue, _settings.ExternalSecureTcpEndPoint, _workersHandler,
TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
internalAuthenticationProvider, _settings.Certificate);
_mainBus.Subscribe<SystemMessage.SystemInit>(secureTcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(secureTcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(secureTcpService);
}
SubscribeWorkers(bus =>
{
var tcpSendService = new TcpSendService();
// ReSharper disable RedundantTypeArgumentsOfMethod
bus.Subscribe<TcpMessage.TcpSend>(tcpSendService);
// ReSharper restore RedundantTypeArgumentsOfMethod
});
// HTTP
var httpAuthenticationProviders = new List<HttpAuthenticationProvider>
{
new BasicHttpAuthenticationProvider(internalAuthenticationProvider),
};
if (_settings.EnableTrustedAuth)
httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());
var httpPipe = new HttpMessagePipe();
var httpSendService = new HttpSendService(httpPipe, forwardRequests: false);
_mainBus.Subscribe<SystemMessage.StateChangeMessage>(httpSendService);
_mainBus.Subscribe(new WideningHandler<HttpMessage.SendOverHttp, Message>(_workersHandler));
SubscribeWorkers(bus =>
{
bus.Subscribe<HttpMessage.HttpSend>(httpSendService);
bus.Subscribe<HttpMessage.HttpSendPart>(httpSendService);
bus.Subscribe<HttpMessage.HttpBeginSend>(httpSendService);
bus.Subscribe<HttpMessage.HttpEndSend>(httpSendService);
bus.Subscribe<HttpMessage.SendOverHttp>(httpSendService);
});
_httpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
_workersHandler, vNodeSettings.HttpPrefixes);
_httpService.SetupController(new AdminController(_mainQueue));
_httpService.SetupController(new PingController());
_httpService.SetupController(new StatController(monitoringQueue, _workersHandler));
_httpService.SetupController(new AtomController(httpSendService, _mainQueue, _workersHandler));
_httpService.SetupController(new GuidController(_mainQueue));
_httpService.SetupController(new UsersController(httpSendService, _mainQueue, _workersHandler));
_mainBus.Subscribe<SystemMessage.SystemInit>(_httpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_httpService);
_mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_httpService);
SubscribeWorkers(bus =>
{
HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
});
// REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(_mainQueue, 1, 1, vNodeSettings.PrepareTimeout, vNodeSettings.CommitTimeout);
_mainBus.Subscribe<SystemMessage.SystemInit>(requestManagement);
_mainBus.Subscribe<ClientMessage.WriteEvents>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionStart>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionWrite>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionCommit>(requestManagement);
_mainBus.Subscribe<ClientMessage.DeleteStream>(requestManagement);
_mainBus.Subscribe<StorageMessage.RequestCompleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.CheckStreamAccessCompleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.AlreadyCommitted>(requestManagement);
_mainBus.Subscribe<StorageMessage.CommitAck>(requestManagement);
_mainBus.Subscribe<StorageMessage.PrepareAck>(requestManagement);
_mainBus.Subscribe<StorageMessage.WrongExpectedVersion>(requestManagement);
_mainBus.Subscribe<StorageMessage.InvalidTransaction>(requestManagement);
_mainBus.Subscribe<StorageMessage.StreamDeleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.RequestManagerTimerTick>(requestManagement);
// SUBSCRIPTIONS
var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
_mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.SystemStart, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<TcpMessage.ConnectionClosed, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.SubscribeToStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.UnsubscribeFromStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.PollStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.CheckPollTimeout, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<StorageMessage.EventCommited, Message>());
var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
subscrBus.Subscribe<SystemMessage.SystemStart>(subscription);
subscrBus.Subscribe<SystemMessage.BecomeShuttingDown>(subscription);
subscrBus.Subscribe<TcpMessage.ConnectionClosed>(subscription);
subscrBus.Subscribe<ClientMessage.SubscribeToStream>(subscription);
subscrBus.Subscribe<ClientMessage.UnsubscribeFromStream>(subscription);
subscrBus.Subscribe<SubscriptionMessage.PollStream>(subscription);
subscrBus.Subscribe<SubscriptionMessage.CheckPollTimeout>(subscription);
subscrBus.Subscribe<StorageMessage.EventCommited>(subscription);
// USER MANAGEMENT
var ioDispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_mainQueue));
_mainBus.Subscribe(ioDispatcher.BackwardReader);
_mainBus.Subscribe(ioDispatcher.ForwardReader);
_mainBus.Subscribe(ioDispatcher.Writer);
_mainBus.Subscribe(ioDispatcher.StreamDeleter);
_mainBus.Subscribe(ioDispatcher);
var userManagement = new UserManagementService(
_mainQueue, ioDispatcher, passwordHashAlgorithm, vNodeSettings.SkipInitializeStandardUsersCheck);
_mainBus.Subscribe<UserManagementMessage.Create>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Update>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Enable>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Disable>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Delete>(userManagement);
_mainBus.Subscribe<UserManagementMessage.ResetPassword>(userManagement);
_mainBus.Subscribe<UserManagementMessage.ChangePassword>(userManagement);
_mainBus.Subscribe<UserManagementMessage.Get>(userManagement);
_mainBus.Subscribe<UserManagementMessage.GetAll>(userManagement);
_mainBus.Subscribe<SystemMessage.BecomeMaster>(userManagement);
// TIMER
_timeProvider = new RealTimeProvider();
_timerService = new TimerService(new ThreadBasedScheduler(_timeProvider));
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(_timerService);
_mainBus.Subscribe<TimerMessage.Schedule>(_timerService);
_workersHandler.Start();
_mainQueue.Start();
monitoringQueue.Start();
subscrQueue.Start();
if (subsystems != null)
foreach (var subsystem in subsystems)
subsystem.Register(db, _mainQueue, _mainBus, _timerService, _timeProvider, httpSendService, new[]{_httpService}, _workersHandler);
}
public ClusterVNode(TFChunkDb db,
ClusterVNodeSettings vNodeSettings,
IGossipSeedSource gossipSeedSource,
bool dbVerifyHashes,
int memTableEntryCount,
params ISubsystem[] subsystems)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
Ensure.NotNull(gossipSeedSource, "gossipSeedSource");
_nodeInfo = vNodeSettings.NodeInfo;
_mainBus = new InMemoryBus("MainBus");
var forwardingProxy = new MessageForwardingProxy();
// MISC WORKERS
_workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
_workersHandler = new MultiQueuedHandler(
vNodeSettings.WorkerThreads,
queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
string.Format("Worker #{0}", queueNum + 1),
groupName: "Workers",
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200)));
_controller = new ClusterVNodeController(_mainBus, _nodeInfo, db, vNodeSettings, this, forwardingProxy);
_mainQueue = new QueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(_mainQueue);
_subsystems = subsystems;
// MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true, TimeSpan.FromMilliseconds(100));
var monitoring = new MonitoringService(monitoringQueue,
monitoringRequestBus,
_mainQueue,
db.Config.WriterCheckpoint,
db.Config.Path,
vNodeSettings.StatsPeriod,
_nodeInfo.ExternalHttp,
vNodeSettings.StatsStorage);
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.StateChangeMessage, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShutdown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom<ClientMessage.WriteEventsCompleted, Message>());
monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.StateChangeMessage>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.BecomeShutdown>(monitoring);
monitoringInnerBus.Subscribe<ClientMessage.WriteEventsCompleted>(monitoring);
monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring);
var truncPos = db.Config.TruncateCheckpoint.Read();
if (truncPos != -1)
{
Log.Info("Truncate checkpoint is present. Truncate: {0} (0x{0:X}), Writer: {1} (0x{1:X}), Chaser: {2} (0x{2:X}), Epoch: {3} (0x{3:X})",
truncPos, db.Config.WriterCheckpoint.Read(), db.Config.ChaserCheckpoint.Read(), db.Config.EpochCheckpoint.Read());
var truncator = new TFChunkDbTruncator(db.Config);
truncator.TruncateDb(truncPos);
}
// STORAGE SUBSYSTEM
db.Open(dbVerifyHashes);
var indexPath = Path.Combine(db.Config.Path, "index");
var readerPool = new ObjectPool<ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, ESConsts.PTableMaxReaderCount,
() => new TFChunkReader(db, db.Config.WriterCheckpoint));
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(maxSize: memTableEntryCount * 2),
() => new TFReaderLease(readerPool),
maxSizeForMemory: memTableEntryCount,
maxTablesPerLevel: 2,
inMem: db.Config.InMemDb);
var hash = new XXHashUnsafe();
var readIndex = new ReadIndex(_mainQueue,
readerPool,
tableIndex,
hash,
ESConsts.StreamInfoCacheCapacity,
Application.IsDefined(Application.AdditionalCommitChecks),
Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1);
var writer = new TFChunkWriter(db);
var epochManager = new EpochManager(ESConsts.CachedEpochCount,
db.Config.EpochCheckpoint,
writer,
initialReaderCount: 1,
maxReaderCount: 5,
readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint));
epochManager.Init();
var storageWriter = new ClusterStorageWriterService(_mainQueue, _mainBus, vNodeSettings.MinFlushDelay,
db, writer, readIndex.IndexWriter, epochManager,
() => readIndex.LastCommitPosition); // subscribes internally
monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageWriter);
var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex, ESConsts.StorageReaderThreadCount, db.Config.WriterCheckpoint);
_mainBus.Subscribe<SystemMessage.SystemInit>(storageReader);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageReader);
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(storageReader);
monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader);
var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, readIndex.IndexCommitter, epochManager);
_mainBus.Subscribe<SystemMessage.SystemInit>(storageChaser);
_mainBus.Subscribe<SystemMessage.SystemStart>(storageChaser);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser);
var storageScavenger = new StorageScavenger(db, tableIndex, hash, readIndex,
Application.IsDefined(Application.AlwaysKeepScavenged),
mergeChunks: !vNodeSettings.DisableScavengeMerging);
// ReSharper disable RedundantTypeArgumentsOfMethod
_mainBus.Subscribe<ClientMessage.ScavengeDatabase>(storageScavenger);
// ReSharper restore RedundantTypeArgumentsOfMethod
// AUTHENTICATION INFRASTRUCTURE - delegate to plugins
var authenticationProvider = vNodeSettings.AuthenticationProviderFactory.BuildAuthenticationProvider(_mainQueue, _mainBus, _workersHandler, _workerBuses);
Ensure.NotNull(authenticationProvider, "authenticationProvider");
{
// EXTERNAL TCP
var extTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalTcp, _workersHandler,
TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
authenticationProvider, null);
_mainBus.Subscribe<SystemMessage.SystemInit>(extTcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(extTcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(extTcpService);
// EXTERNAL SECURE TCP
if (_nodeInfo.ExternalSecureTcp != null)
{
var extSecTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalSecureTcp, _workersHandler,
TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
ESConsts.ExternalHeartbeatInterval, ESConsts.ExternalHeartbeatTimeout,
authenticationProvider, vNodeSettings.Certificate);
_mainBus.Subscribe<SystemMessage.SystemInit>(extSecTcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(extSecTcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(extSecTcpService);
}
// INTERNAL TCP
var intTcpService = new TcpService(_mainQueue, _nodeInfo.InternalTcp, _workersHandler,
TcpServiceType.Internal, TcpSecurityType.Normal,
new InternalTcpDispatcher(),
ESConsts.InternalHeartbeatInterval, ESConsts.InternalHeartbeatTimeout,
authenticationProvider, null);
_mainBus.Subscribe<SystemMessage.SystemInit>(intTcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(intTcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(intTcpService);
// INTERNAL SECURE TCP
if (_nodeInfo.InternalSecureTcp != null)
{
var intSecTcpService = new TcpService(_mainQueue, _nodeInfo.InternalSecureTcp, _workersHandler,
TcpServiceType.Internal, TcpSecurityType.Secure,
new InternalTcpDispatcher(),
ESConsts.InternalHeartbeatInterval, ESConsts.InternalHeartbeatTimeout,
authenticationProvider, vNodeSettings.Certificate);
_mainBus.Subscribe<SystemMessage.SystemInit>(intSecTcpService);
_mainBus.Subscribe<SystemMessage.SystemStart>(intSecTcpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(intSecTcpService);
}
}
SubscribeWorkers(bus =>
{
var tcpSendService = new TcpSendService();
// ReSharper disable RedundantTypeArgumentsOfMethod
bus.Subscribe<TcpMessage.TcpSend>(tcpSendService);
// ReSharper restore RedundantTypeArgumentsOfMethod
});
var httpAuthenticationProviders = new List<HttpAuthenticationProvider>
{
new BasicHttpAuthenticationProvider(authenticationProvider),
};
if (vNodeSettings.EnableTrustedAuth)
httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());
var httpPipe = new HttpMessagePipe();
var httpSendService = new HttpSendService(httpPipe, forwardRequests: true);
_mainBus.Subscribe<SystemMessage.StateChangeMessage>(httpSendService);
_mainBus.Subscribe(new WideningHandler<HttpMessage.SendOverHttp, Message>(_workersHandler));
SubscribeWorkers(bus =>
{
bus.Subscribe<HttpMessage.HttpSend>(httpSendService);
bus.Subscribe<HttpMessage.HttpSendPart>(httpSendService);
bus.Subscribe<HttpMessage.HttpBeginSend>(httpSendService);
bus.Subscribe<HttpMessage.HttpEndSend>(httpSendService);
bus.Subscribe<HttpMessage.SendOverHttp>(httpSendService);
});
var adminController = new AdminController(_mainQueue);
var pingController = new PingController();
var statController = new StatController(monitoringQueue, _workersHandler);
var atomController = new AtomController(httpSendService, _mainQueue, _workersHandler);
var gossipController = new GossipController(_mainQueue, _workersHandler);
var electController = new ElectController(_mainQueue);
// HTTP SENDERS
gossipController.SubscribeSenders(httpPipe);
electController.SubscribeSenders(httpPipe);
// EXTERNAL HTTP
_externalHttpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
_workersHandler, vNodeSettings.HttpPrefixes);
_externalHttpService.SetupController(adminController);
_externalHttpService.SetupController(pingController);
_externalHttpService.SetupController(statController);
_externalHttpService.SetupController(atomController);
_externalHttpService.SetupController(gossipController);
_mainBus.Subscribe<SystemMessage.SystemInit>(_externalHttpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_externalHttpService);
_mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_externalHttpService);
// INTERNAL HTTP
_internalHttpService = new HttpService(ServiceAccessibility.Private, _mainQueue, new TrieUriRouter(),
_workersHandler, _nodeInfo.InternalHttp.ToHttpUrl());
_internalHttpService.SetupController(adminController);
_internalHttpService.SetupController(pingController);
_internalHttpService.SetupController(statController);
_internalHttpService.SetupController(atomController);
_internalHttpService.SetupController(gossipController);
_internalHttpService.SetupController(electController);
// Authentication plugin HTTP
vNodeSettings.AuthenticationProviderFactory.RegisterHttpControllers(_externalHttpService, _internalHttpService, httpSendService, _mainQueue, _workersHandler);
_mainBus.Subscribe<SystemMessage.SystemInit>(_internalHttpService);
_mainBus.Subscribe<SystemMessage.BecomeShuttingDown>(_internalHttpService);
_mainBus.Subscribe<HttpMessage.PurgeTimedOutRequests>(_internalHttpService);
SubscribeWorkers(bus =>
{
HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
});
// REQUEST FORWARDING
var forwardingService = new RequestForwardingService(_mainQueue, forwardingProxy, TimeSpan.FromSeconds(1));
_mainBus.Subscribe<SystemMessage.SystemStart>(forwardingService);
_mainBus.Subscribe<SystemMessage.RequestForwardingTimerTick>(forwardingService);
_mainBus.Subscribe<ClientMessage.NotHandled>(forwardingService);
_mainBus.Subscribe<ClientMessage.WriteEventsCompleted>(forwardingService);
_mainBus.Subscribe<ClientMessage.TransactionStartCompleted>(forwardingService);
_mainBus.Subscribe<ClientMessage.TransactionWriteCompleted>(forwardingService);
_mainBus.Subscribe<ClientMessage.TransactionCommitCompleted>(forwardingService);
_mainBus.Subscribe<ClientMessage.DeleteStreamCompleted>(forwardingService);
// REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(_mainQueue,
vNodeSettings.PrepareAckCount,
vNodeSettings.CommitAckCount,
vNodeSettings.PrepareTimeout,
vNodeSettings.CommitTimeout);
_mainBus.Subscribe<SystemMessage.SystemInit>(requestManagement);
_mainBus.Subscribe<ClientMessage.WriteEvents>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionStart>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionWrite>(requestManagement);
_mainBus.Subscribe<ClientMessage.TransactionCommit>(requestManagement);
_mainBus.Subscribe<ClientMessage.DeleteStream>(requestManagement);
_mainBus.Subscribe<StorageMessage.RequestCompleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.CheckStreamAccessCompleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.AlreadyCommitted>(requestManagement);
_mainBus.Subscribe<StorageMessage.CommitAck>(requestManagement);
_mainBus.Subscribe<StorageMessage.PrepareAck>(requestManagement);
_mainBus.Subscribe<StorageMessage.WrongExpectedVersion>(requestManagement);
_mainBus.Subscribe<StorageMessage.InvalidTransaction>(requestManagement);
_mainBus.Subscribe<StorageMessage.StreamDeleted>(requestManagement);
_mainBus.Subscribe<StorageMessage.RequestManagerTimerTick>(requestManagement);
// SUBSCRIPTIONS
var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
_mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.SystemStart, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<TcpMessage.ConnectionClosed, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.SubscribeToStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<ClientMessage.UnsubscribeFromStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.PollStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<SubscriptionMessage.CheckPollTimeout, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom<StorageMessage.EventCommited, Message>());
var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
subscrBus.Subscribe<SystemMessage.SystemStart>(subscription);
subscrBus.Subscribe<SystemMessage.BecomeShuttingDown>(subscription);
subscrBus.Subscribe<TcpMessage.ConnectionClosed>(subscription);
subscrBus.Subscribe<ClientMessage.SubscribeToStream>(subscription);
subscrBus.Subscribe<ClientMessage.UnsubscribeFromStream>(subscription);
subscrBus.Subscribe<SubscriptionMessage.PollStream>(subscription);
subscrBus.Subscribe<SubscriptionMessage.CheckPollTimeout>(subscription);
subscrBus.Subscribe<StorageMessage.EventCommited>(subscription);
// TIMER
_timeProvider = new RealTimeProvider();
_timerService = new TimerService(new ThreadBasedScheduler(_timeProvider));
_mainBus.Subscribe<SystemMessage.BecomeShutdown>(_timerService);
_mainBus.Subscribe<TimerMessage.Schedule>(_timerService);
// MASTER REPLICATION
var masterReplicationService = new MasterReplicationService(_mainQueue, _nodeInfo.InstanceId, db, _workersHandler,
epochManager, vNodeSettings.ClusterNodeCount);
_mainBus.Subscribe<SystemMessage.SystemStart>(masterReplicationService);
_mainBus.Subscribe<SystemMessage.StateChangeMessage>(masterReplicationService);
_mainBus.Subscribe<ReplicationMessage.ReplicaSubscriptionRequest>(masterReplicationService);
_mainBus.Subscribe<ReplicationMessage.ReplicaLogPositionAck>(masterReplicationService);
// REPLICA REPLICATION
var replicaService = new ReplicaService(_mainQueue, db, epochManager, _workersHandler, authenticationProvider,
_nodeInfo, vNodeSettings.UseSsl, vNodeSettings.SslTargetHost, vNodeSettings.SslValidateServer);
_mainBus.Subscribe<SystemMessage.StateChangeMessage>(replicaService);
_mainBus.Subscribe<ReplicationMessage.ReconnectToMaster>(replicaService);
_mainBus.Subscribe<ReplicationMessage.SubscribeToMaster>(replicaService);
_mainBus.Subscribe<ReplicationMessage.AckLogPosition>(replicaService);
_mainBus.Subscribe<StorageMessage.PrepareAck>(replicaService);
_mainBus.Subscribe<StorageMessage.CommitAck>(replicaService);
_mainBus.Subscribe<ClientMessage.TcpForwardMessage>(replicaService);
// ELECTIONS
var electionsService = new ElectionsService(_mainQueue, _nodeInfo, vNodeSettings.ClusterNodeCount,
db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint,
epochManager, () => readIndex.LastCommitPosition, vNodeSettings.NodePriority);
electionsService.SubscribeMessages(_mainBus);
// GOSSIP
var gossip = new NodeGossipService(_mainQueue, gossipSeedSource, _nodeInfo, db.Config.WriterCheckpoint,
db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastCommitPosition,
vNodeSettings.NodePriority);
_mainBus.Subscribe<SystemMessage.SystemInit>(gossip);
_mainBus.Subscribe<GossipMessage.RetrieveGossipSeedSources>(gossip);
_mainBus.Subscribe<GossipMessage.GotGossipSeedSources>(gossip);
_mainBus.Subscribe<GossipMessage.Gossip>(gossip);
_mainBus.Subscribe<GossipMessage.GossipReceived>(gossip);
_mainBus.Subscribe<SystemMessage.StateChangeMessage>(gossip);
_mainBus.Subscribe<GossipMessage.GossipSendFailed>(gossip);
_mainBus.Subscribe<SystemMessage.VNodeConnectionEstablished>(gossip);
_mainBus.Subscribe<SystemMessage.VNodeConnectionLost>(gossip);
_workersHandler.Start();
_mainQueue.Start();
monitoringQueue.Start();
subscrQueue.Start();
if (subsystems != null)
{
foreach (var subsystem in subsystems)
{
subsystem.Register(db, _mainQueue, _mainBus, _timerService, _timeProvider,
httpSendService, new[] { _internalHttpService, _externalHttpService }, _workersHandler);
}
}
}