/// <summary>
/// Main run method for performing our calculations and storing data.
/// </summary>
/// <param name="cancelToken">The cancel token.</param>
public void Run(CancellationTokenSource cancelToken)
{
string fullFileName = AbsoluteFileName;
if (fullFileName != null && File.Exists(fullFileName))
{
// Open the file
FileStream stream = File.Open(fullFileName, FileMode.Open, FileAccess.Read, FileShare.Read);
// Create a reader.
IExcelDataReader excelReader;
if (Path.GetExtension(fullFileName).Equals(".xls", StringComparison.CurrentCultureIgnoreCase))
{
throw new Exception("EXCEL file must be in .xlsx format. Filename: " + fullFileName);
}
else
{
// Reading from a OpenXml Excel file (2007 format; *.xlsx)
excelReader = ExcelReaderFactory.CreateOpenXmlReader(stream);
}
// Read all sheets from the EXCEL file as a data set
// excelReader.IsFirstRowAsColumnNames = true;
DataSet dataSet = excelReader.AsDataSet(new ExcelDataSetConfiguration()
{
ConfigureDataTable = (_) => new ExcelDataTableConfiguration()
{
UseHeaderRow = true
}
});
// Write all sheets that are specified in 'SheetNames' to the data store
foreach (DataTable table in dataSet.Tables)
{
bool keep = StringUtilities.IndexOfCaseInsensitive(this.SheetNames, table.TableName) != -1;
if (keep)
{
TruncateDates(table);
StorageWriter.WriteTable(table);
}
}
// Close the reader and free resources.
excelReader.Close();
}
else
{
throw new ApsimXException(this, string.Format("Unable to read Excel file '{0}': file does not exist.", fullFileName));
}
}
public override void ToXml(StorageWriter storage, FileParsingContext parsingContext)
{
storage.Properties.SetInt32("width", Size.Width);
storage.Properties.SetInt32("height", Size.Height);
storage.Properties.SetInt32("points", Points);
storage.Properties.SetString("name", Name);
var writer = storage.GetXmlWriter();
ChartStyle.ToXml(writer);
PropertiesSource.ToXml(writer);
Grid.ToXml(writer);
XAxis.ToXml(writer);
YAxis.ToXml(writer);
Y2Axis.ToXml(writer);
Title.ToXml(writer);
XYLabel.ToXml(writer);
Legend.ToXml(writer);
SeriesToXml(writer);
}
public SingleVNode(TFChunkDb db, SingleVNodeSettings vNodeSettings, SingleVNodeAppSettings appSettings)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
db.OpenVerifyAndClean();
_tcpEndPoint = vNodeSettings.ExternalTcpEndPoint;
_httpEndPoint = vNodeSettings.HttpEndPoint;
_outputBus = new InMemoryBus("OutputBus");
_controller = new SingleVNodeController(Bus, _httpEndPoint);
_mainQueue = new QueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(MainQueue);
//MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandler(monitoringInnerBus, "MonitoringQueue", watchSlowMsg: true, slowMsgThresholdMs: 100);
var monitoring = new MonitoringService(monitoringQueue, monitoringRequestBus, db.Config.WriterCheckpoint, appSettings.StatsPeriod);
Bus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>());
Bus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring);
//STORAGE SUBSYSTEM
var indexPath = Path.Combine(db.Config.Path, "index");
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(),
new InMemoryCheckpoint(),
maxSizeForMemory: 1000000,
maxTablesPerLevel: 2);
var readIndex = new ReadIndex(_mainQueue,
pos => new TFChunkChaser(db, db.Config.WriterCheckpoint, new InMemoryCheckpoint(pos)),
() => new TFChunkReader(db, db.Config.WriterCheckpoint),
TFConsts.ReadIndexReaderCount,
tableIndex,
new XXHashUnsafe());
var writer = new TFChunkWriter(db);
var storageWriter = new StorageWriter(_mainQueue, _outputBus, writer, readIndex);
var storageReader = new StorageReader(_mainQueue, _outputBus, readIndex, TFConsts.StorageReaderHandlerCount);
monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader);
var chaser = new TFChunkChaser(db,
db.Config.WriterCheckpoint,
db.Config.GetNamedCheckpoint(Checkpoint.Chaser));
var storageChaser = new StorageChaser(_mainQueue, chaser);
_outputBus.Subscribe <SystemMessage.SystemInit>(storageChaser);
_outputBus.Subscribe <SystemMessage.SystemStart>(storageChaser);
_outputBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser);
var storageScavenger = new StorageScavenger(db, readIndex);
_outputBus.Subscribe <SystemMessage.ScavengeDatabase>(storageScavenger);
//TCP
var tcpService = new TcpService(MainQueue, _tcpEndPoint);
Bus.Subscribe <SystemMessage.SystemInit>(tcpService);
Bus.Subscribe <SystemMessage.SystemStart>(tcpService);
Bus.Subscribe <SystemMessage.BecomeShuttingDown>(tcpService);
//HTTP
HttpService = new HttpService(MainQueue, vNodeSettings.HttpPrefixes);
Bus.Subscribe <SystemMessage.SystemInit>(HttpService);
Bus.Subscribe <SystemMessage.BecomeShuttingDown>(HttpService);
Bus.Subscribe <HttpMessage.SendOverHttp>(HttpService);
Bus.Subscribe <HttpMessage.UpdatePendingRequests>(HttpService);
HttpService.SetupController(new AdminController(MainQueue));
HttpService.SetupController(new PingController());
HttpService.SetupController(new StatController(monitoringQueue));
HttpService.SetupController(new ReadEventDataController(MainQueue));
HttpService.SetupController(new AtomController(MainQueue));
HttpService.SetupController(new WebSiteController(MainQueue));
//REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(MainQueue, 1, 1);
Bus.Subscribe <ReplicationMessage.EventCommited>(requestManagement);
Bus.Subscribe <ReplicationMessage.CreateStreamRequestCreated>(requestManagement);
Bus.Subscribe <ReplicationMessage.WriteRequestCreated>(requestManagement);
Bus.Subscribe <ReplicationMessage.TransactionStartRequestCreated>(requestManagement);
Bus.Subscribe <ReplicationMessage.TransactionWriteRequestCreated>(requestManagement);
Bus.Subscribe <ReplicationMessage.TransactionCommitRequestCreated>(requestManagement);
Bus.Subscribe <ReplicationMessage.DeleteStreamRequestCreated>(requestManagement);
Bus.Subscribe <ReplicationMessage.RequestCompleted>(requestManagement);
Bus.Subscribe <ReplicationMessage.CommitAck>(requestManagement);
Bus.Subscribe <ReplicationMessage.PrepareAck>(requestManagement);
Bus.Subscribe <ReplicationMessage.WrongExpectedVersion>(requestManagement);
Bus.Subscribe <ReplicationMessage.InvalidTransaction>(requestManagement);
Bus.Subscribe <ReplicationMessage.StreamDeleted>(requestManagement);
Bus.Subscribe <ReplicationMessage.PreparePhaseTimeout>(requestManagement);
Bus.Subscribe <ReplicationMessage.CommitPhaseTimeout>(requestManagement);
var clientService = new ClientService();
Bus.Subscribe <TcpMessage.ConnectionClosed>(clientService);
Bus.Subscribe <ClientMessage.SubscribeToStream>(clientService);
Bus.Subscribe <ClientMessage.UnsubscribeFromStream>(clientService);
Bus.Subscribe <ClientMessage.SubscribeToAllStreams>(clientService);
Bus.Subscribe <ClientMessage.UnsubscribeFromAllStreams>(clientService);
Bus.Subscribe <ReplicationMessage.EventCommited>(clientService);
//TIMER
//var timer = new TimerService(new TimerBasedScheduler(new RealTimer(), new RealTimeProvider()));
TimerService = new TimerService(new ThreadBasedScheduler(new RealTimeProvider()));
Bus.Subscribe <TimerMessage.Schedule>(TimerService);
MainQueue.Start();
monitoringQueue.Start();
}
public SingleVNode(TFChunkDb db, SingleVNodeSettings vNodeSettings, SingleVNodeAppSettings appSettings)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
db.OpenVerifyAndClean();
_tcpEndPoint = vNodeSettings.ExternalTcpEndPoint;
_httpEndPoint = vNodeSettings.HttpEndPoint;
_outputBus = new InMemoryBus("OutputBus");
_controller = new SingleVNodeController(Bus, _httpEndPoint);
_mainQueue = new QueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(MainQueue);
//MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandler(monitoringInnerBus, "MonitoringQueue", watchSlowMsg: true, slowMsgThresholdMs: 100);
var monitoring = new MonitoringService(monitoringQueue, monitoringRequestBus, db.Config.WriterCheckpoint, appSettings.StatsPeriod);
Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>());
Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring);
//STORAGE SUBSYSTEM
var indexPath = Path.Combine(db.Config.Path, "index");
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(),
new InMemoryCheckpoint(),
maxSizeForMemory: 1000000,
maxTablesPerLevel: 2);
var readIndex = new ReadIndex(_mainQueue,
db,
() => new TFChunkReader(db, db.Config.WriterCheckpoint),
TFConsts.ReadIndexReaderCount,
db.Config.WriterCheckpoint,
tableIndex,
new XXHashUnsafe());
var writer = new TFChunkWriter(db);
var storageWriter = new StorageWriter(_mainQueue, _outputBus, writer, readIndex);
var storageReader = new StorageReader(_mainQueue, _outputBus, readIndex, TFConsts.StorageReaderHandlerCount);
monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader);
var chaser = new TFChunkChaser(db,
db.Config.WriterCheckpoint,
db.Config.GetNamedCheckpoint(Checkpoint.Chaser));
var storageChaser = new StorageChaser(_mainQueue, chaser);
_outputBus.Subscribe<SystemMessage.SystemInit>(storageChaser);
_outputBus.Subscribe<SystemMessage.SystemStart>(storageChaser);
_outputBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser);
var storageScavenger = new StorageScavenger(db, readIndex);
_outputBus.Subscribe<SystemMessage.ScavengeDatabase>(storageScavenger);
//TCP
var tcpService = new TcpService(MainQueue, _tcpEndPoint);
Bus.Subscribe<SystemMessage.SystemInit>(tcpService);
Bus.Subscribe<SystemMessage.SystemStart>(tcpService);
Bus.Subscribe<SystemMessage.BecomeShuttingDown>(tcpService);
//HTTP
HttpService = new HttpService(MainQueue, _httpEndPoint.ToHttpUrl());
Bus.Subscribe<SystemMessage.SystemInit>(HttpService);
Bus.Subscribe<SystemMessage.BecomeShuttingDown>(HttpService);
Bus.Subscribe<HttpMessage.SendOverHttp>(HttpService);
Bus.Subscribe<HttpMessage.UpdatePendingRequests>(HttpService);
HttpService.SetupController(new AdminController(MainQueue));
HttpService.SetupController(new PingController());
HttpService.SetupController(new StatController(monitoringQueue));
HttpService.SetupController(new ReadEventDataController(MainQueue));
HttpService.SetupController(new AtomController(MainQueue));
HttpService.SetupController(new WebSiteController(MainQueue));
//REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(MainQueue, 1, 1);
Bus.Subscribe<ReplicationMessage.EventCommited>(requestManagement);
Bus.Subscribe<ReplicationMessage.CreateStreamRequestCreated>(requestManagement);
Bus.Subscribe<ReplicationMessage.WriteRequestCreated>(requestManagement);
Bus.Subscribe<ReplicationMessage.TransactionStartRequestCreated>(requestManagement);
Bus.Subscribe<ReplicationMessage.TransactionWriteRequestCreated>(requestManagement);
Bus.Subscribe<ReplicationMessage.TransactionCommitRequestCreated>(requestManagement);
Bus.Subscribe<ReplicationMessage.DeleteStreamRequestCreated>(requestManagement);
Bus.Subscribe<ReplicationMessage.RequestCompleted>(requestManagement);
Bus.Subscribe<ReplicationMessage.CommitAck>(requestManagement);
Bus.Subscribe<ReplicationMessage.PrepareAck>(requestManagement);
Bus.Subscribe<ReplicationMessage.WrongExpectedVersion>(requestManagement);
Bus.Subscribe<ReplicationMessage.InvalidTransaction>(requestManagement);
Bus.Subscribe<ReplicationMessage.StreamDeleted>(requestManagement);
Bus.Subscribe<ReplicationMessage.PreparePhaseTimeout>(requestManagement);
Bus.Subscribe<ReplicationMessage.CommitPhaseTimeout>(requestManagement);
var clientService = new ClientService();
Bus.Subscribe<TcpMessage.ConnectionClosed>(clientService);
Bus.Subscribe<ClientMessage.SubscribeToStream>(clientService);
Bus.Subscribe<ClientMessage.UnsubscribeFromStream>(clientService);
Bus.Subscribe<ClientMessage.SubscribeToAllStreams>(clientService);
Bus.Subscribe<ClientMessage.UnsubscribeFromAllStreams>(clientService);
Bus.Subscribe<ReplicationMessage.EventCommited>(clientService);
//TIMER
//var timer = new TimerService(new TimerBasedScheduler(new RealTimer(), new RealTimeProvider()));
TimerService = new TimerService(new ThreadBasedScheduler(new RealTimeProvider()));
Bus.Subscribe<TimerMessage.Schedule>(TimerService);
MainQueue.Start();
monitoringQueue.Start();
}
internal ClientProtocol(Connection dtc) : base(dtc)
{
_packet = new PacketHeader();
_output = new StorageWriter(null);
_source = new StorageReader(null);
}
public void WriteTo(StorageWriter output) { output.WriteBytes(_data, 0, _data.Length); }
public JSDataWriter(StorageWriter sw, bool decorate = false) { _storage = sw; Decorate = decorate; }
public PBDataWriter(byte[] bts, int pos, int count) { _storage = new StorageWriter(bts, pos, count); }
public PBDataWriter(StorageWriter sw) { _storage = sw; }
