/// <summary> /// Main run method for performing our calculations and storing data. /// </summary> /// <param name="cancelToken">The cancel token.</param> public void Run(CancellationTokenSource cancelToken) { string fullFileName = AbsoluteFileName; if (fullFileName != null && File.Exists(fullFileName)) { // Open the file FileStream stream = File.Open(fullFileName, FileMode.Open, FileAccess.Read, FileShare.Read); // Create a reader. IExcelDataReader excelReader; if (Path.GetExtension(fullFileName).Equals(".xls", StringComparison.CurrentCultureIgnoreCase)) { throw new Exception("EXCEL file must be in .xlsx format. Filename: " + fullFileName); } else { // Reading from a OpenXml Excel file (2007 format; *.xlsx) excelReader = ExcelReaderFactory.CreateOpenXmlReader(stream); } // Read all sheets from the EXCEL file as a data set // excelReader.IsFirstRowAsColumnNames = true; DataSet dataSet = excelReader.AsDataSet(new ExcelDataSetConfiguration() { ConfigureDataTable = (_) => new ExcelDataTableConfiguration() { UseHeaderRow = true } }); // Write all sheets that are specified in 'SheetNames' to the data store foreach (DataTable table in dataSet.Tables) { bool keep = StringUtilities.IndexOfCaseInsensitive(this.SheetNames, table.TableName) != -1; if (keep) { TruncateDates(table); StorageWriter.WriteTable(table); } } // Close the reader and free resources. excelReader.Close(); } else { throw new ApsimXException(this, string.Format("Unable to read Excel file '{0}': file does not exist.", fullFileName)); } }
public override void ToXml(StorageWriter storage, FileParsingContext parsingContext) { storage.Properties.SetInt32("width", Size.Width); storage.Properties.SetInt32("height", Size.Height); storage.Properties.SetInt32("points", Points); storage.Properties.SetString("name", Name); var writer = storage.GetXmlWriter(); ChartStyle.ToXml(writer); PropertiesSource.ToXml(writer); Grid.ToXml(writer); XAxis.ToXml(writer); YAxis.ToXml(writer); Y2Axis.ToXml(writer); Title.ToXml(writer); XYLabel.ToXml(writer); Legend.ToXml(writer); SeriesToXml(writer); }
public SingleVNode(TFChunkDb db, SingleVNodeSettings vNodeSettings, SingleVNodeAppSettings appSettings) { Ensure.NotNull(db, "db"); Ensure.NotNull(vNodeSettings, "vNodeSettings"); db.OpenVerifyAndClean(); _tcpEndPoint = vNodeSettings.ExternalTcpEndPoint; _httpEndPoint = vNodeSettings.HttpEndPoint; _outputBus = new InMemoryBus("OutputBus"); _controller = new SingleVNodeController(Bus, _httpEndPoint); _mainQueue = new QueuedHandler(_controller, "MainQueue"); _controller.SetMainQueue(MainQueue); //MONITORING var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false); var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false); var monitoringQueue = new QueuedHandler(monitoringInnerBus, "MonitoringQueue", watchSlowMsg: true, slowMsgThresholdMs: 100); var monitoring = new MonitoringService(monitoringQueue, monitoringRequestBus, db.Config.WriterCheckpoint, appSettings.StatsPeriod); Bus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>()); Bus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>()); monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring); monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring); monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring); //STORAGE SUBSYSTEM var indexPath = Path.Combine(db.Config.Path, "index"); var tableIndex = new TableIndex(indexPath, () => new HashListMemTable(), new InMemoryCheckpoint(), maxSizeForMemory: 1000000, maxTablesPerLevel: 2); var readIndex = new ReadIndex(_mainQueue, pos => new TFChunkChaser(db, db.Config.WriterCheckpoint, new InMemoryCheckpoint(pos)), () => new TFChunkReader(db, db.Config.WriterCheckpoint), TFConsts.ReadIndexReaderCount, tableIndex, new XXHashUnsafe()); var writer = new TFChunkWriter(db); var storageWriter = new StorageWriter(_mainQueue, _outputBus, writer, readIndex); var storageReader = new StorageReader(_mainQueue, _outputBus, readIndex, TFConsts.StorageReaderHandlerCount); monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader); var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.GetNamedCheckpoint(Checkpoint.Chaser)); var storageChaser = new StorageChaser(_mainQueue, chaser); _outputBus.Subscribe <SystemMessage.SystemInit>(storageChaser); _outputBus.Subscribe <SystemMessage.SystemStart>(storageChaser); _outputBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser); var storageScavenger = new StorageScavenger(db, readIndex); _outputBus.Subscribe <SystemMessage.ScavengeDatabase>(storageScavenger); //TCP var tcpService = new TcpService(MainQueue, _tcpEndPoint); Bus.Subscribe <SystemMessage.SystemInit>(tcpService); Bus.Subscribe <SystemMessage.SystemStart>(tcpService); Bus.Subscribe <SystemMessage.BecomeShuttingDown>(tcpService); //HTTP HttpService = new HttpService(MainQueue, vNodeSettings.HttpPrefixes); Bus.Subscribe <SystemMessage.SystemInit>(HttpService); Bus.Subscribe <SystemMessage.BecomeShuttingDown>(HttpService); Bus.Subscribe <HttpMessage.SendOverHttp>(HttpService); Bus.Subscribe <HttpMessage.UpdatePendingRequests>(HttpService); HttpService.SetupController(new AdminController(MainQueue)); HttpService.SetupController(new PingController()); HttpService.SetupController(new StatController(monitoringQueue)); HttpService.SetupController(new ReadEventDataController(MainQueue)); HttpService.SetupController(new AtomController(MainQueue)); HttpService.SetupController(new WebSiteController(MainQueue)); //REQUEST MANAGEMENT var requestManagement = new RequestManagementService(MainQueue, 1, 1); Bus.Subscribe <ReplicationMessage.EventCommited>(requestManagement); Bus.Subscribe <ReplicationMessage.CreateStreamRequestCreated>(requestManagement); Bus.Subscribe <ReplicationMessage.WriteRequestCreated>(requestManagement); Bus.Subscribe <ReplicationMessage.TransactionStartRequestCreated>(requestManagement); Bus.Subscribe <ReplicationMessage.TransactionWriteRequestCreated>(requestManagement); Bus.Subscribe <ReplicationMessage.TransactionCommitRequestCreated>(requestManagement); Bus.Subscribe <ReplicationMessage.DeleteStreamRequestCreated>(requestManagement); Bus.Subscribe <ReplicationMessage.RequestCompleted>(requestManagement); Bus.Subscribe <ReplicationMessage.CommitAck>(requestManagement); Bus.Subscribe <ReplicationMessage.PrepareAck>(requestManagement); Bus.Subscribe <ReplicationMessage.WrongExpectedVersion>(requestManagement); Bus.Subscribe <ReplicationMessage.InvalidTransaction>(requestManagement); Bus.Subscribe <ReplicationMessage.StreamDeleted>(requestManagement); Bus.Subscribe <ReplicationMessage.PreparePhaseTimeout>(requestManagement); Bus.Subscribe <ReplicationMessage.CommitPhaseTimeout>(requestManagement); var clientService = new ClientService(); Bus.Subscribe <TcpMessage.ConnectionClosed>(clientService); Bus.Subscribe <ClientMessage.SubscribeToStream>(clientService); Bus.Subscribe <ClientMessage.UnsubscribeFromStream>(clientService); Bus.Subscribe <ClientMessage.SubscribeToAllStreams>(clientService); Bus.Subscribe <ClientMessage.UnsubscribeFromAllStreams>(clientService); Bus.Subscribe <ReplicationMessage.EventCommited>(clientService); //TIMER //var timer = new TimerService(new TimerBasedScheduler(new RealTimer(), new RealTimeProvider())); TimerService = new TimerService(new ThreadBasedScheduler(new RealTimeProvider())); Bus.Subscribe <TimerMessage.Schedule>(TimerService); MainQueue.Start(); monitoringQueue.Start(); }
public SingleVNode(TFChunkDb db, SingleVNodeSettings vNodeSettings, SingleVNodeAppSettings appSettings) { Ensure.NotNull(db, "db"); Ensure.NotNull(vNodeSettings, "vNodeSettings"); db.OpenVerifyAndClean(); _tcpEndPoint = vNodeSettings.ExternalTcpEndPoint; _httpEndPoint = vNodeSettings.HttpEndPoint; _outputBus = new InMemoryBus("OutputBus"); _controller = new SingleVNodeController(Bus, _httpEndPoint); _mainQueue = new QueuedHandler(_controller, "MainQueue"); _controller.SetMainQueue(MainQueue); //MONITORING var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false); var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false); var monitoringQueue = new QueuedHandler(monitoringInnerBus, "MonitoringQueue", watchSlowMsg: true, slowMsgThresholdMs: 100); var monitoring = new MonitoringService(monitoringQueue, monitoringRequestBus, db.Config.WriterCheckpoint, appSettings.StatsPeriod); Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>()); Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>()); monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring); monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring); monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring); //STORAGE SUBSYSTEM var indexPath = Path.Combine(db.Config.Path, "index"); var tableIndex = new TableIndex(indexPath, () => new HashListMemTable(), new InMemoryCheckpoint(), maxSizeForMemory: 1000000, maxTablesPerLevel: 2); var readIndex = new ReadIndex(_mainQueue, db, () => new TFChunkReader(db, db.Config.WriterCheckpoint), TFConsts.ReadIndexReaderCount, db.Config.WriterCheckpoint, tableIndex, new XXHashUnsafe()); var writer = new TFChunkWriter(db); var storageWriter = new StorageWriter(_mainQueue, _outputBus, writer, readIndex); var storageReader = new StorageReader(_mainQueue, _outputBus, readIndex, TFConsts.StorageReaderHandlerCount); monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader); var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.GetNamedCheckpoint(Checkpoint.Chaser)); var storageChaser = new StorageChaser(_mainQueue, chaser); _outputBus.Subscribe<SystemMessage.SystemInit>(storageChaser); _outputBus.Subscribe<SystemMessage.SystemStart>(storageChaser); _outputBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser); var storageScavenger = new StorageScavenger(db, readIndex); _outputBus.Subscribe<SystemMessage.ScavengeDatabase>(storageScavenger); //TCP var tcpService = new TcpService(MainQueue, _tcpEndPoint); Bus.Subscribe<SystemMessage.SystemInit>(tcpService); Bus.Subscribe<SystemMessage.SystemStart>(tcpService); Bus.Subscribe<SystemMessage.BecomeShuttingDown>(tcpService); //HTTP HttpService = new HttpService(MainQueue, _httpEndPoint.ToHttpUrl()); Bus.Subscribe<SystemMessage.SystemInit>(HttpService); Bus.Subscribe<SystemMessage.BecomeShuttingDown>(HttpService); Bus.Subscribe<HttpMessage.SendOverHttp>(HttpService); Bus.Subscribe<HttpMessage.UpdatePendingRequests>(HttpService); HttpService.SetupController(new AdminController(MainQueue)); HttpService.SetupController(new PingController()); HttpService.SetupController(new StatController(monitoringQueue)); HttpService.SetupController(new ReadEventDataController(MainQueue)); HttpService.SetupController(new AtomController(MainQueue)); HttpService.SetupController(new WebSiteController(MainQueue)); //REQUEST MANAGEMENT var requestManagement = new RequestManagementService(MainQueue, 1, 1); Bus.Subscribe<ReplicationMessage.EventCommited>(requestManagement); Bus.Subscribe<ReplicationMessage.CreateStreamRequestCreated>(requestManagement); Bus.Subscribe<ReplicationMessage.WriteRequestCreated>(requestManagement); Bus.Subscribe<ReplicationMessage.TransactionStartRequestCreated>(requestManagement); Bus.Subscribe<ReplicationMessage.TransactionWriteRequestCreated>(requestManagement); Bus.Subscribe<ReplicationMessage.TransactionCommitRequestCreated>(requestManagement); Bus.Subscribe<ReplicationMessage.DeleteStreamRequestCreated>(requestManagement); Bus.Subscribe<ReplicationMessage.RequestCompleted>(requestManagement); Bus.Subscribe<ReplicationMessage.CommitAck>(requestManagement); Bus.Subscribe<ReplicationMessage.PrepareAck>(requestManagement); Bus.Subscribe<ReplicationMessage.WrongExpectedVersion>(requestManagement); Bus.Subscribe<ReplicationMessage.InvalidTransaction>(requestManagement); Bus.Subscribe<ReplicationMessage.StreamDeleted>(requestManagement); Bus.Subscribe<ReplicationMessage.PreparePhaseTimeout>(requestManagement); Bus.Subscribe<ReplicationMessage.CommitPhaseTimeout>(requestManagement); var clientService = new ClientService(); Bus.Subscribe<TcpMessage.ConnectionClosed>(clientService); Bus.Subscribe<ClientMessage.SubscribeToStream>(clientService); Bus.Subscribe<ClientMessage.UnsubscribeFromStream>(clientService); Bus.Subscribe<ClientMessage.SubscribeToAllStreams>(clientService); Bus.Subscribe<ClientMessage.UnsubscribeFromAllStreams>(clientService); Bus.Subscribe<ReplicationMessage.EventCommited>(clientService); //TIMER //var timer = new TimerService(new TimerBasedScheduler(new RealTimer(), new RealTimeProvider())); TimerService = new TimerService(new ThreadBasedScheduler(new RealTimeProvider())); Bus.Subscribe<TimerMessage.Schedule>(TimerService); MainQueue.Start(); monitoringQueue.Start(); }
internal ClientProtocol(Connection dtc) : base(dtc) { _packet = new PacketHeader(); _output = new StorageWriter(null); _source = new StorageReader(null); }
public void WriteTo(StorageWriter output) { output.WriteBytes(_data, 0, _data.Length); }
public JSDataWriter(StorageWriter sw, bool decorate = false) { _storage = sw; Decorate = decorate; }
public PBDataWriter(byte[] bts, int pos, int count) { _storage = new StorageWriter(bts, pos, count); }
public PBDataWriter(StorageWriter sw) { _storage = sw; }