Ejemplo n.º 1
0
        /// <summary>
        /// Main run method for performing our calculations and storing data.
        /// </summary>
        /// <param name="cancelToken">The cancel token.</param>
        public void Run(CancellationTokenSource cancelToken)
        {
            string fullFileName = AbsoluteFileName;

            if (fullFileName != null && File.Exists(fullFileName))
            {
                // Open the file
                FileStream stream = File.Open(fullFileName, FileMode.Open, FileAccess.Read, FileShare.Read);

                // Create a reader.
                IExcelDataReader excelReader;
                if (Path.GetExtension(fullFileName).Equals(".xls", StringComparison.CurrentCultureIgnoreCase))
                {
                    throw new Exception("EXCEL file must be in .xlsx format. Filename: " + fullFileName);
                }
                else
                {
                    // Reading from a OpenXml Excel file (2007 format; *.xlsx)
                    excelReader = ExcelReaderFactory.CreateOpenXmlReader(stream);
                }

                // Read all sheets from the EXCEL file as a data set
                // excelReader.IsFirstRowAsColumnNames = true;
                DataSet dataSet = excelReader.AsDataSet(new ExcelDataSetConfiguration()
                {
                    ConfigureDataTable = (_) => new ExcelDataTableConfiguration()
                    {
                        UseHeaderRow = true
                    }
                });

                // Write all sheets that are specified in 'SheetNames' to the data store
                foreach (DataTable table in dataSet.Tables)
                {
                    bool keep = StringUtilities.IndexOfCaseInsensitive(this.SheetNames, table.TableName) != -1;
                    if (keep)
                    {
                        TruncateDates(table);
                        StorageWriter.WriteTable(table);
                    }
                }

                // Close the reader and free resources.
                excelReader.Close();
            }
            else
            {
                throw new ApsimXException(this, string.Format("Unable to read Excel file '{0}': file does not exist.", fullFileName));
            }
        }
Ejemplo n.º 2
0
        public override void ToXml(StorageWriter storage, FileParsingContext parsingContext)
        {
            storage.Properties.SetInt32("width", Size.Width);
            storage.Properties.SetInt32("height", Size.Height);

            storage.Properties.SetInt32("points", Points);
            storage.Properties.SetString("name", Name);

            var writer = storage.GetXmlWriter();

            ChartStyle.ToXml(writer);
            PropertiesSource.ToXml(writer);
            Grid.ToXml(writer);
            XAxis.ToXml(writer);
            YAxis.ToXml(writer);
            Y2Axis.ToXml(writer);
            Title.ToXml(writer);
            XYLabel.ToXml(writer);
            Legend.ToXml(writer);
            SeriesToXml(writer);
        }
Ejemplo n.º 3
0
        public SingleVNode(TFChunkDb db, SingleVNodeSettings vNodeSettings, SingleVNodeAppSettings appSettings)
        {
            Ensure.NotNull(db, "db");
            Ensure.NotNull(vNodeSettings, "vNodeSettings");

            db.OpenVerifyAndClean();

            _tcpEndPoint  = vNodeSettings.ExternalTcpEndPoint;
            _httpEndPoint = vNodeSettings.HttpEndPoint;

            _outputBus  = new InMemoryBus("OutputBus");
            _controller = new SingleVNodeController(Bus, _httpEndPoint);
            _mainQueue  = new QueuedHandler(_controller, "MainQueue");
            _controller.SetMainQueue(MainQueue);

            //MONITORING
            var monitoringInnerBus   = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
            var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
            var monitoringQueue      = new QueuedHandler(monitoringInnerBus, "MonitoringQueue", watchSlowMsg: true, slowMsgThresholdMs: 100);
            var monitoring           = new MonitoringService(monitoringQueue, monitoringRequestBus, db.Config.WriterCheckpoint, appSettings.StatsPeriod);

            Bus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>());
            Bus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
            monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring);
            monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring);
            monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring);

            //STORAGE SUBSYSTEM
            var indexPath  = Path.Combine(db.Config.Path, "index");
            var tableIndex = new TableIndex(indexPath,
                                            () => new HashListMemTable(),
                                            new InMemoryCheckpoint(),
                                            maxSizeForMemory: 1000000,
                                            maxTablesPerLevel: 2);

            var readIndex = new ReadIndex(_mainQueue,
                                          pos => new TFChunkChaser(db, db.Config.WriterCheckpoint, new InMemoryCheckpoint(pos)),
                                          () => new TFChunkReader(db, db.Config.WriterCheckpoint),
                                          TFConsts.ReadIndexReaderCount,
                                          tableIndex,
                                          new XXHashUnsafe());
            var writer        = new TFChunkWriter(db);
            var storageWriter = new StorageWriter(_mainQueue, _outputBus, writer, readIndex);
            var storageReader = new StorageReader(_mainQueue, _outputBus, readIndex, TFConsts.StorageReaderHandlerCount);

            monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader);

            var chaser = new TFChunkChaser(db,
                                           db.Config.WriterCheckpoint,
                                           db.Config.GetNamedCheckpoint(Checkpoint.Chaser));
            var storageChaser = new StorageChaser(_mainQueue, chaser);

            _outputBus.Subscribe <SystemMessage.SystemInit>(storageChaser);
            _outputBus.Subscribe <SystemMessage.SystemStart>(storageChaser);
            _outputBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser);

            var storageScavenger = new StorageScavenger(db, readIndex);

            _outputBus.Subscribe <SystemMessage.ScavengeDatabase>(storageScavenger);

            //TCP
            var tcpService = new TcpService(MainQueue, _tcpEndPoint);

            Bus.Subscribe <SystemMessage.SystemInit>(tcpService);
            Bus.Subscribe <SystemMessage.SystemStart>(tcpService);
            Bus.Subscribe <SystemMessage.BecomeShuttingDown>(tcpService);

            //HTTP
            HttpService = new HttpService(MainQueue, vNodeSettings.HttpPrefixes);
            Bus.Subscribe <SystemMessage.SystemInit>(HttpService);
            Bus.Subscribe <SystemMessage.BecomeShuttingDown>(HttpService);
            Bus.Subscribe <HttpMessage.SendOverHttp>(HttpService);
            Bus.Subscribe <HttpMessage.UpdatePendingRequests>(HttpService);
            HttpService.SetupController(new AdminController(MainQueue));
            HttpService.SetupController(new PingController());
            HttpService.SetupController(new StatController(monitoringQueue));
            HttpService.SetupController(new ReadEventDataController(MainQueue));
            HttpService.SetupController(new AtomController(MainQueue));
            HttpService.SetupController(new WebSiteController(MainQueue));

            //REQUEST MANAGEMENT
            var requestManagement = new RequestManagementService(MainQueue, 1, 1);

            Bus.Subscribe <ReplicationMessage.EventCommited>(requestManagement);
            Bus.Subscribe <ReplicationMessage.CreateStreamRequestCreated>(requestManagement);
            Bus.Subscribe <ReplicationMessage.WriteRequestCreated>(requestManagement);
            Bus.Subscribe <ReplicationMessage.TransactionStartRequestCreated>(requestManagement);
            Bus.Subscribe <ReplicationMessage.TransactionWriteRequestCreated>(requestManagement);
            Bus.Subscribe <ReplicationMessage.TransactionCommitRequestCreated>(requestManagement);
            Bus.Subscribe <ReplicationMessage.DeleteStreamRequestCreated>(requestManagement);
            Bus.Subscribe <ReplicationMessage.RequestCompleted>(requestManagement);
            Bus.Subscribe <ReplicationMessage.CommitAck>(requestManagement);
            Bus.Subscribe <ReplicationMessage.PrepareAck>(requestManagement);
            Bus.Subscribe <ReplicationMessage.WrongExpectedVersion>(requestManagement);
            Bus.Subscribe <ReplicationMessage.InvalidTransaction>(requestManagement);
            Bus.Subscribe <ReplicationMessage.StreamDeleted>(requestManagement);
            Bus.Subscribe <ReplicationMessage.PreparePhaseTimeout>(requestManagement);
            Bus.Subscribe <ReplicationMessage.CommitPhaseTimeout>(requestManagement);

            var clientService = new ClientService();

            Bus.Subscribe <TcpMessage.ConnectionClosed>(clientService);
            Bus.Subscribe <ClientMessage.SubscribeToStream>(clientService);
            Bus.Subscribe <ClientMessage.UnsubscribeFromStream>(clientService);
            Bus.Subscribe <ClientMessage.SubscribeToAllStreams>(clientService);
            Bus.Subscribe <ClientMessage.UnsubscribeFromAllStreams>(clientService);
            Bus.Subscribe <ReplicationMessage.EventCommited>(clientService);

            //TIMER
            //var timer = new TimerService(new TimerBasedScheduler(new RealTimer(), new RealTimeProvider()));
            TimerService = new TimerService(new ThreadBasedScheduler(new RealTimeProvider()));
            Bus.Subscribe <TimerMessage.Schedule>(TimerService);

            MainQueue.Start();
            monitoringQueue.Start();
        }
Ejemplo n.º 4
0
        public SingleVNode(TFChunkDb db, SingleVNodeSettings vNodeSettings, SingleVNodeAppSettings appSettings)
        {
            Ensure.NotNull(db, "db");
            Ensure.NotNull(vNodeSettings, "vNodeSettings");

            db.OpenVerifyAndClean();

            _tcpEndPoint = vNodeSettings.ExternalTcpEndPoint;
            _httpEndPoint = vNodeSettings.HttpEndPoint;

            _outputBus = new InMemoryBus("OutputBus");
            _controller = new SingleVNodeController(Bus, _httpEndPoint);
            _mainQueue = new QueuedHandler(_controller, "MainQueue");
            _controller.SetMainQueue(MainQueue);

            //MONITORING
            var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
            var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
            var monitoringQueue = new QueuedHandler(monitoringInnerBus, "MonitoringQueue", watchSlowMsg: true, slowMsgThresholdMs: 100);
            var monitoring = new MonitoringService(monitoringQueue, monitoringRequestBus, db.Config.WriterCheckpoint, appSettings.StatsPeriod);
            Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.SystemInit, Message>());
            Bus.Subscribe(monitoringQueue.WidenFrom<SystemMessage.BecomeShuttingDown, Message>());
            monitoringInnerBus.Subscribe<SystemMessage.SystemInit>(monitoring);
            monitoringInnerBus.Subscribe<SystemMessage.BecomeShuttingDown>(monitoring);
            monitoringInnerBus.Subscribe<MonitoringMessage.GetFreshStats>(monitoring);

            //STORAGE SUBSYSTEM
            var indexPath = Path.Combine(db.Config.Path, "index");
            var tableIndex = new TableIndex(indexPath,
                                            () => new HashListMemTable(),
                                            new InMemoryCheckpoint(),
                                            maxSizeForMemory: 1000000,
                                            maxTablesPerLevel: 2);

            var readIndex = new ReadIndex(_mainQueue,
                                          db,
                                          () => new TFChunkReader(db, db.Config.WriterCheckpoint),
                                          TFConsts.ReadIndexReaderCount,
                                          db.Config.WriterCheckpoint,
                                          tableIndex,
                                          new XXHashUnsafe());
            var writer = new TFChunkWriter(db);
            var storageWriter = new StorageWriter(_mainQueue, _outputBus, writer, readIndex);
            var storageReader = new StorageReader(_mainQueue, _outputBus, readIndex, TFConsts.StorageReaderHandlerCount);
            monitoringRequestBus.Subscribe<MonitoringMessage.InternalStatsRequest>(storageReader);

            var chaser = new TFChunkChaser(db,
                                           db.Config.WriterCheckpoint,
                                           db.Config.GetNamedCheckpoint(Checkpoint.Chaser));
            var storageChaser = new StorageChaser(_mainQueue, chaser);
            _outputBus.Subscribe<SystemMessage.SystemInit>(storageChaser);
            _outputBus.Subscribe<SystemMessage.SystemStart>(storageChaser);
            _outputBus.Subscribe<SystemMessage.BecomeShuttingDown>(storageChaser);

            var storageScavenger = new StorageScavenger(db, readIndex);
            _outputBus.Subscribe<SystemMessage.ScavengeDatabase>(storageScavenger);

            //TCP
            var tcpService = new TcpService(MainQueue, _tcpEndPoint);
            Bus.Subscribe<SystemMessage.SystemInit>(tcpService);
            Bus.Subscribe<SystemMessage.SystemStart>(tcpService);
            Bus.Subscribe<SystemMessage.BecomeShuttingDown>(tcpService);

            //HTTP
            HttpService = new HttpService(MainQueue, _httpEndPoint.ToHttpUrl());
            Bus.Subscribe<SystemMessage.SystemInit>(HttpService);
            Bus.Subscribe<SystemMessage.BecomeShuttingDown>(HttpService);
            Bus.Subscribe<HttpMessage.SendOverHttp>(HttpService);
            Bus.Subscribe<HttpMessage.UpdatePendingRequests>(HttpService);
            HttpService.SetupController(new AdminController(MainQueue));
            HttpService.SetupController(new PingController());
            HttpService.SetupController(new StatController(monitoringQueue));
            HttpService.SetupController(new ReadEventDataController(MainQueue));
            HttpService.SetupController(new AtomController(MainQueue));
            HttpService.SetupController(new WebSiteController(MainQueue));

            //REQUEST MANAGEMENT
            var requestManagement = new RequestManagementService(MainQueue, 1, 1);
            Bus.Subscribe<ReplicationMessage.EventCommited>(requestManagement);
            Bus.Subscribe<ReplicationMessage.CreateStreamRequestCreated>(requestManagement);
            Bus.Subscribe<ReplicationMessage.WriteRequestCreated>(requestManagement);
            Bus.Subscribe<ReplicationMessage.TransactionStartRequestCreated>(requestManagement);
            Bus.Subscribe<ReplicationMessage.TransactionWriteRequestCreated>(requestManagement);
            Bus.Subscribe<ReplicationMessage.TransactionCommitRequestCreated>(requestManagement);
            Bus.Subscribe<ReplicationMessage.DeleteStreamRequestCreated>(requestManagement);
            Bus.Subscribe<ReplicationMessage.RequestCompleted>(requestManagement);
            Bus.Subscribe<ReplicationMessage.CommitAck>(requestManagement);
            Bus.Subscribe<ReplicationMessage.PrepareAck>(requestManagement);
            Bus.Subscribe<ReplicationMessage.WrongExpectedVersion>(requestManagement);
            Bus.Subscribe<ReplicationMessage.InvalidTransaction>(requestManagement);
            Bus.Subscribe<ReplicationMessage.StreamDeleted>(requestManagement);
            Bus.Subscribe<ReplicationMessage.PreparePhaseTimeout>(requestManagement);
            Bus.Subscribe<ReplicationMessage.CommitPhaseTimeout>(requestManagement);

            var clientService = new ClientService();
            Bus.Subscribe<TcpMessage.ConnectionClosed>(clientService);
            Bus.Subscribe<ClientMessage.SubscribeToStream>(clientService);
            Bus.Subscribe<ClientMessage.UnsubscribeFromStream>(clientService);
            Bus.Subscribe<ClientMessage.SubscribeToAllStreams>(clientService);
            Bus.Subscribe<ClientMessage.UnsubscribeFromAllStreams>(clientService);
            Bus.Subscribe<ReplicationMessage.EventCommited>(clientService);

            //TIMER
            //var timer = new TimerService(new TimerBasedScheduler(new RealTimer(), new RealTimeProvider()));
            TimerService = new TimerService(new ThreadBasedScheduler(new RealTimeProvider()));
            Bus.Subscribe<TimerMessage.Schedule>(TimerService);

            MainQueue.Start();
            monitoringQueue.Start();
        }
 internal ClientProtocol(Connection dtc) : base(dtc)
 {
     _packet = new PacketHeader();
     _output = new StorageWriter(null);
     _source = new StorageReader(null);
 }
 public void WriteTo(StorageWriter output) { output.WriteBytes(_data, 0, _data.Length); }
Ejemplo n.º 7
0
 public JSDataWriter(StorageWriter sw, bool decorate = false) { _storage = sw; Decorate = decorate; }
Ejemplo n.º 8
0
 public PBDataWriter(byte[] bts, int pos, int count) { _storage = new StorageWriter(bts, pos, count); }
Ejemplo n.º 9
0
 public PBDataWriter(StorageWriter sw) { _storage = sw; }