Exemplo n.º 1
0
        public void WriteFile()
        {
            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            if (File.Exists("c:\\temp\\ArchiveTestFileBig.d2"))
            {
                File.Delete("c:\\temp\\ArchiveTestFileBig.d2");
            }
            //using (var af = ArchiveFile.CreateInMemory(CompressionMethod.TimeSeriesEncoded))
            using (SortedTreeFile af = SortedTreeFile.CreateFile("c:\\temp\\ArchiveTestFileBig.d2"))
                using (SortedTreeTable <HistorianKey, HistorianValue> af2 = af.OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
                {
                    Random r = new Random(3);

                    for (ulong v1 = 1; v1 < 36; v1++)
                    {
                        using (var edit = af2.BeginEdit())
                        {
                            for (ulong v2 = 1; v2 < 86000; v2++)
                            {
                                key.Timestamp = v1 * 2342523;
                                key.PointID   = v2;
                                value.Value1  = (ulong)r.Next();
                                value.Value3  = 0;

                                edit.AddPoint(key, value);
                            }
                            edit.Commit();
                        }
                        af2.Count();
                    }
                    af2.Count();
                }
        }
 /// <summary>
 /// Initializes a new instance of the <see cref="RemoteOutputAdapter"/> class.
 /// </summary>
 public RemoteOutputAdapter()
 {
     m_port = LocalOutputAdapter.DefaultPort;
     m_server = DefaultServer;
     m_key = new HistorianKey();
     m_value = new HistorianValue();
 }
Exemplo n.º 3
0
        public void EnduranceTest()
        {
            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            using (SortedTreeTable <HistorianKey, HistorianValue> target = SortedTreeFile.CreateInMemory().OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
            {
                for (uint x = 0; x < 100; x++)
                {
                    using (SortedTreeTableEditor <HistorianKey, HistorianValue> fileEditor = target.BeginEdit())
                    {
                        for (int y = 0; y < 10; y++)
                        {
                            key.Timestamp = x;
                            key.PointID   = x;
                            value.Value1  = x;
                            value.Value3  = x;
                            fileEditor.AddPoint(key, value);
                            x++;
                        }
                        fileEditor.Commit();
                    }
                    Assert.AreEqual(target.FirstKey.Timestamp, 0);
                    Assert.AreEqual(target.LastKey.Timestamp, x - 1);
                }
            }
        }
Exemplo n.º 4
0
        public void GetDifference()
        {
            StringBuilder sb = new StringBuilder();

            sb.AppendLine("Bucket Number, Count");
            using (SortedTreeFile file = SortedTreeFile.OpenFile(@"C:\Unison\GPA\Codeplex\openHistorian\Main\Build\Output\Release\Applications\openHistorian\Archive\635293583194231435-Stage2-0ef36dcc-4264-498f-b194-01b2043a9231.d2", true))
                using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>())
                    using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader = table.BeginRead())
                        using (SortedTreeScannerBase <HistorianKey, HistorianValue> scan = reader.GetTreeScanner())
                        {
                            HistorianKey   key1  = new HistorianKey();
                            HistorianKey   key2  = new HistorianKey();
                            HistorianValue value = new HistorianValue();

                            int count = 0;
                            scan.SeekToStart();
                            while (scan.Read(key1, value))
                            {
                                count++;
                            }



                            int[] bucket = new int[130];
                            scan.SeekToStart();

                            while (true)
                            {
                                if (!scan.Read(key1, value))
                                {
                                    break;
                                }

                                if (key1.Timestamp == key2.Timestamp)
                                {
                                    int diff = Math.Abs((int)(key1.PointID - key2.PointID));
                                    diff = Math.Min(129, diff);
                                    bucket[diff]++;
                                }

                                if (!scan.Read(key2, value))
                                {
                                    break;
                                }

                                if (key1.Timestamp == key2.Timestamp)
                                {
                                    int diff = Math.Abs((int)(key1.PointID - key2.PointID));
                                    diff = Math.Min(129, diff);
                                    bucket[diff]++;
                                }
                            }

                            for (uint x = 0; x < bucket.Length; x++)
                            {
                                sb.AppendLine(x.ToString() + "," + (bucket[x] / (double)count * 100.0).ToString("0.00"));
                            }
                        }
            Console.WriteLine(sb.ToString());
        }
        public void WriteFile()
        {
            HistorianKey key = new HistorianKey();
            HistorianValue value = new HistorianValue();

            if (File.Exists("c:\\temp\\ArchiveTestFileBig.d2"))
                File.Delete("c:\\temp\\ArchiveTestFileBig.d2");
            //using (var af = ArchiveFile.CreateInMemory(CompressionMethod.TimeSeriesEncoded))
            using (SortedTreeFile af = SortedTreeFile.CreateFile("c:\\temp\\ArchiveTestFileBig.d2"))
            using (SortedTreeTable<HistorianKey, HistorianValue> af2 = af.OpenOrCreateTable<HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
            {
                Random r = new Random(3);

                for (ulong v1 = 1; v1 < 36; v1++)
                {
                    using (var edit = af2.BeginEdit())
                    {
                        for (ulong v2 = 1; v2 < 86000; v2++)
                        {
                            key.Timestamp = v1 * 2342523;
                            key.PointID = v2;
                            value.Value1 = (ulong)r.Next();
                            value.Value3 = 0;

                            edit.AddPoint(key, value);
                        }
                        edit.Commit();
                    }
                    af2.Count();
                }
                af2.Count();
            }
        }
        void ScannerThread()
        {
            int threadId = Interlocked.Increment(ref ThreadNumber);

            try
            {
                //DateTime start = DateTime.FromBinary(Convert.ToDateTime("2/1/2014").Date.Ticks + Convert.ToDateTime("6:00:00PM").TimeOfDay.Ticks).ToUniversalTime();
                while (!StopReading)
                {
                    Stopwatch sw = new Stopwatch();
                    using (var client = new HistorianClient("127.0.0.1", 12345))
                        using (var database = client.GetDatabase <HistorianKey, HistorianValue>(String.Empty))
                        {
                            HistorianKey   key   = new HistorianKey();
                            HistorianValue value = new HistorianValue();

                            sw.Start();
                            var scan = database.Read(0, ulong.MaxValue, new ulong[] { 65, 953, 5562 });
                            while (scan.Read(key, value))
                            {
                                ;
                            }
                            sw.Stop();
                        }

                    //Console.WriteLine("Thread: " + threadId.ToString() + " " + "Run Number: " + myId.ToString() + " " + (pointCount / sw.Elapsed.TotalSeconds / 1000000).ToString());
                }
            }
            catch (Exception ex)
            {
                //Console.WriteLine(ex.ToString());
            }
            Console.WriteLine("Thread: " + threadId.ToString() + " Quit");
        }
        public void TestSmall()
        {
            var key = new HistorianKey();
            var value = new HistorianValue();
            using (var af = SortedTreeFile.CreateInMemory())
            using (var file = af.OpenOrCreateTable<HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
            {
                using (var edit = file.BeginEdit())
                {
                  
                    for (int x = 0; x < 10000000; x++)
                    {
                        key.Timestamp = (ulong)x;
                        edit.AddPoint(key, value);
                    }
                    edit.Commit();
                }

                using (var read = file.BeginRead())
                using (var scan = read.GetTreeScanner())
                {
                    int count = 0;
                    scan.SeekToStart();
                    while (scan.Read(key,value))
                    {
                        count++;
                    }
                    System.Console.WriteLine(count.ToString());
                }
            }
        }
Exemplo n.º 8
0
        private void TestFile(int pageSize, string fileName)
        {
            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            key.Timestamp = 1;

            if (File.Exists(fileName))
            {
                File.Delete(fileName);
            }
            Stopwatch sw = new Stopwatch();

            sw.Start();
            using (SortedTreeTable <HistorianKey, HistorianValue> af = SortedTreeFile.CreateFile(fileName, blockSize: pageSize).OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
                using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = af.BeginEdit())
                {
                    for (uint x = 0; x < 1000000; x++)
                    {
                        key.PointID = x;
                        edit.AddPoint(key, value);
                    }
                    edit.Commit();
                }
            sw.Stop();
            System.Console.WriteLine("Size: " + pageSize + " Rate: " + (1 / sw.Elapsed.TotalSeconds).ToString());
        }
        public void RollbackChangesToAFile()
        {
            string fileName = @"C:\Temp\ArchiveFile.d2";
            var key = new HistorianKey();
            var value = new HistorianValue();
            using (var file = SortedTreeFile.OpenFile(fileName, isReadOnly: false))
            using (var table = file.OpenOrCreateTable<HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
            {
                using (var editor = table.BeginEdit())
                {
                    key.TimestampAsDate = DateTime.Now.AddDays(-1);
                    key.PointID = 234;
                    value.AsString = "Add Me";
                    editor.AddPoint(key, value);
                    editor.Commit();
                }

                using (var editor = table.BeginEdit())
                {
                    key.Timestamp = 31;
                    value.AsString = "But Not Me";
                    editor.AddPoint(key, value);
                    editor.Rollback(); //These changes will not be written to the disk
                }
            }
            ReadDataFromAFile();

        }
Exemplo n.º 10
0
        public void TestReadPoints()
        {
            Stopwatch sw         = new Stopwatch();
            int       pointCount = 0;

            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            HistorianServerDatabaseConfig settings = new HistorianServerDatabaseConfig("PPA", @"C:\Program Files\openHistorian\Archive\", true);

            using (HistorianServer server = new HistorianServer(settings))
            {
                using (HistorianClient client = new HistorianClient("127.0.0.1", 12345))
                    using (ClientDatabaseBase <HistorianKey, HistorianValue> database = client.GetDatabase <HistorianKey, HistorianValue>(String.Empty))
                    {
                        TreeStream <HistorianKey, HistorianValue> stream = database.Read(0, (ulong)DateTime.MaxValue.Ticks, new ulong[] { 1 });
                        while (stream.Read(key, value))
                        {
                            ;
                        }

                        sw.Start();
                        stream = database.Read(0, (ulong)DateTime.MaxValue.Ticks, new ulong[] { 65, 953, 5562 });
                        while (stream.Read(key, value))
                        {
                            pointCount++;
                        }

                        sw.Stop();
                    }
            }
            Console.WriteLine(pointCount);
            Console.WriteLine(sw.Elapsed.TotalSeconds.ToString());
        }
Exemplo n.º 11
0
        private void StartStream(object args)
        {
            HistorianKey key = new HistorianKey();
            HistorianValue value = new HistorianValue();

            using (HistorianClient client = new HistorianClient("127.0.0.1", 54996))
            using (ClientDatabaseBase<HistorianKey, HistorianValue> database = client.GetDatabase<HistorianKey, HistorianValue>(string.Empty))
            {
                using (SortedTreeTable<HistorianKey, HistorianValue> file = SortedTreeFile.OpenFile(@"H:\OGE 2009.d2", isReadOnly: true).OpenOrCreateTable<HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
                {
                    using (SortedTreeTableReadSnapshot<HistorianKey, HistorianValue> read = file.BeginRead())
                    {
                        SortedTreeScannerBase<HistorianKey, HistorianValue> scan = read.GetTreeScanner();
                        scan.SeekToStart();
                        long count = 0;
                        while (scan.Read(key, value))
                        {
                            count++;
                            database.Write(key, value);
                            if ((count % 10) == 1)
                                Thread.Sleep(1);
                        }
                    }
                }
            }
        }
 public void EnduranceTest()
 {
     HistorianKey key = new HistorianKey();
     HistorianValue value = new HistorianValue();
     using (SortedTreeTable<HistorianKey, HistorianValue> target = SortedTreeFile.CreateInMemory().OpenOrCreateTable<HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
     {
         for (uint x = 0; x < 100; x++)
         {
             using (var fileEditor = target.BeginEdit())
             {
                 for (int y = 0; y < 10; y++)
                 {
                     key.Timestamp = x;
                     key.PointID = x;
                     value.Value1 = x;
                     value.Value3 = x;
                     fileEditor.AddPoint(key, value);
                     x++;
                 }
                 fileEditor.Commit();
             }
             Assert.AreEqual(target.FirstKey.Timestamp, (ulong)0);
             Assert.AreEqual(target.LastKey.Timestamp, (ulong)(x - 1));
         }
     }
 }
        /// <summary>
        /// Gets concentrated frames from the provided stream
        /// </summary>
        /// <param name="stream">the database to use</param>
        /// <returns></returns>
        public static SortedList<DateTime, FrameData> GetFrames(this TreeStream<HistorianKey, HistorianValue> stream)
        {
            SortedList<DateTime, FrameDataConstructor> results = new SortedList<DateTime, FrameDataConstructor>();
            ulong lastTime = ulong.MinValue;
            FrameDataConstructor lastFrame = null;
            HistorianKey key = new HistorianKey();
            HistorianValue value = new HistorianValue();
            while (stream.Read(key, value))
            {
                if (lastFrame == null || key.Timestamp != lastTime)
                {
                    lastTime = key.Timestamp;
                    DateTime timestamp = new DateTime((long)lastTime);

                    if (!results.TryGetValue(timestamp, out lastFrame))
                    {
                        lastFrame = new FrameDataConstructor();
                        results.Add(timestamp, lastFrame);
                    }
                }
                lastFrame.PointId.Add(key.PointID);
                lastFrame.Values.Add(value.ToStruct());
            }
            List<FrameData> data = new List<FrameData>(results.Count);
            data.AddRange(results.Values.Select(x => x.ToFrameData()));
            return SortedListConstructor.Create(results.Keys, data);
        }
Exemplo n.º 14
0
        public void BenchmarkOld(int pointCount)
        {
            SortedPointBuffer <HistorianKey, HistorianValue> points = new SortedPointBuffer <HistorianKey, HistorianValue>(pointCount, true);

            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            for (int x = 0; x < pointCount; x++)
            {
                key.PointID = (ulong)x;
                points.TryEnqueue(key, value);
            }

            points.IsReadingMode = true;

            Stopwatch sw = new Stopwatch();

            sw.Start();
            using (BinaryStream bs = new BinaryStream(true))
            {
                SortedTree <HistorianKey, HistorianValue> st = SortedTree <HistorianKey, HistorianValue> .Create(bs, 4096, HistorianFileEncodingDefinition.TypeGuid);

                st.TryAddRange(points);
                //SequentialSortedTreeWriter<HistorianKey, HistorianValue>.Create(bs, 4096, SortedTree.FixedSizeNode, points);
            }
            sw.Stop();

            System.Console.WriteLine("Points {0}: {1}MPPS", pointCount, (pointCount / sw.Elapsed.TotalSeconds / 1000000).ToString("0.0"));
        }
Exemplo n.º 15
0
        public void Test(int count)
        {
            var lst = new List <SortedTreeTable <HistorianKey, HistorianValue> >();

            for (int x = 0; x < count; x++)
            {
                lst.Add(CreateTable());
            }

            using (var reader = new UnionTreeStream <HistorianKey, HistorianValue>(lst.Select(x => new ArchiveTreeStreamWrapper <HistorianKey, HistorianValue>(x)), true))
            {
                var       key   = new HistorianKey();
                var       value = new HistorianValue();
                Stopwatch sw    = new Stopwatch();
                sw.Start();
                while (reader.Read(key, value))
                {
                    ;
                }
                sw.Stop();
                System.Console.Write("{0}\t{1}\t{2}", count, sw.Elapsed.TotalSeconds, sw.Elapsed.TotalSeconds / count);
                System.Console.WriteLine();
            }

            lst.ForEach(x => x.Dispose());
        }
        public void CountIO()
        {
            Test(1000, false);

            int pointCount = 10000000;
            SortedPointBuffer<HistorianKey, HistorianValue> points = new SortedPointBuffer<HistorianKey, HistorianValue>(pointCount, true);

            HistorianKey key = new HistorianKey();
            HistorianValue value = new HistorianValue();

            for (int x = 0; x < pointCount; x++)
            {
                key.PointID = (ulong)x;
                points.TryEnqueue(key, value);
            }

            points.IsReadingMode = true;

            File.Delete(@"C:\Temp\fileTemp.~d2i");
            File.Delete(@"C:\Temp\fileTemp.d2i");

            Stopwatch sw = new Stopwatch();
            sw.Start();

            SortedTreeFileSimpleWriter<HistorianKey, HistorianValue>.Create(@"C:\Temp\fileTemp.~d2i", @"C:\Temp\fileTemp.d2i", 4096, null, EncodingDefinition.FixedSizeCombinedEncoding, points);

            sw.Stop();

            System.Console.WriteLine(SimplifiedSubFileStreamIoSession.ReadBlockCount);
            System.Console.WriteLine(SimplifiedSubFileStreamIoSession.WriteBlockCount);
            System.Console.WriteLine(sw.Elapsed.TotalSeconds.ToString());

        }
        public void BenchmarkRawFile()
        {
            MemoryPoolTest.TestMemoryLeak();
            const int Max = 1000000;

            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            SortedTreeTable <HistorianKey, HistorianValue> master = CreateTable();

            AddData(master, 100, 100, Max);


            DebugStopwatch sw = new DebugStopwatch();

            using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> masterRead = master.BeginRead())
            {
                double sec = sw.TimeEvent(() =>
                {
                    SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = masterRead.GetTreeScanner();
                    scanner.SeekToStart();
                    while (scanner.Read(key, value))
                    {
                    }
                });
                System.Console.WriteLine(Max / sec / 1000000);
            }
            master.Dispose();
            MemoryPoolTest.TestMemoryLeak();
        }
        public void CreateScadaDatabase()
        {
            Array.ForEach(Directory.GetFiles(@"c:\temp\Scada\", "*.d2", SearchOption.AllDirectories), File.Delete);

            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            HistorianServerDatabaseConfig settings = new HistorianServerDatabaseConfig("DB", @"c:\temp\Scada\", true);

            using (HistorianServer server = new HistorianServer(settings, 12345))
            {
                using (HistorianClient client = new HistorianClient("127.0.0.1", 12345))
                    using (ClientDatabaseBase <HistorianKey, HistorianValue> database = client.GetDatabase <HistorianKey, HistorianValue>("DB"))
                    {
                        for (ulong x = 0; x < 1000; x++)
                        {
                            key.Timestamp = x;
                            database.Write(key, value);
                        }

                        database.HardCommit();
                        System.Threading.Thread.Sleep(1200);
                    }
            }
        }
Exemplo n.º 19
0
        private TestResults TestRandom(int pageSize, uint count)
        {
            //StringBuilder sb = new StringBuilder();
            Random         R     = new Random(1);
            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            DiskIoSession.ReadCount     = 0;
            DiskIoSession.WriteCount    = 0;
            Stats.ChecksumCount         = 0;
            DiskIoSession.Lookups       = 0;
            DiskIoSession.CachedLookups = 0;

            Stopwatch sw = new Stopwatch();

            sw.Start();
            using (SortedTreeFile af = SortedTreeFile.CreateInMemory(blockSize: pageSize))
                using (SortedTreeTable <HistorianKey, HistorianValue> af2 = af.OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
                {
                    uint pointPairs = count / 5000;
                    for (uint i = 0; i < pointPairs; i++)
                    {
                        uint max = i * 5000 + 5000;
                        using (var edit = af2.BeginEdit())
                        {
                            for (ulong x = i * 5000; x < max; x++)
                            {
                                key.Timestamp = (uint)R.Next();
                                key.PointID   = 2 * x;
                                value.Value3  = 3 * x;
                                value.Value1  = 4 * x;
                                //if ((x % 100) == 0)
                                //    sb.AppendLine(x + "," + DiskIoSession.ReadCount + "," + DiskIoSession.WriteCount);
                                //if (x == 1000)
                                //    DiskIoSession.BreakOnIO = true;
                                edit.AddPoint(key, value);
                                //edit.AddPoint(uint.MaxValue - x, 2 * x, 3 * x, 4 * x);
                            }
                            edit.Commit();
                        }
                    }
                    //long cnt = af.Count();
                }

            sw.Stop();

            //File.WriteAllText(@"C:\temp\" + pageSize + ".csv",sb.ToString());


            return(new TestResults()
            {
                PageSize = pageSize,
                Rate = (float)(count / sw.Elapsed.TotalSeconds / 1000000),
                ReadCount = DiskIoSession.ReadCount,
                WriteCount = DiskIoSession.WriteCount,
                ChecksumCount = Stats.ChecksumCount,
                Lookups = DiskIoSession.Lookups,
                CachedLookups = DiskIoSession.CachedLookups
            });
        }
        public void TestOneFile()
        {
            HistorianKey   key1   = new HistorianKey();
            HistorianKey   key2   = new HistorianKey();
            HistorianValue value1 = new HistorianValue();
            HistorianValue value2 = new HistorianValue();

            Logger.Console.Verbose = VerboseLevel.All;
            MemoryPoolTest.TestMemoryLeak();
            ArchiveList <HistorianKey, HistorianValue> list = new ArchiveList <HistorianKey, HistorianValue>(null);

            SortedTreeTable <HistorianKey, HistorianValue> master = CreateTable();
            SortedTreeTable <HistorianKey, HistorianValue> table1 = CreateTable();

            AddData(master, 100, 100, 100);
            AddData(table1, 100, 100, 100);
            using (ArchiveListEditor <HistorianKey, HistorianValue> editor = list.AcquireEditLock())
            {
                editor.Add(table1);
            }

            using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> masterRead = master.BeginRead())
            {
                SortedTreeScannerBase <HistorianKey, HistorianValue> masterScan = masterRead.GetTreeScanner();
                masterScan.SeekToStart();
                TreeStreamSequential <HistorianKey, HistorianValue> masterScanSequential = masterScan.TestSequential();

                using (SequentialReaderStream <HistorianKey, HistorianValue> sequencer = new SequentialReaderStream <HistorianKey, HistorianValue>(list))
                {
                    TreeStreamSequential <HistorianKey, HistorianValue> scanner = sequencer.TestSequential();

                    int count = 0;
                    while (scanner.Read(key1, value1))
                    {
                        count++;
                        if (!masterScanSequential.Read(key2, value2))
                        {
                            throw new Exception();
                        }

                        if (!key1.IsEqualTo(key2))
                        {
                            throw new Exception();
                        }

                        if (!value1.IsEqualTo(value2))
                        {
                            throw new Exception();
                        }
                    }
                    if (masterScan.Read(key2, value2))
                    {
                        throw new Exception();
                    }
                }
            }
            list.Dispose();
            master.Dispose();
            MemoryPoolTest.TestMemoryLeak();
        }
        public void TestFile()
        {
            //		FilePath	"C:\\Temp\\Historian\\635287587300536177-Stage1-d559e63e-d938-46a9-8d57-268f7c8ba194.d2"	string
            //635329017197429979-Stage1-38887e11-4097-4937-b269-ce4037157691.d2
            //using (var file = SortedTreeFile.OpenFile(@"C:\Temp\Historian\635287587300536177-Stage1-d559e63e-d938-46a9-8d57-268f7c8ba194.d2", true))
            using (var file = SortedTreeFile.OpenFile(@"C:\Archive\635329017197429979-Stage1-38887e11-4097-4937-b269-ce4037157691.d2", true))
            //using (var file = SortedTreeFile.OpenFile(@"C:\Temp\Historian\635255664136496199-Stage2-6e758046-b2af-40ff-ae4e-85cd0c0e4501.d2", true))
            using (var table = file.OpenTable<HistorianKey, HistorianValue>())
            using (var reader = table.BeginRead())
            {
                var scanner = reader.GetTreeScanner();
                scanner.SeekToStart();
                scanner.TestSequential().Count();

                HistorianKey key = new HistorianKey();
                HistorianValue value = new HistorianValue();

                int x = 0;
                scanner.Peek(key, value);

                while (scanner.Read(key, value) && x < 10000)
                {
                    System.Console.WriteLine(key.PointID);
                    x++;

                    scanner.Peek(key, value);
                }

                scanner.Count();
            }
        }
Exemplo n.º 22
0
        public void Test()
        {
            const int MaxCount = 1000;
            Stopwatch sw       = new Stopwatch();
            SortedPointBuffer <HistorianKey, HistorianValue> buffer = new SortedPointBuffer <HistorianKey, HistorianValue>(MaxCount, true);

            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();
            Random         r     = new Random(1);

            for (int x = 0; x < MaxCount; x++)
            {
                key.Timestamp = (ulong)r.Next();
                key.PointID   = (ulong)x;

                buffer.TryEnqueue(key, value);
            }

            sw.Start();
            buffer.IsReadingMode = true;
            sw.Stop();

            System.Console.WriteLine(sw.ElapsedMilliseconds);
            System.Console.WriteLine(MaxCount / sw.Elapsed.TotalSeconds / 1000000);

            for (int x = 0; x < MaxCount; x++)
            {
                buffer.ReadSorted(x, key, value);
                System.Console.WriteLine(key.Timestamp.ToString() + "\t" + key.PointID.ToString());
            }
        }
        public void Test()
        {
            const int MaxCount = 1000;
            Stopwatch sw = new Stopwatch();
            var buffer = new SortedPointBuffer<HistorianKey, HistorianValue>(MaxCount, true);

            var key = new HistorianKey();
            var value = new HistorianValue();
            Random r = new Random(1);

            for (int x = 0; x < MaxCount; x++)
            {
                key.Timestamp = (ulong)r.Next();
                key.PointID = (ulong)x;

                buffer.TryEnqueue(key, value);
            }

            sw.Start();
            buffer.IsReadingMode = true;
            sw.Stop();

            System.Console.WriteLine(sw.ElapsedMilliseconds);
            System.Console.WriteLine(MaxCount / sw.Elapsed.TotalSeconds / 1000000);

            for (int x = 0; x < MaxCount; x++)
            {
                buffer.ReadSorted(x, key, value);
                System.Console.WriteLine(key.Timestamp.ToString() + "\t" + key.PointID.ToString());
            }
        }
        public void BenchmarkRandomData(int pointCount)
        {
            Stopwatch sw = new Stopwatch();
            var buffer = new SortedPointBuffer<HistorianKey, HistorianValue>(pointCount, true);

            var key = new HistorianKey();
            var value = new HistorianValue();

            List<double> times = new List<double>();
            for (int cnt = 0; cnt < 10; cnt++)
            {
                Random r = new Random(1);
                buffer.IsReadingMode = false;
                for (int x = 0; x < pointCount; x++)
                {
                    key.Timestamp = (ulong)r.Next();
                    key.PointID = (ulong)x;

                    buffer.TryEnqueue(key, value);
                }
                sw.Restart();
                buffer.IsReadingMode = true;
                sw.Stop();
                times.Add(sw.Elapsed.TotalSeconds);
            }
            times.Sort();
            System.Console.WriteLine("{0} points {1}ms {2} Million/second ", pointCount, times[5] * 1000, pointCount / times[5] / 1000000);
        }
Exemplo n.º 25
0
        public void GetBits()
        {
            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();
            StringBuilder  sb    = new StringBuilder();

            sb.AppendLine("Higher Bits, Bucket Number, Count, FloatValue");
            using (SortedTreeFile file = SortedTreeFile.OpenFile(@"C:\Archive\635184227258021940-Stage2-8b835d6a-8299-45bb-9624-d4a470e4abe1.d2", true))
                using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>())
                    using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader = table.BeginRead())
                        using (SortedTreeScannerBase <HistorianKey, HistorianValue> scan = reader.GetTreeScanner())
                        {
                            int count = 0;
                            scan.SeekToStart();
                            while (scan.Read(key, value))
                            {
                                count++;
                            }

                            for (int x = 1; x < 24; x++)
                            {
                                scan.SeekToStart();
                                int[] bucket = MeasureBits(scan, x);
                                Write(sb, bucket, x, count);
                            }
                        }

            Console.WriteLine(sb.ToString());
        }
Exemplo n.º 26
0
        private IEnumerable <TrendingDataPoint> Read(IEnumerable <ulong> measurementIDs, DateTime startTime, DateTime stopTime)
        {
            SeekFilterBase <HistorianKey> timeFilter = TimestampSeekFilter.CreateFromRange <HistorianKey>(startTime, stopTime);
            MatchFilterBase <HistorianKey, HistorianValue> pointFilter = null;
            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            if ((object)measurementIDs != null)
            {
                pointFilter = PointIdMatchFilter.CreateFromList <HistorianKey, HistorianValue>(measurementIDs);
            }

            // Start stream reader for the provided time window and selected points
            using (TreeStream <HistorianKey, HistorianValue> stream = m_database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter))
            {
                m_treeStreams.Add(stream);

                while (stream.Read(key, value))
                {
                    yield return(new TrendingDataPoint()
                    {
                        ChannelID = (int)key.PointID.HighDoubleWord(),
                        SeriesID = (SeriesID)(int)key.PointID.LowDoubleWord(),
                        Timestamp = key.TimestampAsDate,
                        Value = value.AsSingle
                    });
                }
            }
        }
Exemplo n.º 27
0
        public void TestFile()
        {
            //		FilePath	"C:\\Temp\\Historian\\635287587300536177-Stage1-d559e63e-d938-46a9-8d57-268f7c8ba194.d2"	string
            //635329017197429979-Stage1-38887e11-4097-4937-b269-ce4037157691.d2
            //using (var file = SortedTreeFile.OpenFile(@"C:\Temp\Historian\635287587300536177-Stage1-d559e63e-d938-46a9-8d57-268f7c8ba194.d2", true))
            using (var file = SortedTreeFile.OpenFile(@"C:\Archive\635329017197429979-Stage1-38887e11-4097-4937-b269-ce4037157691.d2", true))
                //using (var file = SortedTreeFile.OpenFile(@"C:\Temp\Historian\635255664136496199-Stage2-6e758046-b2af-40ff-ae4e-85cd0c0e4501.d2", true))
                using (var table = file.OpenTable <HistorianKey, HistorianValue>())
                    using (var reader = table.BeginRead())
                    {
                        var scanner = reader.GetTreeScanner();
                        scanner.SeekToStart();
                        scanner.TestSequential().Count();

                        HistorianKey   key   = new HistorianKey();
                        HistorianValue value = new HistorianValue();

                        int x = 0;
                        scanner.Peek(key, value);

                        while (scanner.Read(key, value) && x < 10000)
                        {
                            System.Console.WriteLine(key.PointID);
                            x++;

                            scanner.Peek(key, value);
                        }

                        scanner.Count();
                    }
        }
Exemplo n.º 28
0
        public void RollbackChangesToAFile()
        {
            string         fileName = @"C:\Temp\ArchiveFile.d2";
            HistorianKey   key      = new HistorianKey();
            HistorianValue value    = new HistorianValue();

            using (SortedTreeFile file = SortedTreeFile.OpenFile(fileName, isReadOnly: false))
                using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
                {
                    using (SortedTreeTableEditor <HistorianKey, HistorianValue> editor = table.BeginEdit())
                    {
                        key.TimestampAsDate = DateTime.Now.AddDays(-1);
                        key.PointID         = 234;
                        value.AsString      = "Add Me";
                        editor.AddPoint(key, value);
                        editor.Commit();
                    }

                    using (SortedTreeTableEditor <HistorianKey, HistorianValue> editor = table.BeginEdit())
                    {
                        key.Timestamp  = 31;
                        value.AsString = "But Not Me";
                        editor.AddPoint(key, value);
                        editor.Rollback(); //These changes will not be written to the disk
                    }
                }
            ReadDataFromAFile();
        }
Exemplo n.º 29
0
        private void StartStream(object args)
        {
            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            using (HistorianClient client = new HistorianClient("127.0.0.1", 54996))
                using (ClientDatabaseBase <HistorianKey, HistorianValue> database = client.GetDatabase <HistorianKey, HistorianValue>(string.Empty))
                {
                    using (SortedTreeTable <HistorianKey, HistorianValue> file = SortedTreeFile.OpenFile(@"H:\OGE 2009.d2", isReadOnly: true).OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
                    {
                        using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> read = file.BeginRead())
                        {
                            SortedTreeScannerBase <HistorianKey, HistorianValue> scan = read.GetTreeScanner();
                            scan.SeekToStart();
                            long count = 0;
                            while (scan.Read(key, value))
                            {
                                count++;
                                database.Write(key, value);
                                if ((count % 10) == 1)
                                {
                                    Thread.Sleep(1);
                                }
                            }
                        }
                    }
                }
        }
Exemplo n.º 30
0
        /// <summary>
        /// Read historian data from server.
        /// </summary>
        /// <param name="connection">openHistorian connection.</param>
        /// <param name="startTime">Start time of query.</param>
        /// <param name="stopTime">Stop time of query.</param>
        /// <param name="measurementIDs">Array of measurement IDs to query - or <c>null</c> for all available points.</param>
        /// <param name="resolution">Resolution for data query.</param>
        /// <returns>Enumeration of <see cref="IMeasurement"/> values read for time range.</returns>
        /// <remarks>
        /// <example>
        /// <code>
        /// using (var connection = new Connection("127.0.0.1", "PPA"))
        ///     foreach(var measurement in GetHistorianData(connection, DateTime.UtcNow.AddMinutes(-1.0D), DateTime.UtcNow))
        ///         Console.WriteLine("{0}:{1} @ {2} = {3}, quality: {4}", measurement.Key.Source, measurement.Key.ID, measurement.Timestamp, measurement.Value, measurement.StateFlags);
        /// </code>
        /// </example>
        /// </remarks>
        public static IEnumerable <IMeasurement> GetHistorianData(Connection connection, DateTime startTime, DateTime stopTime, IEnumerable <ulong> measurementIDs = null, Resolution resolution = Resolution.Full)
        {
            SeekFilterBase <HistorianKey> timeFilter;
            MatchFilterBase <HistorianKey, HistorianValue> pointFilter = null;
            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            // Set data scan resolution
            if (resolution == Resolution.Full)
            {
                timeFilter = TimestampSeekFilter.CreateFromRange <HistorianKey>(startTime, stopTime);
            }
            else
            {
                TimeSpan             resolutionInterval = resolution.GetInterval();
                BaselineTimeInterval interval           = BaselineTimeInterval.Second;

                if (resolutionInterval.Ticks < Ticks.PerMinute)
                {
                    interval = BaselineTimeInterval.Second;
                }
                else if (resolutionInterval.Ticks < Ticks.PerHour)
                {
                    interval = BaselineTimeInterval.Minute;
                }
                else if (resolutionInterval.Ticks == Ticks.PerHour)
                {
                    interval = BaselineTimeInterval.Hour;
                }

                startTime = startTime.BaselinedTimestamp(interval);
                stopTime  = stopTime.BaselinedTimestamp(interval);

                timeFilter = TimestampSeekFilter.CreateFromIntervalData <HistorianKey>(startTime, stopTime, resolutionInterval, new TimeSpan(TimeSpan.TicksPerMillisecond));
            }

            // Setup point ID selections
            if (measurementIDs != null)
            {
                pointFilter = PointIdMatchFilter.CreateFromList <HistorianKey, HistorianValue>(measurementIDs);
            }

            // Start stream reader for the provided time window and selected points
            using (Database database = connection.OpenDatabase())
            {
                TreeStream <HistorianKey, HistorianValue> stream = database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter);

                while (stream.Read(key, value))
                {
                    yield return new Measurement
                           {
                               Metadata   = MeasurementKey.LookUpOrCreate(connection.InstanceName, (uint)key.PointID).Metadata,
                               Timestamp  = key.TimestampAsDate,
                               Value      = value.AsSingle,
                               StateFlags = (MeasurementStateFlags)value.Value3
                           }
                }
                ;
            }
        }
Exemplo n.º 31
0
        public void ReadDataFromAFile()
        {
            string   fileName = @"c:\temp\Tulsa Bank 1 LTC 1.d2";
            DateTime start    = DateTime.Parse("4/17/2013 10:38 AM");
            DateTime stop     = DateTime.Parse("4/17/2013 10:38 AM");

            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            using (var file = SortedTreeFile.OpenFile(fileName, isReadOnly: true))
                using (var table = file.OpenTable <HistorianKey, HistorianValue>())
                    using (var snapshot = table.BeginRead())
                    {
                        var scanner = snapshot.GetTreeScanner();
                        var seekKey = new HistorianKey();
                        seekKey.TimestampAsDate = start;
                        seekKey.PointID         = 3142023;
                        scanner.SeekToKey(seekKey);
                        while (scanner.Read(key, value) && key.TimestampAsDate <= stop)
                        {
                            Console.WriteLine("{0}, {1}, {2}",
                                              key.TimestampAsDate.ToString(), key.PointID, value.AsString);
                        }
                    }
        }
        SortedTreeTable<HistorianKey, HistorianValue> CreateTable()
        {
            var r = new Random(seed++);
            var key = new HistorianKey();
            var value = new HistorianValue();
            var file = SortedTreeFile.CreateInMemory();
            var table = file.OpenOrCreateTable<HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding);

            using (var edit = table.BeginEdit())
            {
                for (int x = 0; x < 1000; x++)
                {
                    key.Timestamp = (ulong)r.Next();
                    key.PointID = (ulong)r.Next();
                    key.EntryNumber = (ulong)r.Next();
                    edit.AddPoint(key, value);
                }
                edit.Commit();
            }




            return table;
        }
        public void GetBits()
        {
            HistorianKey key = new HistorianKey();
            HistorianValue value = new HistorianValue();
            StringBuilder sb = new StringBuilder();
            sb.AppendLine("Higher Bits, Bucket Number, Count, FloatValue");
            using (var file = SortedTreeFile.OpenFile(@"C:\Archive\635184227258021940-Stage2-8b835d6a-8299-45bb-9624-d4a470e4abe1.d2", true))
            using (var table = file.OpenTable<HistorianKey, HistorianValue>())
            using (var reader = table.BeginRead())
            using (var scan = reader.GetTreeScanner())
            {
                int count = 0;
                scan.SeekToStart();
                while (scan.Read(key,value))
                    count++;

                for (int x = 1; x < 24; x++)
                {
                    scan.SeekToStart();
                    var bucket = MeasureBits(scan, x);
                    Write(sb, bucket, x, count);
                }
            }

            Console.WriteLine(sb.ToString());
        }
        void AddDataTerminal(SortedTreeTable <HistorianKey, HistorianValue> table, ulong pointID, DateTime startTime, TimeSpan stepTime, ulong startValue, ulong stepValue, int count)
        {
            using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = table.BeginEdit())
            {
                HistorianKey   key   = new HistorianKey();
                HistorianValue value = new HistorianValue();
                key.SetMin();
                ulong t = (ulong)startTime.Ticks;
                ulong v = startValue;

                while (count > 0)
                {
                    count--;
                    key.Timestamp = t;
                    key.PointID   = pointID;
                    value.Value1  = v;

                    edit.AddPoint(key, value);
                    t += (ulong)stepTime.Ticks;
                    v += stepValue;
                }

                edit.Commit();
            }
        }
Exemplo n.º 35
0
        public void TestSmall()
        {
            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            using (SortedTreeFile af = SortedTreeFile.CreateInMemory())
                using (SortedTreeTable <HistorianKey, HistorianValue> file = af.OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
                {
                    using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = file.BeginEdit())
                    {
                        for (int x = 0; x < 10000000; x++)
                        {
                            key.Timestamp = (ulong)x;
                            edit.AddPoint(key, value);
                        }
                        edit.Commit();
                    }

                    using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> read = file.BeginRead())
                        using (SortedTreeScannerBase <HistorianKey, HistorianValue> scan = read.GetTreeScanner())
                        {
                            int count = 0;
                            scan.SeekToStart();
                            while (scan.Read(key, value))
                            {
                                count++;
                            }
                            System.Console.WriteLine(count.ToString());
                        }
                }
        }
        public void BenchmarkTwoFiles()
        {
            MemoryPoolTest.TestMemoryLeak();
            const int Max = 1000000;
            ArchiveList <HistorianKey, HistorianValue> list = new ArchiveList <HistorianKey, HistorianValue>(null);
            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();
            SortedTreeTable <HistorianKey, HistorianValue> table1 = CreateTable();
            SortedTreeTable <HistorianKey, HistorianValue> table2 = CreateTable();

            AddData(table1, 100, 100, Max / 2);
            AddData(table2, 101, 100, Max / 2);
            using (ArchiveListEditor <HistorianKey, HistorianValue> editor = list.AcquireEditLock())
            {
                editor.Add(table1);
                editor.Add(table2);
            }

            SequentialReaderStream <HistorianKey, HistorianValue> sequencer = new SequentialReaderStream <HistorianKey, HistorianValue>(list);

            DebugStopwatch sw = new DebugStopwatch();

            double sec = sw.TimeEvent(() =>
            {
                SequentialReaderStream <HistorianKey, HistorianValue> scanner = sequencer;
                while (scanner.Read(key, value))
                {
                }
            });

            System.Console.WriteLine(Max / sec / 1000000);
            list.Dispose();
            MemoryPoolTest.TestMemoryLeak();
        }
        /// <summary>
        /// Gets concentrated frames from the provided stream
        /// </summary>
        /// <param name="stream">the database to use</param>
        /// <returns></returns>
        public static SortedList <DateTime, FrameData> GetFrames(this TreeStream <HistorianKey, HistorianValue> stream)
        {
            SortedList <DateTime, FrameDataConstructor> results = new SortedList <DateTime, FrameDataConstructor>();
            ulong lastTime = ulong.MinValue;
            FrameDataConstructor lastFrame = null;
            HistorianKey         key       = new HistorianKey();
            HistorianValue       value     = new HistorianValue();

            while (stream.Read(key, value))
            {
                if (lastFrame == null || key.Timestamp != lastTime)
                {
                    lastTime = key.Timestamp;
                    DateTime timestamp = new DateTime((long)lastTime);

                    if (!results.TryGetValue(timestamp, out lastFrame))
                    {
                        lastFrame = new FrameDataConstructor();
                        results.Add(timestamp, lastFrame);
                    }
                }
                lastFrame.PointId.Add(key.PointID);
                lastFrame.Values.Add(value.ToStruct());
            }
            List <FrameData> data = new List <FrameData>(results.Count);

            data.AddRange(results.Values.Select(x => x.ToFrameData()));
            return(SortedListConstructor.Create(results.Keys, data));
        }
        public void BenchmarkOld2(int pointCount)
        {
            SortedPointBuffer<HistorianKey, HistorianValue> points = new SortedPointBuffer<HistorianKey, HistorianValue>(pointCount, true);

            HistorianKey key = new HistorianKey();
            HistorianValue value = new HistorianValue();

            for (int x = 0; x < pointCount; x++)
            {
                key.PointID = (ulong)x;
                points.TryEnqueue(key, value);
            }

            points.IsReadingMode = true;

            var sw = new Stopwatch();
            sw.Start();
            using (var bs = new BinaryStream(true))
            {
                var st = SortedTree<HistorianKey, HistorianValue>.Create(bs, 4096, HistorianFileEncodingDefinition.TypeGuid);

                st.AddRange(points);

                //SequentialSortedTreeWriter<HistorianKey, HistorianValue>.Create(bs, 4096, SortedTree.FixedSizeNode, points);
            }
            sw.Stop();

            System.Console.WriteLine("Points {0}: {1}MPPS", pointCount, (pointCount / sw.Elapsed.TotalSeconds / 1000000).ToString("0.0"));
        }
Exemplo n.º 39
0
 /// <summary>
 /// Initializes a new instance of the <see cref="RemoteOutputAdapter"/> class.
 /// </summary>
 public RemoteOutputAdapter()
 {
     m_port   = LocalOutputAdapter.DefaultPort;
     m_server = DefaultServer;
     m_key    = new HistorianKey();
     m_value  = new HistorianValue();
 }
Exemplo n.º 40
0
        public void BenchmarkRandomData(int pointCount)
        {
            Stopwatch sw = new Stopwatch();
            SortedPointBuffer <HistorianKey, HistorianValue> buffer = new SortedPointBuffer <HistorianKey, HistorianValue>(pointCount, true);

            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            List <double> times = new List <double>();

            for (int cnt = 0; cnt < 10; cnt++)
            {
                Random r = new Random(1);
                buffer.IsReadingMode = false;
                for (int x = 0; x < pointCount; x++)
                {
                    key.Timestamp = (ulong)r.Next();
                    key.PointID   = (ulong)x;

                    buffer.TryEnqueue(key, value);
                }
                sw.Restart();
                buffer.IsReadingMode = true;
                sw.Stop();
                times.Add(sw.Elapsed.TotalSeconds);
            }
            times.Sort();
            System.Console.WriteLine("{0} points {1}ms {2} Million/second ", pointCount, times[5] * 1000, pointCount / times[5] / 1000000);
        }
Exemplo n.º 41
0
        public void ReadFromAFileWhileWritingToIt()
        {
            HistorianKey   key      = new HistorianKey();
            HistorianValue value    = new HistorianValue();
            string         fileName = @"C:\Temp\ArchiveFile.d2";

            using (SortedTreeFile file = SortedTreeFile.OpenFile(fileName, isReadOnly: false))
                using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>())
                {
                    using (SortedTreeTableEditor <HistorianKey, HistorianValue> writer = table.BeginEdit())
                        using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader = table.BeginRead())
                        {
                            SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = reader.GetTreeScanner();
                            scanner.SeekToStart();
                            while (scanner.Read(key, value))
                            {
                                key.Timestamp++;
                                value.Value1++;
                                writer.AddPoint(key, value);
                            }

                            writer.Commit();
                        }
                }
            ReadDataFromAFile();
        }
Exemplo n.º 42
0
        public void CountIO()
        {
            Test(1000, false);

            int pointCount = 10000000;
            SortedPointBuffer <HistorianKey, HistorianValue> points = new SortedPointBuffer <HistorianKey, HistorianValue>(pointCount, true);

            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            for (int x = 0; x < pointCount; x++)
            {
                key.PointID = (ulong)x;
                points.TryEnqueue(key, value);
            }

            points.IsReadingMode = true;

            File.Delete(@"C:\Temp\fileTemp.~d2i");
            File.Delete(@"C:\Temp\fileTemp.d2i");

            Stopwatch sw = new Stopwatch();

            sw.Start();

            SortedTreeFileSimpleWriter <HistorianKey, HistorianValue> .Create(@"C:\Temp\fileTemp.~d2i", @"C:\Temp\fileTemp.d2i", 4096, null, EncodingDefinition.FixedSizeCombinedEncoding, points);

            sw.Stop();

            System.Console.WriteLine(SimplifiedSubFileStreamIoSession.ReadBlockCount);
            System.Console.WriteLine(SimplifiedSubFileStreamIoSession.WriteBlockCount);
            System.Console.WriteLine(sw.Elapsed.TotalSeconds.ToString());
        }
 public SnapDBClient(SnapDBEngine engine, string instanceName)
 {
     m_client   = SnapClient.Connect(engine.ServerHost);
     m_database = m_client.GetDatabase <HistorianKey, HistorianValue>(instanceName);
     m_key      = new HistorianKey();
     m_value    = new HistorianValue();
     m_lastKey  = new HistorianKey();
 }
Exemplo n.º 44
0
 void FillData(ulong pointId, HistorianValue key)
 {
     if (m_fillMeasurements.TryGetValue(pointId, out Measurements m))
     {
         m.ReadNumber = m_readIndex;
         m.Value      = key.AsSingle;
     }
 }
        public SnapDBClient(string hostAddress, int port, string instanceName, ulong startTime, ulong endTime, int frameRate, IEnumerable<ulong> pointIDs)
        {
            m_client = new HistorianClient(hostAddress, port);
            m_database = m_client.GetDatabase<HistorianKey, HistorianValue>(instanceName);
            m_key = new HistorianKey();
            m_value = new HistorianValue();

            SeekFilterBase<HistorianKey> timeFilter = TimestampSeekFilter.CreateFromRange<HistorianKey>(DataPoint.RoundTimestamp(startTime, frameRate), DataPoint.RoundTimestamp(endTime, frameRate));
            MatchFilterBase<HistorianKey, HistorianValue> pointFilter = PointIdMatchFilter.CreateFromList<HistorianKey, HistorianValue>(pointIDs);

            m_stream = m_database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter);
        }
        public void TestTwoFiles()
        {
            MemoryPoolTest.TestMemoryLeak();
            ArchiveList<HistorianKey, HistorianValue> list = new ArchiveList<HistorianKey, HistorianValue>(null);

            HistorianKey key1 = new HistorianKey();
            HistorianKey key2 = new HistorianKey();
            HistorianValue value1 = new HistorianValue();
            HistorianValue value2 = new HistorianValue();

            var master = CreateTable();
            var table1 = CreateTable();
            var table2 = CreateTable();
            AddData(master, 100, 100, 100);
            AddData(table1, 100, 100, 100);
            AddData(master, 101, 100, 100);
            AddData(table2, 101, 100, 100);
            using (var editor = list.AcquireEditLock())
            {
                editor.Add(table1);
                editor.Add(table2);
            }

            using (var masterRead = master.BeginRead())
            {
                var masterScan = masterRead.GetTreeScanner();
                masterScan.SeekToStart();
                var masterScanSequential = masterScan.TestSequential();

                using (var sequencer = new SequentialReaderStream<HistorianKey, HistorianValue>(list))
                {
                    var scanner = sequencer.TestSequential();

                    while (scanner.Read(key1, value1))
                    {
                        if (!masterScanSequential.Read(key2, value2))
                            throw new Exception();

                        if (!key1.IsEqualTo(key2))
                            throw new Exception();

                        if (!value1.IsEqualTo(value2))
                            throw new Exception();
                    }
                    if (masterScan.Read(key2, value2))
                        throw new Exception();
                }
            }
            master.Dispose();
            list.Dispose();
            MemoryPoolTest.TestMemoryLeak();
        }
 public int[] MeasureBits(TreeStream<HistorianKey, HistorianValue> stream, int higherBits)
 {
     HistorianKey hkey = new HistorianKey();
     HistorianValue hvalue = new HistorianValue();
     int[] bucket = new int[1 << higherBits];
     int shiftBits = 32 - higherBits;
     while (stream.Read(hkey,hvalue))
     {
         uint value = (uint)hvalue.Value1 >> shiftBits;
         bucket[value]++;
     }
     return bucket;
 }
 public void Test()
 {
     HistorianKey key = new HistorianKey();
     HistorianValue value = new HistorianValue();
     using (var file = SortedTreeFile.OpenFile(@"C:\Unison\GPA\Codeplex\openHistorian\Main\Build\Output\Release\Applications\openHistorian\Archive\635293583194231435-Stage2-0ef36dcc-4264-498f-b194-01b2043a9231.d2", true))
     using (var table = file.OpenTable<HistorianKey, HistorianValue>())
     using (var reader = table.BeginRead())
     using (var scan = reader.GetTreeScanner())
     {
         scan.SeekToStart();
         while (scan.Read(key,value))
             ;
     }
 }
        /// <summary>
        /// Queries the provided signals within a the provided time window [Inclusive]
        /// </summary>
        /// <param name="database"></param>
        /// <param name="startTime">the lower bound of the time</param>
        /// <param name="endTime">the upper bound of the time. [Inclusive]</param>
        /// <param name="signals">an IEnumerable of all of the signals to query as part of the results set.</param>
        /// <returns></returns>
        public static Dictionary<ulong, RawSignalTimeValue> GetRawSignals(this ClientDatabaseBase<HistorianKey, HistorianValue> database, DateTime startTime, DateTime endTime, IEnumerable<ulong> signals)
        {
            HistorianKey key = new HistorianKey();
            HistorianValue value = new HistorianValue();

            Dictionary<ulong, RawSignalTimeValue> results = signals.ToDictionary((x) => x, (x) => new RawSignalTimeValue());

            TreeStream<HistorianKey, HistorianValue> stream = database.Read((ulong)startTime.Ticks, (ulong)endTime.Ticks, signals);

            while (stream.Read(key, value))
            {
                results[key.PointID].Signals.Add(key.TimestampAsDate, value.ToStruct());
            }
            return results;
        }
 public void AppendDataToAnExistingFile()
 {
     string fileName = @"C:\Temp\ArchiveFile.d2";
     var key = new HistorianKey();
     var value = new HistorianValue();
     using (var file = SortedTreeFile.OpenFile(fileName, isReadOnly: false))
     using (var table = file.OpenTable<HistorianKey, HistorianValue>())
     using (var editor = table.BeginEdit())
     {
         key.TimestampAsDate = DateTime.Now;
         key.PointID = 2;
         value.AsString = "Test Append";
         editor.AddPoint(key, value);
         editor.Commit();
     }
     ReadDataFromAFile();
 }
        public void TestReadData()
        {
            var key = new HistorianKey();
            var value = new HistorianValue();

            var settings = new HistorianServerDatabaseConfig("DB", @"c:\temp\Scada\", true);
            using (HistorianServer server = new HistorianServer(settings, 12345))
            {
                using (var client = new HistorianClient("127.0.0.1", 12345))
                using (var database = client.GetDatabase<HistorianKey, HistorianValue>("DB"))
                {
                    var stream = database.Read(0, 1000);
                    while (stream.Read(key, value))
                        Console.WriteLine(key.Timestamp);
                }
            }
        }
        public void WriteData()
        {
            Logger.Console.Verbose = VerboseLevel.All;

            using (var server = CreateServer())
            {
                using (var client = SnapClient.Connect(server))
                using (var db = client.GetDatabase<HistorianKey, HistorianValue>("PPA"))
                {
                    var key = new HistorianKey();
                    var value = new HistorianValue();
                    key.TimestampAsDate = DateTime.Now;
                    key.PointID = LittleEndian.ToUInt64(Guid.NewGuid().ToByteArray(), 0);
                    db.Write(key, value);
                }
            }
        }
        public void CreateSnapshotTest()
        {
            HistorianKey key = new HistorianKey();
            HistorianValue value = new HistorianValue();
            key.Timestamp = 1;
            key.PointID = 2;
            value.Value1 = 3;
            value.Value2 = 4;
            using (SortedTreeTable<HistorianKey, HistorianValue> target = SortedTreeFile.CreateInMemory().OpenOrCreateTable<HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
            {
                ulong date = 1;
                ulong pointId = 2;
                ulong value1 = 3;
                ulong value2 = 4;
                SortedTreeTableSnapshotInfo<HistorianKey, HistorianValue> snap1;
                using (var fileEditor = target.BeginEdit())
                {
                    fileEditor.AddPoint(key, value);
                    key.Timestamp++;
                    fileEditor.AddPoint(key, value);
                    snap1 = target.AcquireReadSnapshot();
                    fileEditor.Commit();
                }
                SortedTreeTableSnapshotInfo<HistorianKey, HistorianValue> snap2 = target.AcquireReadSnapshot();

                using (SortedTreeTableReadSnapshot<HistorianKey, HistorianValue> instance = snap1.CreateReadSnapshot())
                {
                    SortedTreeScannerBase<HistorianKey, HistorianValue> scanner = instance.GetTreeScanner();
                    scanner.SeekToStart();
                    Assert.AreEqual(false, scanner.Read(key, value));
                }
                using (SortedTreeTableReadSnapshot<HistorianKey, HistorianValue> instance = snap2.CreateReadSnapshot())
                {
                    SortedTreeScannerBase<HistorianKey, HistorianValue> scanner = instance.GetTreeScanner();
                    scanner.SeekToStart();
                    Assert.AreEqual(true, scanner.Read(key, value));
                    Assert.AreEqual(1uL, key.Timestamp);
                    Assert.AreEqual(2uL, key.PointID);
                    Assert.AreEqual(3uL, value.Value1);
                    Assert.AreEqual(4uL, value.Value2);
                }
                Assert.AreEqual(1uL, target.FirstKey.Timestamp);
                Assert.AreEqual(2uL, target.LastKey.Timestamp);
            }
        }
        public void TestReadData()
        {
            using (var server = new HistorianServer(new HistorianServerDatabaseConfig("DB", @"c:\temp\Scada\", false), 1234))
            {
                using (var client = SnapClient.Connect(server.Host))
                {
                    var database = client.GetDatabase<HistorianKey, HistorianValue>("DB");
                    var stream = database.Read(10, 800 - 1);
                    HistorianKey key = new HistorianKey();
                    HistorianValue value = new HistorianValue();
                    while (stream.Read(key, value))
                    {
                        Console.WriteLine(key.Timestamp);
                    }
                }

            }
        }
        public void BenchmarkWriteSpeed()
        {
            DebugStopwatch sw = new DebugStopwatch();

            double time;
            double count = 0;

            using (SortedTreeFile file = SortedTreeFile.CreateInMemory())
            {
                var table = file.OpenOrCreateTable<HistorianKey, HistorianValue>(HistorianFileEncodingDefinition.TypeGuid);
                HistorianKey key = new HistorianKey();
                HistorianValue value = new HistorianValue();

                time = sw.TimeEvent(() =>
                {
                    //TreeKeyMethodsBase<HistorianKey>.ClearStats();
                    //TreeValueMethodsBase<HistorianKey>.ClearStats();
                    count = 0;
                    using (var scan = table.BeginEdit())
                    {
                        for (uint x = 0; x < 10000000; x++)
                        {
                            key.PointID = x;
                            scan.AddPoint(key, value);
                            count++;
                        }
                        scan.Rollback();
                    }
                });
            }

            Console.WriteLine((count / 1000000 / time).ToString() + " Million PPS");

            //Console.WriteLine("KeyMethodsBase calls");
            //for (int x = 0; x < 23; x++)
            //{
            //    Console.WriteLine(TreeKeyMethodsBase<HistorianKey>.CallMethods[x] + "\t" + ((TreeKeyMethodsBase<HistorianKey>.Method)(x)).ToString());
            //}
            //Console.WriteLine("ValueMethodsBase calls");
            //for (int x = 0; x < 5; x++)
            //{
            //    Console.WriteLine(TreeValueMethodsBase<HistorianValue>.CallMethods[x] + "\t" + ((TreeValueMethodsBase<HistorianValue>.Method)(x)).ToString());
            //}
        }
        public void VerifyDB()
        {
            //Logger.ReportToConsole(VerboseLevel.All ^ VerboseLevel.DebugLow);
            //Logger.ConsoleSubscriber.AddIgnored(Logger.LookupType("GSF.SortedTreeStore"));
            Globals.MemoryPool.SetMaximumBufferSize(1000 * 1024 * 1024);
            Globals.MemoryPool.SetTargetUtilizationLevel(TargetUtilizationLevels.Low);

            var settings = new HistorianServerDatabaseConfig("DB", "c:\\temp\\benchmark\\", true);
            using (var engine = new SnapServer(settings))
            using (var client = SnapClient.Connect(engine))
            using (var db = client.GetDatabase<HistorianKey, HistorianValue>("DB"))
            using (var scan = db.Read(null, null, null))
            {
                var key = new HistorianKey();
                var value = new HistorianValue();

                Stopwatch sw = new Stopwatch();
                sw.Start();

                for (int x = 0; x < PointsToArchive; x++)
                {
                    if (!scan.Read(key, value))
                        throw new Exception("Missing points");
                    if (key.PointID != (ulong)x)
                        throw new Exception("Corrupt");
                    if (key.Timestamp != 0)
                        throw new Exception("Corrupt");
                    if (key.EntryNumber != 0)
                        throw new Exception("Corrupt");
                    if (value.Value1 != 0)
                        throw new Exception("Corrupt");
                    if (value.Value1 != 0)
                        throw new Exception("Corrupt");
                    if (value.Value1 != 0)
                        throw new Exception("Corrupt");
                }

                double totalTime = sw.Elapsed.TotalSeconds;
                Console.WriteLine("Completed read test in {0:#,##0.00} seconds at {1:#,##0.00} points per second", totalTime, PointsToArchive / totalTime);

                if (scan.Read(key, value))
                    throw new Exception("too many points");
            }
        }
 public void WriteDataToAFile()
 {
     Logger.Console.Verbose = VerboseLevel.All;
     string fileName = @"C:\Temp\ArchiveFile.d2";
     if (File.Exists(fileName))
         File.Delete(fileName);
     var key = new HistorianKey();
     var value = new HistorianValue();
     using (var file = SortedTreeFile.CreateFile(fileName))
     using (var table = file.OpenOrCreateTable<HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
     using (var editor = table.BeginEdit())
     {
         key.TimestampAsDate = DateTime.Now;
         key.PointID = 1;
         value.AsString = "Test Write";
         editor.AddPoint(key, value);
         editor.Commit();
     }
 }
        public void TestOld()
        {
            Test(1000, false);

            int pointCount = 10000000;
            SortedPointBuffer<HistorianKey, HistorianValue> points = new SortedPointBuffer<HistorianKey, HistorianValue>(pointCount, true);

            HistorianKey key = new HistorianKey();
            HistorianValue value = new HistorianValue();

            for (int x = 0; x < pointCount; x++)
            {
                key.PointID = (ulong)x;
                points.TryEnqueue(key, value);
            }

            points.IsReadingMode = true;

            File.Delete(@"C:\Temp\fileTemp.~d2i");
            File.Delete(@"C:\Temp\fileTemp.d2i");

            Stopwatch sw = new Stopwatch();
            sw.Start();

            using (var file = SortedTreeFile.CreateFile(@"C:\Temp\fileTemp.~d2i"))
            using (var table = file.OpenOrCreateTable<HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
            {
                using (var edit = table.BeginEdit())
                {
                    edit.AddPoints(points);
                    edit.Commit();
                }
            }

            //SortedTreeFileSimpleWriter<HistorianKey, HistorianValue>.Create(@"C:\Temp\fileTemp.~d2i", @"C:\Temp\fileTemp.d2i", 4096, SortedTree.FixedSizeNode, points);

            sw.Stop();

            System.Console.WriteLine(SimplifiedSubFileStreamIoSession.ReadBlockCount);
            System.Console.WriteLine(SimplifiedSubFileStreamIoSession.WriteBlockCount);
            System.Console.WriteLine(sw.Elapsed.TotalSeconds.ToString());

        }
        public void ReadDataFromAFile()
        {
            string fileName = @"C:\Temp\ArchiveFile.d2";

            HistorianKey key = new HistorianKey();
            HistorianValue value = new HistorianValue();

            using (var file = SortedTreeFile.OpenFile(fileName, isReadOnly: true))
            using (var table = file.OpenTable<HistorianKey, HistorianValue>())
            using (var snapshot = table.BeginRead())
            {
                var scanner = snapshot.GetTreeScanner();
                scanner.SeekToStart();
                while (scanner.Read(key,value))
                {
                    Console.WriteLine("{0}, {1}, {2}",
                        key.TimestampAsDate.ToString(), key.PointID, value.AsString);
                }
            }
        }
Exemplo n.º 60
0
        private void BuildListOfAllPoints()
        {
            HashSet<ulong> keys = new HashSet<ulong>();
            var client = SnapClient.Connect(m_archiveFile.Host);
            var db = client.GetDatabase<HistorianKey, HistorianValue>("");
            TreeStream<HistorianKey, HistorianValue> scanner = db.Read(0, ulong.MaxValue);
            HistorianKey key = new HistorianKey();
            HistorianValue value = new HistorianValue();

            while (scanner.Read(key, value))
            {
                keys.Add(key.PointID);
            }
            List<ulong> AllKeys = keys.ToList();
            AllKeys.Sort();

            chkAllPoints.Items.Clear();
            AllKeys.ForEach((x) => chkAllPoints.Items.Add(x));
            db.Dispose();
            client.Dispose();
        }