Beispiel #1
0
        public async Task RunAsync(CancellationToken token)
        {
            _logger.LogInformation("RfmTrainingWorker.RunAsync");

            IReadOnlyList <string>         tableNames           = _options.TableNames;
            List <Task <TableStatistics> > tableStatisticsTasks = new List <Task <TableStatistics> >(tableNames.Count);

            foreach (string tableName in tableNames)
            {
                tableStatisticsTasks.Add(this._tableStore.GetTableStatisticsAsync(tableName, token));
            }
            TableStatistics[] tableStatisticsArray = await Task.WhenAll(tableStatisticsTasks).ConfigureAwait(false);

            List <TableDefinition> tableDefinitionList = new List <TableDefinition>(tableStatisticsTasks.Count);

            for (int index = 0; index < tableStatisticsTasks.Count; ++index)
            {
                TableStatistics result = tableStatisticsTasks[index].Result;
                if (result == null)
                {
                    this._logger.LogWarning(string.Format("Statistics data for {0} table could not be retrieved. It will not participate in model training.", (object)tableNames[index]));
                }
                else
                {
                    tableDefinitionList.Add(result.Definition);
                }
            }
            ModelStatistics modelStatistics = await _model.TrainAsync(_options.SchemaName, token, tableDefinitionList.ToArray()).ConfigureAwait(false);

            if (modelStatistics != null)
            {
                await UpdateRfmFacets(modelStatistics as RfmStatistics, token);
            }
        }
Beispiel #2
0
        public Table(String handHistoryFilePath, Window window, PokerClient pokerClient, PlayerDatabase playerDatabase)
        {
            this.handHistoryFilePath = handHistoryFilePath;
            this.window = window;
            this.pokerClient = pokerClient;
            this.playerDatabase = playerDatabase;
            this.Game = PokerGame.Unknown;
            this.maxSeatingCapacity = 0; // We don't know yet
            this.TableId = String.Empty; // We don't know yet
            this.GameID = String.Empty; // We don't know yet
            this.currentHeroName = String.Empty;
            this.currentHeroSeat = 0;
            this.gameType = pokerClient.GetPokerGameTypeFromWindowTitle(WindowTitle);
            this.statistics = new TableStatistics(this); // We don't know what specific kind
            this.Hud = new Hud(this);
            this.visualRecognitionManager = null; // Not all tables have a visual recognition manager
            this.displayWindow = DaCMain.TableDisplayWindow.CreateForTable(this);

            // By default we use the universal parser
            handHistoryParser = new UniversalHHParser(pokerClient, System.IO.Path.GetFileName(handHistoryFilePath));

            // But as soon as we find what kind of game we're using, we're going to update our parser */
            ((UniversalHHParser)handHistoryParser).GameDiscovered += new UniversalHHParser.GameDiscoveredHandler(handHistoryParser_GameDiscovered);

            playerList = new List<Player>(10); //Usually no more than 10 players per table

            // Init hand history monitor
            hhMonitor = new HHMonitor(handHistoryFilePath, this);
            hhMonitor.StartMonitoring();
        }
Beispiel #3
0
        /// <summary>
        /// Loads statistics of a table.
        /// </summary>
        /// <param name="obj"></param>
        /// <returns></returns>
        internal override TableStatistics LoadTableStatistics(TableOrView obj)
        {
            var sql = @"
-- Data space
SELECT SUM(a.total_pages), SUM(a.used_pages), SUM(a.data_pages)
FROM sys.objects o
INNER JOIN sys.schemas s ON s.schema_id = o.schema_id
INNER JOIN sys.indexes i ON i.object_id = o.object_id AND i.type IN (0, 1)  -- heap and clustered
INNER JOIN sys.partitions p	ON p.object_id = i.object_id AND p.index_id = i.index_id
INNER JOIN sys.allocation_units a ON a.container_id = p.hobt_id
WHERE s.name = @schemaName AND o.name = @objectName

-- Index space
SELECT SUM(a.total_pages), SUM(a.used_pages), SUM(a.data_pages)
FROM sys.objects o
INNER JOIN sys.schemas s ON s.schema_id = o.schema_id
INNER JOIN sys.indexes i ON i.object_id = o.object_id AND i.type IN (2)  -- non-clustered
INNER JOIN sys.partitions p	ON p.object_id = i.object_id AND p.index_id = i.index_id
INNER JOIN sys.allocation_units a ON a.container_id = p.hobt_id
WHERE s.name = @schemaName AND o.name = @objectName

-- Row count
SELECT SUM(p.rows)
FROM sys.objects o
INNER JOIN sys.schemas s ON s.schema_id = o.schema_id
INNER JOIN sys.indexes i ON i.object_id = o.object_id AND i.type IN (0, 1)
INNER JOIN sys.partitions p	ON p.object_id = i.object_id AND p.index_id = i.index_id
WHERE s.name = @schemaName AND o.name = @objectName
";

            using (var cn = OpenConnection())
            {
                using (var cmd = new SqlCommand(sql, cn))
                {
                    cmd.Parameters.Add("@schemaName", SqlDbType.NVarChar, 128).Value = String.IsNullOrWhiteSpace(obj.SchemaName) ? (object)DBNull.Value : (object)obj.SchemaName;
                    cmd.Parameters.Add("@objectName", SqlDbType.NVarChar, 128).Value = obj.ObjectName;

                    using (var dr = cmd.ExecuteReader())
                    {
                        var stat = new TableStatistics();

                        // Data space
                        dr.Read();
                        stat.DataSpace = dr.IsDBNull(0) ? 0L : dr.GetInt64(0) * 0x2000;    // 8K pages

                        // Index space
                        dr.NextResult();
                        dr.Read();
                        stat.IndexSpace = dr.IsDBNull(0) ? 0L : dr.GetInt64(0) * 0x2000;    // 8K pages

                        // Row count
                        dr.NextResult();
                        dr.Read();
                        stat.RowCount = dr.IsDBNull(0) ? 0L : dr.GetInt64(0);

                        return(stat);
                    }
                }
            }
        }
 public void CalculateStatistics()
 {
     foreach (Table table in _relationManager.Tables)
     {
         TableStatistics statistics = GetStatistics(table.TableDefinition.Id);
         CalculateStatistic(table.TableDefinition, statistics);
     }
 }
Beispiel #5
0
 private void CopyMembers(TableStatistics old)
 {
     this.table = old.table;
     this.binCount = old.binCount;
     this.keyColumn = old.keyColumn;
     this.keyValue = new List<double>(old.keyValue);
     this.keyCount = new List<long>(old.keyCount);
     this.rowCount = old.rowCount;
 }
Beispiel #6
0
        public void TestColumns(string tableName, int[] expectedColumns)
        {
            var path = TestContext.CurrentContext.TestDirectory + "../../../../resources/" + tableName;
            var ts   = new TableStatistics();

            ts.Read(path);
            var columns = ts.Columns(2).Select(c => c.End);

            CollectionAssert.AreEqual(expectedColumns, columns);
        }
Beispiel #7
0
        public void TestContent(string tableName, string[] expectedColumns)
        {
            var path = TestContext.CurrentContext.TestDirectory + "../../../../resources/" + tableName;
            var ts   = new TableStatistics();

            ts.Read(path);
            var p   = new TableParser();
            var res = p.Read(ts, path);

            CollectionAssert.AreEqual(expectedColumns, res.Tables[0].Rows[0].Cells.Select(x => x.Value.Trim()).ToArray());
        }
        public void TestSimpleHeader()
        {
            var stats = new TableStatistics();

            Assert.AreEqual(0, stats.Stats.Count);
            const string Line = "H1 H2 H3";

            stats.AddLine(Line);
            Assert.AreEqual(Line.Length, stats.Stats.Count);
            CollectionAssert.AreEqual(new[] { 1, 1, 0, 1, 1, 0, 1, 1 }, stats.Stats);
        }
        public void TestMulti()
        {
            var stats = new TableStatistics();

            Assert.AreEqual(0, stats.Stats.Count);
            const string Line1 = "Header 1   Header 2    Header 3";
            const string Line2 = "Row 11      Row 12      Row 13";
            const string Line3 = "Row 21      Row 22      Row 23";

            stats.AddLine(Line1);
            stats.AddLine(Line2);
            stats.AddLine(Line3);
            Assert.AreEqual(Line1.Length, stats.Stats.Count);
            CollectionAssert.AreEqual(new[] { 3, 3, 3, 1, 3, 3, 0, 1, 0, 0, 0, 1, 3, 3, 3, 1, 3, 2, 1, 0, 0, 0, 0, 1, 3, 3, 3, 1, 3, 2, 1 }, stats.Stats);
        }
        private String statusColor(TableStatistics statistics)
        {
            switch (statistics.TableState.ToUpper())
            {
            case TableStates.TABLE_NOT_EXIST:
                return("bg-light");

            case TableStates.TABLE_ERROR:
                return("bg-danger");

            case TableStates.TABLE_BEING_RELOADED:
            case TableStates.BEFORE_LOAD:
                return("bg-info");

            case TableStates.TABLE_CANCELLED:
                return("bg-warning");

            case TableStates.TABLE_COMPLETED:
            case TableStates.TABLE_ALL:
            case TableStates.TABLE_UPDATES:
            case TableStates.FULL_LOAD:
                switch (statistics.ValidationState.ToUpper())
                {
                case ValidationStates.ERROR:
                case ValidationStates.MISMATCHED_RECORDS:
                case ValidationStates.TABLE_ERROR:
                    return("bg-danger");

                case ValidationStates.NO_PRIMARY_KEY:
                case ValidationStates.NOT_ENABLED:
                case ValidationStates.SUSPENDED_RECORDS:
                    return("bg-warning");

                case ValidationStates.PENDING_RECORDS:
                    return("bg-primary");

                case ValidationStates.VALIDATED:
                    return("bg-success");

                default:
                    return("bg-secondary");
                }

            default:
                return("bg-secondary");
            }
        }
Beispiel #11
0
        public ParseResult Read(TableStatistics stats, string path)
        {
            var results = new ParseResult();

            var table = results.AddTable();

            using (var sr = new StreamReader(path))
            {
                while (sr.Peek() >= 0)
                {
                    var row  = table.AddRow();
                    var line = sr.ReadLine();
                    row.Parse(stats, line);
                }
            }
            return(results);
        }
        public TableStatistics GetStatistics(int relationId)
        {
            if (_statisticsPerRelation.TryGetValue(relationId, out TableStatistics foundStatistics))
            {
                return(foundStatistics);
            }

            TableStatistics statistics = new TableStatistics()
            {
                RelationId = relationId
            };

            _statisticsPerRelation.Add(relationId, statistics);
            CalculateStatistic(_relationManager.GetRelation(relationId) as TableDefinition, statistics);

            return(statistics);
        }
Beispiel #13
0
        public override void CollectTablesForStatistics()
        {
            TableStatistics.Clear();

            if (IsPartitioned)
            {
                // Partitioning is always done on the table specified right after the FROM keyword
                // TODO: what if more than one QS?
                // *** TODO: test this here, will not work with functions etc!
                var qs = SelectStatement.EnumerateQuerySpecifications().FirstOrDefault();
                var ts = (SimpleTableSource)qs.EnumerateSourceTables(false).First();

                ts.TableReference.Statistics = new SqlParser.TableStatistics()
                {
                    Table     = ts.TableReference,
                    KeyColumn = ts.PartitioningColumnReference.ColumnName,
                };

                TableStatistics.Add(ts.TableReference);
            }
        }
Beispiel #14
0
        public async Task <string> AddTableStats(TableStatistics tableStats)
        {
            try
            {
                TableStatistics auxStats = new TableStatistics();

                auxStats.IDRestaurant   = tableStats.IDRestaurant;
                auxStats.IDTable        = tableStats.IDTable;
                auxStats.AvarageUse     = tableStats.AvarageUse;
                auxStats.DateStatistics = tableStats.DateStatistics;

                _context.TableStatistics.Add(auxStats);
                await _context.SaveChangesAsync();

                return("Restaurante actualizado");
            }
            catch
            {
                return("Error. Hubo un error al actualizar");
            }
        }
Beispiel #15
0
        internal void Parse(TableStatistics stats, string line)
        {
            if (string.IsNullOrWhiteSpace(line))
            {
                return;
            }
            var columns = stats.Columns(2);

            foreach (var column in columns)
            {
                var end = Math.Min(line.Length, column.End + 1) - column.Start;
                if (end <= 0)
                {
                    Cells.Add(new Cell(""));
                    break;
                }
                string v = line.Substring(column.Start, end).Trim();

                Cells.Add(new Cell(v));
            }
            // if (cur < line.Length)
            //   Cells.Add(new Cell(line.Substring(cur)));
        }
Beispiel #16
0
        private void InitializeMembers()
        {
            this.alias = null;

            this.isTableOrView = false;
            this.isUdf = false;
            this.isSubquery = false;
            this.isComputed = false;

            this.columnReferences = new List<ColumnReference>();
            this.statistics = null;
        }
Beispiel #17
0
        private void InitializeMembers()
        {
            this.node = null;

            this.databaseObject = null;

            this.datasetName = null;
            this.databaseName = null;
            this.schemaName = null;
            this.databaseObjectName = null;
            this.alias = null;

            this.isTableOrView = false;
            this.isUdf = false;
            this.isSubquery = false;
            this.isComputed = false;

            this.columnReferences = new List<ColumnReference>();
            //this.conditionReferences = new List<SearchConditionReference>();

            this.statistics = null;
        }
Beispiel #18
0
        private void CopyMembers(TableReference old)
        {
            this.node = old.node;

            this.databaseObject = old.databaseObject;

            this.datasetName = old.datasetName;
            this.databaseName = old.databaseName;
            this.schemaName = old.schemaName;
            this.databaseObjectName = old.databaseObjectName;
            this.alias = old.alias;

            this.isTableOrView = old.isTableOrView;
            this.isUdf = old.isUdf;
            this.isSubquery = old.isSubquery;
            this.isComputed = old.isComputed;

            // Deep copy of column references
            this.columnReferences = new List<ColumnReference>();
            foreach (var cr in old.columnReferences)
            {
                var ncr = new ColumnReference(cr)
                {
                    TableReference = this
                };
                this.columnReferences.Add(ncr);
            }
            //this.conditionReferences = new List<SearchConditionReference>(old.conditionReferences);

            this.statistics = old.statistics == null ? null : new TableStatistics(old.statistics);
        }
Beispiel #19
0
        private void handHistoryParser_GameDiscovered(string game)
        {
            Trace.WriteLine(String.Format("Game discovered! {0}", game));

            // Find to what game this game string corresponds
            Game = pokerClient.GetPokerGameFromGameDescription(game);

            bool foundParser = false;

            // Holdem?
            if (foundParser = (Game == PokerGame.Holdem)) {
                handHistoryParser = new HoldemHHParser(pokerClient, System.IO.Path.GetFileName(handHistoryFilePath));
                statistics = new HoldemTableStatistics(this);
            } else if (Game == PokerGame.Unknown) {
                Trace.WriteLine("We weren't able to find a better parser for this Game");
            }

            // If we replaced our parser, we need to register the event handlers
            if (foundParser) {
                // Generic handlers (all game types)
                handHistoryParser.PlayerIsSeated += new HHParser.PlayerIsSeatedHandler(handHistoryParser_PlayerIsSeated);
                handHistoryParser.RoundHasTerminated += new HHParser.RoundHasTerminatedHandler(handHistoryParser_RoundHasTerminated);
                handHistoryParser.NewTableHasBeenCreated += new HHParser.NewTableHasBeenCreatedHandler(handHistoryParser_NewTableHasBeenCreated);
                handHistoryParser.FoundTableMaxSeatingCapacity += new HHParser.FoundTableMaxSeatingCapacityHandler(handHistoryParser_FoundTableMaxSeatingCapacity);
                handHistoryParser.HeroNameFound += new HHParser.HeroNameFoundHandler(handHistoryParser_HeroNameFound);

                // Game specific handlers
                if (Game == PokerGame.Holdem) {
                    ((HoldemHHParser)handHistoryParser).FinalBoardAvailable += new HoldemHHParser.FinalBoardAvailableHandler(handHistoryParser_FinalBoardAvailable);
                    statistics.RegisterParserHandlers(handHistoryParser);
                }

                // Also, resend the last line to the new parser!
                hhMonitor.ResendLastLine();
            }

            if (Game != PokerGame.Unknown) {
                // Close temporary window
                if (displayWindow != null) displayWindow.Dispose();

                Globals.Director.RunFromGUIThread(
                    (Action)delegate () {
                        if (displayWindow != null) {
                            displayWindow = TableDisplayWindow.CreateForTable(this);
                            displayWindow.Show();
                        }
                    }, false
                 );
            }
        }
Beispiel #20
0
 public TableStatistics(TableStatistics old)
 {
     CopyMembers(old);
 }
        private void CalculateStatistic(TableDefinition tableDefinition, TableStatistics statistics)
        {
            PhysicalOperation operation;

            Table table = _relationManager.GetTable(tableDefinition.Id);

            if (tableDefinition.HasClusteredIndex())
            {
                operation = new IndexScanOperation(new RelationElement(tableDefinition), table, table.GetIndex(tableDefinition.GetClusteredIndex().Column));
            }
            else
            {
                operation = new TableScanOperation(new RelationElement(tableDefinition), table);
            }

            operation.Prepare();

            statistics.ColumnStatistics.Clear();

            foreach (AttributeDefinition attributeDefinition in table.TableDefinition)
            {
                statistics.ColumnStatistics.Add(attributeDefinition.Name, new ColumnStatistics()
                {
                    AttributeDefinition = attributeDefinition
                });
            }

            int i = 0;

            Dictionary <string, HashSet <object> > distinctValuesPerColumn = new Dictionary <string, HashSet <object> >();

            foreach (AttributeDefinition attributeDefinition in table.TableDefinition)
            {
                distinctValuesPerColumn.Add(attributeDefinition.Name, new HashSet <object>());
            }

            Dictionary <string, int> maxSizePerColumn = new Dictionary <string, int>();

            foreach (AttributeDefinition attributeDefinition in table.TableDefinition)
            {
                maxSizePerColumn.Add(attributeDefinition.Name, GetDefaultSize(table.TableDefinition, attributeDefinition));
            }

            CustomTuple tuple;

            do
            {
                tuple = operation.GetNext();

                if (tuple != null)
                {
                    i++;

                    foreach (AttributeDefinition attributeDefinition in table.TableDefinition)
                    {
                        object value = tuple.GetValueFor <object>(attributeDefinition.Name);

                        if (!distinctValuesPerColumn[attributeDefinition.Name].Contains(value))
                        {
                            distinctValuesPerColumn[attributeDefinition.Name].Add(value);
                        }

                        if (attributeDefinition.Type == ValueType.String &&
                            value is string s &&
                            s.Length > maxSizePerColumn[attributeDefinition.Name])
                        {
                            maxSizePerColumn[attributeDefinition.Name] = s.Length;
                        }
                    }
                }
            }while (tuple != null);

            foreach (KeyValuePair <string, HashSet <object> > distinctValue in distinctValuesPerColumn)
            {
                statistics.ColumnStatistics[distinctValue.Key].DistinctValuesCount = distinctValue.Value.Count;
            }

            foreach (KeyValuePair <string, int> maxSize in maxSizePerColumn)
            {
                statistics.ColumnStatistics[maxSize.Key].MaxSize = maxSize.Value;
            }

            statistics.Count = i;
        }
 public DisplayTableStatistics(TableStatistics tableStatistics, String displayIcon, String statusColor)
 {
     this.Statistics  = tableStatistics;
     this.DisplayIcon = displayIcon;
     this.StatusColor = statusColor;
 }
Beispiel #23
0
        private void CopyMembers(TableReference old)
        {
            this.alias = old.alias;

            this.isTableOrView = old.isTableOrView;
            this.isUdf = old.isUdf;
            this.isSubquery = old.isSubquery;
            this.isComputed = old.isComputed;

            // Deep copy of column references
            this.columnReferences = new List<ColumnReference>();
            foreach (var cr in old.columnReferences)
            {
                var ncr = new ColumnReference(cr)
                {
                    TableReference = this
                };
                this.columnReferences.Add(ncr);
            }
            this.statistics = old.statistics == null ? null : new TableStatistics(old.statistics);
        }