public async Task RunAsync(CancellationToken token)
{
_logger.LogInformation("RfmTrainingWorker.RunAsync");
IReadOnlyList <string> tableNames = _options.TableNames;
List <Task <TableStatistics> > tableStatisticsTasks = new List <Task <TableStatistics> >(tableNames.Count);
foreach (string tableName in tableNames)
{
tableStatisticsTasks.Add(this._tableStore.GetTableStatisticsAsync(tableName, token));
}
TableStatistics[] tableStatisticsArray = await Task.WhenAll(tableStatisticsTasks).ConfigureAwait(false);
List <TableDefinition> tableDefinitionList = new List <TableDefinition>(tableStatisticsTasks.Count);
for (int index = 0; index < tableStatisticsTasks.Count; ++index)
{
TableStatistics result = tableStatisticsTasks[index].Result;
if (result == null)
{
this._logger.LogWarning(string.Format("Statistics data for {0} table could not be retrieved. It will not participate in model training.", (object)tableNames[index]));
}
else
{
tableDefinitionList.Add(result.Definition);
}
}
ModelStatistics modelStatistics = await _model.TrainAsync(_options.SchemaName, token, tableDefinitionList.ToArray()).ConfigureAwait(false);
if (modelStatistics != null)
{
await UpdateRfmFacets(modelStatistics as RfmStatistics, token);
}
}
public Table(String handHistoryFilePath, Window window, PokerClient pokerClient, PlayerDatabase playerDatabase)
{
this.handHistoryFilePath = handHistoryFilePath;
this.window = window;
this.pokerClient = pokerClient;
this.playerDatabase = playerDatabase;
this.Game = PokerGame.Unknown;
this.maxSeatingCapacity = 0; // We don't know yet
this.TableId = String.Empty; // We don't know yet
this.GameID = String.Empty; // We don't know yet
this.currentHeroName = String.Empty;
this.currentHeroSeat = 0;
this.gameType = pokerClient.GetPokerGameTypeFromWindowTitle(WindowTitle);
this.statistics = new TableStatistics(this); // We don't know what specific kind
this.Hud = new Hud(this);
this.visualRecognitionManager = null; // Not all tables have a visual recognition manager
this.displayWindow = DaCMain.TableDisplayWindow.CreateForTable(this);
// By default we use the universal parser
handHistoryParser = new UniversalHHParser(pokerClient, System.IO.Path.GetFileName(handHistoryFilePath));
// But as soon as we find what kind of game we're using, we're going to update our parser */
((UniversalHHParser)handHistoryParser).GameDiscovered += new UniversalHHParser.GameDiscoveredHandler(handHistoryParser_GameDiscovered);
playerList = new List<Player>(10); //Usually no more than 10 players per table
// Init hand history monitor
hhMonitor = new HHMonitor(handHistoryFilePath, this);
hhMonitor.StartMonitoring();
}
/// <summary>
/// Loads statistics of a table.
/// </summary>
/// <param name="obj"></param>
/// <returns></returns>
internal override TableStatistics LoadTableStatistics(TableOrView obj)
{
var sql = @"
-- Data space
SELECT SUM(a.total_pages), SUM(a.used_pages), SUM(a.data_pages)
FROM sys.objects o
INNER JOIN sys.schemas s ON s.schema_id = o.schema_id
INNER JOIN sys.indexes i ON i.object_id = o.object_id AND i.type IN (0, 1) -- heap and clustered
INNER JOIN sys.partitions p ON p.object_id = i.object_id AND p.index_id = i.index_id
INNER JOIN sys.allocation_units a ON a.container_id = p.hobt_id
WHERE s.name = @schemaName AND o.name = @objectName
-- Index space
SELECT SUM(a.total_pages), SUM(a.used_pages), SUM(a.data_pages)
FROM sys.objects o
INNER JOIN sys.schemas s ON s.schema_id = o.schema_id
INNER JOIN sys.indexes i ON i.object_id = o.object_id AND i.type IN (2) -- non-clustered
INNER JOIN sys.partitions p ON p.object_id = i.object_id AND p.index_id = i.index_id
INNER JOIN sys.allocation_units a ON a.container_id = p.hobt_id
WHERE s.name = @schemaName AND o.name = @objectName
-- Row count
SELECT SUM(p.rows)
FROM sys.objects o
INNER JOIN sys.schemas s ON s.schema_id = o.schema_id
INNER JOIN sys.indexes i ON i.object_id = o.object_id AND i.type IN (0, 1)
INNER JOIN sys.partitions p ON p.object_id = i.object_id AND p.index_id = i.index_id
WHERE s.name = @schemaName AND o.name = @objectName
";
using (var cn = OpenConnection())
{
using (var cmd = new SqlCommand(sql, cn))
{
cmd.Parameters.Add("@schemaName", SqlDbType.NVarChar, 128).Value = String.IsNullOrWhiteSpace(obj.SchemaName) ? (object)DBNull.Value : (object)obj.SchemaName;
cmd.Parameters.Add("@objectName", SqlDbType.NVarChar, 128).Value = obj.ObjectName;
using (var dr = cmd.ExecuteReader())
{
var stat = new TableStatistics();
// Data space
dr.Read();
stat.DataSpace = dr.IsDBNull(0) ? 0L : dr.GetInt64(0) * 0x2000; // 8K pages
// Index space
dr.NextResult();
dr.Read();
stat.IndexSpace = dr.IsDBNull(0) ? 0L : dr.GetInt64(0) * 0x2000; // 8K pages
// Row count
dr.NextResult();
dr.Read();
stat.RowCount = dr.IsDBNull(0) ? 0L : dr.GetInt64(0);
return(stat);
}
}
}
}
public void CalculateStatistics()
{
foreach (Table table in _relationManager.Tables)
{
TableStatistics statistics = GetStatistics(table.TableDefinition.Id);
CalculateStatistic(table.TableDefinition, statistics);
}
}
private void CopyMembers(TableStatistics old)
{
this.table = old.table;
this.binCount = old.binCount;
this.keyColumn = old.keyColumn;
this.keyValue = new List<double>(old.keyValue);
this.keyCount = new List<long>(old.keyCount);
this.rowCount = old.rowCount;
}
public void TestColumns(string tableName, int[] expectedColumns)
{
var path = TestContext.CurrentContext.TestDirectory + "../../../../resources/" + tableName;
var ts = new TableStatistics();
ts.Read(path);
var columns = ts.Columns(2).Select(c => c.End);
CollectionAssert.AreEqual(expectedColumns, columns);
}
public void TestContent(string tableName, string[] expectedColumns)
{
var path = TestContext.CurrentContext.TestDirectory + "../../../../resources/" + tableName;
var ts = new TableStatistics();
ts.Read(path);
var p = new TableParser();
var res = p.Read(ts, path);
CollectionAssert.AreEqual(expectedColumns, res.Tables[0].Rows[0].Cells.Select(x => x.Value.Trim()).ToArray());
}
public void TestSimpleHeader()
{
var stats = new TableStatistics();
Assert.AreEqual(0, stats.Stats.Count);
const string Line = "H1 H2 H3";
stats.AddLine(Line);
Assert.AreEqual(Line.Length, stats.Stats.Count);
CollectionAssert.AreEqual(new[] { 1, 1, 0, 1, 1, 0, 1, 1 }, stats.Stats);
}
public void TestMulti()
{
var stats = new TableStatistics();
Assert.AreEqual(0, stats.Stats.Count);
const string Line1 = "Header 1 Header 2 Header 3";
const string Line2 = "Row 11 Row 12 Row 13";
const string Line3 = "Row 21 Row 22 Row 23";
stats.AddLine(Line1);
stats.AddLine(Line2);
stats.AddLine(Line3);
Assert.AreEqual(Line1.Length, stats.Stats.Count);
CollectionAssert.AreEqual(new[] { 3, 3, 3, 1, 3, 3, 0, 1, 0, 0, 0, 1, 3, 3, 3, 1, 3, 2, 1, 0, 0, 0, 0, 1, 3, 3, 3, 1, 3, 2, 1 }, stats.Stats);
}
private String statusColor(TableStatistics statistics)
{
switch (statistics.TableState.ToUpper())
{
case TableStates.TABLE_NOT_EXIST:
return("bg-light");
case TableStates.TABLE_ERROR:
return("bg-danger");
case TableStates.TABLE_BEING_RELOADED:
case TableStates.BEFORE_LOAD:
return("bg-info");
case TableStates.TABLE_CANCELLED:
return("bg-warning");
case TableStates.TABLE_COMPLETED:
case TableStates.TABLE_ALL:
case TableStates.TABLE_UPDATES:
case TableStates.FULL_LOAD:
switch (statistics.ValidationState.ToUpper())
{
case ValidationStates.ERROR:
case ValidationStates.MISMATCHED_RECORDS:
case ValidationStates.TABLE_ERROR:
return("bg-danger");
case ValidationStates.NO_PRIMARY_KEY:
case ValidationStates.NOT_ENABLED:
case ValidationStates.SUSPENDED_RECORDS:
return("bg-warning");
case ValidationStates.PENDING_RECORDS:
return("bg-primary");
case ValidationStates.VALIDATED:
return("bg-success");
default:
return("bg-secondary");
}
default:
return("bg-secondary");
}
}
public ParseResult Read(TableStatistics stats, string path)
{
var results = new ParseResult();
var table = results.AddTable();
using (var sr = new StreamReader(path))
{
while (sr.Peek() >= 0)
{
var row = table.AddRow();
var line = sr.ReadLine();
row.Parse(stats, line);
}
}
return(results);
}
public TableStatistics GetStatistics(int relationId)
{
if (_statisticsPerRelation.TryGetValue(relationId, out TableStatistics foundStatistics))
{
return(foundStatistics);
}
TableStatistics statistics = new TableStatistics()
{
RelationId = relationId
};
_statisticsPerRelation.Add(relationId, statistics);
CalculateStatistic(_relationManager.GetRelation(relationId) as TableDefinition, statistics);
return(statistics);
}
public override void CollectTablesForStatistics()
{
TableStatistics.Clear();
if (IsPartitioned)
{
// Partitioning is always done on the table specified right after the FROM keyword
// TODO: what if more than one QS?
// *** TODO: test this here, will not work with functions etc!
var qs = SelectStatement.EnumerateQuerySpecifications().FirstOrDefault();
var ts = (SimpleTableSource)qs.EnumerateSourceTables(false).First();
ts.TableReference.Statistics = new SqlParser.TableStatistics()
{
Table = ts.TableReference,
KeyColumn = ts.PartitioningColumnReference.ColumnName,
};
TableStatistics.Add(ts.TableReference);
}
}
public async Task <string> AddTableStats(TableStatistics tableStats)
{
try
{
TableStatistics auxStats = new TableStatistics();
auxStats.IDRestaurant = tableStats.IDRestaurant;
auxStats.IDTable = tableStats.IDTable;
auxStats.AvarageUse = tableStats.AvarageUse;
auxStats.DateStatistics = tableStats.DateStatistics;
_context.TableStatistics.Add(auxStats);
await _context.SaveChangesAsync();
return("Restaurante actualizado");
}
catch
{
return("Error. Hubo un error al actualizar");
}
}
internal void Parse(TableStatistics stats, string line)
{
if (string.IsNullOrWhiteSpace(line))
{
return;
}
var columns = stats.Columns(2);
foreach (var column in columns)
{
var end = Math.Min(line.Length, column.End + 1) - column.Start;
if (end <= 0)
{
Cells.Add(new Cell(""));
break;
}
string v = line.Substring(column.Start, end).Trim();
Cells.Add(new Cell(v));
}
// if (cur < line.Length)
// Cells.Add(new Cell(line.Substring(cur)));
}
private void InitializeMembers()
{
this.alias = null;
this.isTableOrView = false;
this.isUdf = false;
this.isSubquery = false;
this.isComputed = false;
this.columnReferences = new List<ColumnReference>();
this.statistics = null;
}
private void InitializeMembers()
{
this.node = null;
this.databaseObject = null;
this.datasetName = null;
this.databaseName = null;
this.schemaName = null;
this.databaseObjectName = null;
this.alias = null;
this.isTableOrView = false;
this.isUdf = false;
this.isSubquery = false;
this.isComputed = false;
this.columnReferences = new List<ColumnReference>();
//this.conditionReferences = new List<SearchConditionReference>();
this.statistics = null;
}
private void CopyMembers(TableReference old)
{
this.node = old.node;
this.databaseObject = old.databaseObject;
this.datasetName = old.datasetName;
this.databaseName = old.databaseName;
this.schemaName = old.schemaName;
this.databaseObjectName = old.databaseObjectName;
this.alias = old.alias;
this.isTableOrView = old.isTableOrView;
this.isUdf = old.isUdf;
this.isSubquery = old.isSubquery;
this.isComputed = old.isComputed;
// Deep copy of column references
this.columnReferences = new List<ColumnReference>();
foreach (var cr in old.columnReferences)
{
var ncr = new ColumnReference(cr)
{
TableReference = this
};
this.columnReferences.Add(ncr);
}
//this.conditionReferences = new List<SearchConditionReference>(old.conditionReferences);
this.statistics = old.statistics == null ? null : new TableStatistics(old.statistics);
}
private void handHistoryParser_GameDiscovered(string game)
{
Trace.WriteLine(String.Format("Game discovered! {0}", game));
// Find to what game this game string corresponds
Game = pokerClient.GetPokerGameFromGameDescription(game);
bool foundParser = false;
// Holdem?
if (foundParser = (Game == PokerGame.Holdem)) {
handHistoryParser = new HoldemHHParser(pokerClient, System.IO.Path.GetFileName(handHistoryFilePath));
statistics = new HoldemTableStatistics(this);
} else if (Game == PokerGame.Unknown) {
Trace.WriteLine("We weren't able to find a better parser for this Game");
}
// If we replaced our parser, we need to register the event handlers
if (foundParser) {
// Generic handlers (all game types)
handHistoryParser.PlayerIsSeated += new HHParser.PlayerIsSeatedHandler(handHistoryParser_PlayerIsSeated);
handHistoryParser.RoundHasTerminated += new HHParser.RoundHasTerminatedHandler(handHistoryParser_RoundHasTerminated);
handHistoryParser.NewTableHasBeenCreated += new HHParser.NewTableHasBeenCreatedHandler(handHistoryParser_NewTableHasBeenCreated);
handHistoryParser.FoundTableMaxSeatingCapacity += new HHParser.FoundTableMaxSeatingCapacityHandler(handHistoryParser_FoundTableMaxSeatingCapacity);
handHistoryParser.HeroNameFound += new HHParser.HeroNameFoundHandler(handHistoryParser_HeroNameFound);
// Game specific handlers
if (Game == PokerGame.Holdem) {
((HoldemHHParser)handHistoryParser).FinalBoardAvailable += new HoldemHHParser.FinalBoardAvailableHandler(handHistoryParser_FinalBoardAvailable);
statistics.RegisterParserHandlers(handHistoryParser);
}
// Also, resend the last line to the new parser!
hhMonitor.ResendLastLine();
}
if (Game != PokerGame.Unknown) {
// Close temporary window
if (displayWindow != null) displayWindow.Dispose();
Globals.Director.RunFromGUIThread(
(Action)delegate () {
if (displayWindow != null) {
displayWindow = TableDisplayWindow.CreateForTable(this);
displayWindow.Show();
}
}, false
);
}
}
public TableStatistics(TableStatistics old)
{
CopyMembers(old);
}
private void CalculateStatistic(TableDefinition tableDefinition, TableStatistics statistics)
{
PhysicalOperation operation;
Table table = _relationManager.GetTable(tableDefinition.Id);
if (tableDefinition.HasClusteredIndex())
{
operation = new IndexScanOperation(new RelationElement(tableDefinition), table, table.GetIndex(tableDefinition.GetClusteredIndex().Column));
}
else
{
operation = new TableScanOperation(new RelationElement(tableDefinition), table);
}
operation.Prepare();
statistics.ColumnStatistics.Clear();
foreach (AttributeDefinition attributeDefinition in table.TableDefinition)
{
statistics.ColumnStatistics.Add(attributeDefinition.Name, new ColumnStatistics()
{
AttributeDefinition = attributeDefinition
});
}
int i = 0;
Dictionary <string, HashSet <object> > distinctValuesPerColumn = new Dictionary <string, HashSet <object> >();
foreach (AttributeDefinition attributeDefinition in table.TableDefinition)
{
distinctValuesPerColumn.Add(attributeDefinition.Name, new HashSet <object>());
}
Dictionary <string, int> maxSizePerColumn = new Dictionary <string, int>();
foreach (AttributeDefinition attributeDefinition in table.TableDefinition)
{
maxSizePerColumn.Add(attributeDefinition.Name, GetDefaultSize(table.TableDefinition, attributeDefinition));
}
CustomTuple tuple;
do
{
tuple = operation.GetNext();
if (tuple != null)
{
i++;
foreach (AttributeDefinition attributeDefinition in table.TableDefinition)
{
object value = tuple.GetValueFor <object>(attributeDefinition.Name);
if (!distinctValuesPerColumn[attributeDefinition.Name].Contains(value))
{
distinctValuesPerColumn[attributeDefinition.Name].Add(value);
}
if (attributeDefinition.Type == ValueType.String &&
value is string s &&
s.Length > maxSizePerColumn[attributeDefinition.Name])
{
maxSizePerColumn[attributeDefinition.Name] = s.Length;
}
}
}
}while (tuple != null);
foreach (KeyValuePair <string, HashSet <object> > distinctValue in distinctValuesPerColumn)
{
statistics.ColumnStatistics[distinctValue.Key].DistinctValuesCount = distinctValue.Value.Count;
}
foreach (KeyValuePair <string, int> maxSize in maxSizePerColumn)
{
statistics.ColumnStatistics[maxSize.Key].MaxSize = maxSize.Value;
}
statistics.Count = i;
}
public DisplayTableStatistics(TableStatistics tableStatistics, String displayIcon, String statusColor)
{
this.Statistics = tableStatistics;
this.DisplayIcon = displayIcon;
this.StatusColor = statusColor;
}
private void CopyMembers(TableReference old)
{
this.alias = old.alias;
this.isTableOrView = old.isTableOrView;
this.isUdf = old.isUdf;
this.isSubquery = old.isSubquery;
this.isComputed = old.isComputed;
// Deep copy of column references
this.columnReferences = new List<ColumnReference>();
foreach (var cr in old.columnReferences)
{
var ncr = new ColumnReference(cr)
{
TableReference = this
};
this.columnReferences.Add(ncr);
}
this.statistics = old.statistics == null ? null : new TableStatistics(old.statistics);
}