private ColumnStats GetColumnStatsXYZ(DataColumn <string> col)
{
ColumnStats stats = new ColumnStats();
List <float> vals = new List <float>();
// try convert to float columns then run stats
foreach (string v in col.GetDataList())
{
float val = float.MinValue;
bool parsed = float.TryParse(v, out val);
if (parsed)
{
vals.Add(val);
}
}
if (vals.Count > 0)
{
DataColumn <float> dcf = new DataColumn <float>("tmp");
dcf.SetDataList(vals);
stats = GetColumnStatsXYZ(dcf);
}
return(stats);
}
public static Bitmap GetHistogramBitmap(ColumnStats stats)
{
Bitmap bmp = new Bitmap(stats.Buckets, 150);
Graphics g = Graphics.FromImage(bmp);
g.Clear(Color.FromArgb(128, 68, 82, 105));
Pen pen = new Pen(Color.FromArgb(127, 137, 157));
double logMax = Math.Log(stats.HistogramMax);
if (stats.Histogram != null)
{
for (int i = 0; i < stats.Histogram.Length; i++)
{
double height = Math.Log(stats.Histogram[i]) / logMax;
if (height < 0)
{
height = 0;
}
g.DrawLine(Pens.White, new System.Drawing.Point(i, 150), new System.Drawing.Point(i, (int)(150 - (height * 150))));
}
}
pen.Dispose();
g.Flush();
g.Dispose();
return(bmp);
}
private ColumnStats GetColumnStatsXYZ(DataColumn <float> col)
{
ColumnStats stats = new ColumnStats();
//float total = 0;
float min = float.MaxValue;
float max = float.MinValue;
int ct = 0;
foreach (float cv in col.GetDataList())
{
ct++;
min = Math.Min(cv, min);
max = Math.Max(cv, max);
// total += cv;
}
float diff = max - min;
stats.min = min;
stats.max = max;
stats.diff = diff;
stats.count = ct;
stats.message = "Min " + min + ", max " + max + ", length: " + diff + " ,samples counted " + ct;
return(stats);
}
public void ComputeChart()
{
if (chartType == ChartTypes.Histogram)
{
if (!Stats.Computed)
{
Stats = layer.GetSingleColumnHistogram(TargetColumn);
}
}
else if (chartType == ChartTypes.BarChart)
{
if (!Stats.Computed)
{
Stats = layer.GetDomainValueBarChart(domainColumn, targetColumn, denominatorColumn, statType);
}
}
else if (chartType == ChartTypes.TimeChart)
{
if (!Stats.Computed)
{
Stats = layer.GetDateHistogram(TargetColumn, DateFilter);
}
}
}
private Thrift.ColumnChunk Write(SchemaElement schema, IList values,
CompressionMethod compression,
ColumnStats stats)
{
Thrift.ColumnChunk chunk = _meta.AddColumnChunk(compression, _output, schema, values.Count);
Thrift.PageHeader ph = _meta.CreateDataPage(values.Count);
List <PageTag> pages = WriteValues(schema, values, ph, compression, stats);
//the following counters must include both data size and header size
chunk.Meta_data.Total_compressed_size = pages.Sum(p => p.HeaderMeta.Compressed_page_size + p.HeaderSize);
chunk.Meta_data.Total_uncompressed_size = pages.Sum(p => p.HeaderMeta.Uncompressed_page_size + p.HeaderSize);
return(chunk);
}
private Bitmap GetChartImageBitmap(Earth3d window)
{
Bitmap bmp = null;
if (chartType == ChartTypes.Histogram)
{
if (!Stats.Computed)
{
Stats = layer.GetSingleColumnHistogram(TargetColumn);
}
bmp = GetBarChartBitmap(Stats);
texture = Texture11.FromBitmap(bmp, 0);
}
else if (chartType == ChartTypes.BarChart)
{
if (!Stats.Computed)
{
Stats = layer.GetDomainValueBarChart(domainColumn, targetColumn, denominatorColumn, statType);
}
bmp = GetBarChartBitmap(Stats);
texture = Texture11.FromBitmap(bmp, 0);
}
else if (chartType == ChartTypes.TimeChart)
{
if (!Stats.Computed)
{
Stats = layer.GetDateHistogram(TargetColumn, DateFilter);
}
bmp = GetBarChartBitmap(Stats);
texture = Texture11.FromBitmap(bmp, 0);
}
Width = bmp.Width;
Height = bmp.Height;
Top = (int)window.RenderContext11.ViewPort.Height - (Height + 120);
Left = (int)window.RenderContext11.ViewPort.Width / 2 - (Width / 2);
return(bmp);
}
private Thrift.ColumnChunk Write(SchemaElement schema, IList values,
CompressionMethod compression,
ColumnStats stats)
{
Thrift.ColumnChunk chunk = _meta.AddColumnChunk(compression, _output, schema, values.Count);
var ph = new Thrift.PageHeader(Thrift.PageType.DATA_PAGE, 0, 0);
ph.Data_page_header = new Thrift.DataPageHeader
{
Encoding = Thrift.Encoding.PLAIN,
Definition_level_encoding = Thrift.Encoding.RLE,
Repetition_level_encoding = Thrift.Encoding.BIT_PACKED,
Num_values = values.Count
};
WriteValues(schema, values, ph, compression, stats);
return(chunk);
}
internal ColumnStats GetXYZColumnData(int colNum)
{
ColumnStats stats = null;
try
{
Object ob = columnData[colNum];
if (ob.GetType() == floatTypeColumn)
{
DataColumn <float> col = (DataColumn <float>)ob;
stats = GetColumnStatsXYZ(col);
}
else
{
DataColumn <string> col = (DataColumn <string>)ob;
stats = GetColumnStatsXYZ(col);
}
}
catch (Exception ex) { }
return(stats);
}
/// <summary>
/// Carry out the block model import
/// </summary>
/// <param name="SelectedBMFile"></param>
/// <param name="SelectedFormatBMFile"></param>
/// <param name="importMap"></param>
/// <param name="rawFileReader"></param>
/// <returns></returns>
internal bool DoBMImport(string SelectedBMFile, string SelectedFormatBMFile, ImportDataMap importMap, RawFileReader rawFileReader, string NKDProjectID, string modelAlias)
{
BaseImportTools bit = new BaseImportTools();
int cxColumnID = importMap.GetColumnIDMappedTo("CentroidX");
int cyColumnID = importMap.GetColumnIDMappedTo("CentroidY");
int czColumnID = importMap.GetColumnIDMappedTo("CentroidZ");
ColumnStats xOrigin = rawFileReader.GetDimensions(cxColumnID);
ColumnStats yOrigin = rawFileReader.GetDimensions(cyColumnID);
ColumnStats zOrigin = rawFileReader.GetDimensions(czColumnID);
int approxNumLines = xOrigin.count;
Stream bmFileStream = new FileStream(SelectedBMFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
// Stream bmFileStream = new FileStream(SelectedBMFile, FileMode.Open);
ModelImportStatus mos = new ModelImportStatus();
Guid newModelGuid = Guid.NewGuid();
Guid authorGuid = new Guid();
List <string> status = bit.PerformBMImport(mos, newModelGuid, bmFileStream, null, importMap, xOrigin.min, yOrigin.min, zOrigin.min, backgroundWorker, approxNumLines, NKDProjectID, modelAlias, authorGuid, ConnectionString);
return(true);
}
private UIElement GetColumnVisual(ColumnStats column, Style headerStyle, Style promptStyle, Style valueStyle, Style sliderStyle)
{
Grid grid = new Grid();// { HorizontalAlignment = System.Windows.HorizontalAlignment.Stretch, Background = Brushes.Yellow, };
grid.ColumnDefinitions.Add(new ColumnDefinition() { Width = new GridLength(10) });
grid.ColumnDefinitions.Add(new ColumnDefinition() { Width = new GridLength(1, GridUnitType.Auto), SharedSizeGroup = "itemPrompt" });
grid.ColumnDefinitions.Add(new ColumnDefinition() { Width = new GridLength(5) });
grid.ColumnDefinitions.Add(new ColumnDefinition() { Width = new GridLength(1, GridUnitType.Star), SharedSizeGroup = "itemValue" });
#region header
TextBlock text = new TextBlock()
{
Text = column.Name,
Style = headerStyle,
};
grid.RowDefinitions.Add(new RowDefinition() { Height = new GridLength() });
Grid.SetColumn(text, 0);
Grid.SetColumnSpan(text, 4);
Grid.SetRow(text, grid.RowDefinitions.Count - 1);
grid.Children.Add(text);
#endregion
#region datatype
// Type
text = new TextBlock()
{
Text = column.FieldStats.FieldType.ToString(),
Style = promptStyle,
};
grid.RowDefinitions.Add(new RowDefinition() { Height = new GridLength(2) });
grid.RowDefinitions.Add(new RowDefinition() { Height = new GridLength() });
Grid.SetColumn(text, 1);
Grid.SetRow(text, grid.RowDefinitions.Count - 1);
grid.Children.Add(text);
// Checkbox
if (column.FieldStats.FieldType != SOMFieldType.AlphaNumeric && column.FieldStats.FieldType != SOMFieldType.AnyText)
{
CheckBox checkbox = new CheckBox()
{
Content = "Force text",
Tag = column,
};
checkbox.Checked += ColumnTextCheckbox_Checked;
checkbox.Unchecked += ColumnTextCheckbox_Checked;
Grid.SetColumn(checkbox, 3);
Grid.SetRow(checkbox, grid.RowDefinitions.Count - 1);
grid.Children.Add(checkbox);
}
#endregion
#region vector width
// Prompt
text = new TextBlock()
{
Text = "vector width",
Style = promptStyle,
};
grid.RowDefinitions.Add(new RowDefinition() { Height = new GridLength(2) });
grid.RowDefinitions.Add(new RowDefinition() { Height = new GridLength() });
Grid.SetColumn(text, 1);
Grid.SetRow(text, grid.RowDefinitions.Count - 1);
grid.Children.Add(text);
// Value
SliderShowValues slider = new SliderShowValues()
{
Minimum = 1,
Maximum = 10,
Value = 5,
IsInteger = true,
Tag = column,
Style = sliderStyle,
};
slider.ValueChanged += ColumnSlider_ValueChanged;
Grid.SetColumn(slider, 3);
Grid.SetRow(slider, grid.RowDefinitions.Count - 1);
grid.Children.Add(slider);
#endregion
// counts
AddDescriptionRow(grid, "count", column.FieldStats.Count.ToString("N0"), promptStyle, valueStyle);
AddDescriptionRow(grid, "unique", column.FieldStats.UniqueCount.ToString("N0"), promptStyle, valueStyle);
#region datatype specific fields
switch (column.FieldStats.FieldType)
{
case SOMFieldType.AlphaNumeric:
case SOMFieldType.AnyText:
ShowUsedColumnStats_Text(grid, column.FieldStats, promptStyle, valueStyle);
break;
case SOMFieldType.DateTime:
ShowUsedColumnStats_Datetime(grid, column.FieldStats, promptStyle, valueStyle);
break;
case SOMFieldType.FloatingPoint:
case SOMFieldType.Integer:
ShowUsedColumnStats_Number(grid, column.FieldStats, promptStyle, valueStyle);
break;
default:
throw new ApplicationException("Unknown SOMFieldType: " + column.FieldStats.FieldType.ToString());
}
#endregion
return new Border()
{
Child = grid,
Padding = new Thickness(4),
};
}
private static ColumnSetStats DoWork_ColumnStats(ColumnStatsRequest req, CancellationToken cancel)
{
var indices = GetIndices(req.Names, req.Results.ColumnNames);
if (indices.Item2 != null)
{
return new ColumnSetStats(req.Names, indices.Item2);
}
ColumnStats[] columns = new ColumnStats[req.Names.Length];
for (int cntr = 0; cntr < req.Names.Length; cntr++)
{
int index = indices.Item1[cntr];
string[] values = req.Results.Results.
Select(o => o[index]).
ToArray();
SOMFieldStats stats = SelfOrganizingMapsDB.GetFieldStats(values);
SOMFieldStats textStats = null;
switch (stats.FieldType)
{
case SOMFieldType.AlphaNumeric:
case SOMFieldType.AnyText:
textStats = stats;
break;
default:
textStats = SelfOrganizingMapsDB.GetFieldStats(values, SOMFieldType.AnyText);
break;
}
columns[cntr] = new ColumnStats(req.Names[cntr], index, stats, textStats);
}
return new ColumnSetStats(columns);
}
public static RowInput[] GetSOMInputs(ColumnStats[] columns, QueryResults results, bool randomSample)
{
var finalColumns = GetFinalColumnStats(columns);
IEnumerable<int> indices = randomSample ?
UtilityCore.RandomRange(0, results.Results.Length, StaticRandom.Next(100, 200)) :
Enumerable.Range(0, results.Results.Length);
return indices.
Select(o =>
{
return new RowInput()
{
Row = results.Results[o],
Weights = finalColumns.
Select(p => SelfOrganizingMapsDB.ConvertToVector(results.Results[o][p.Item1.Index], p.Item2, p.Item3)).
SelectMany(p => p).
ToArray(),
};
}).
ToArray();
}
public ColumnSetStats(ColumnStats[] columns)
{
this.Names = columns.Select(o => o.Name).ToArray();
this.Columns = columns;
this.Exception = null;
}
private static Tuple<ColumnStats, SOMFieldStats, SOMConvertToVectorProps>[] GetFinalColumnStats(ColumnStats[] columns)
{
var retVal = new Tuple<ColumnStats, SOMFieldStats, SOMConvertToVectorProps>[columns.Length];
for (int cntr = 0; cntr < columns.Length; cntr++)
{
// Field
SOMFieldStats field = null;
if (columns[cntr].ForceText)
{
field = columns[cntr].FieldStatsText;
}
else
{
field = columns[cntr].FieldStats;
}
//TODO: look at overrides
// Convert
SOMConvertToVectorProps convertProps;
switch (field.FieldType)
{
case SOMFieldType.AlphaNumeric:
case SOMFieldType.AnyText:
convertProps = new SOMConvertToVectorProps(columns[cntr].Width, columns[cntr].Override_Text_Justify ?? TextAlignment.Center);
break;
case SOMFieldType.DateTime:
convertProps = SelfOrganizingMapsDB.GetConvertToProps(field.Date_Min.Value, field.Date_Max.Value, columns[cntr].Width);
break;
case SOMFieldType.FloatingPoint:
case SOMFieldType.Integer:
convertProps = SelfOrganizingMapsDB.GetConvertToProps(field.Numeric_Min.Value, field.Numeric_Max.Value, columns[cntr].Width);
break;
default:
throw new ApplicationException("Unknown SOMFieldType: " + field.FieldType.ToString());
}
// Build it
retVal[cntr] = Tuple.Create(columns[cntr], field, convertProps);
}
return retVal;
}
public Bitmap GetBarChartBitmap(ColumnStats stats)
{
int Chrome = 25;
int height = 150;
int count = Math.Min(MaxUnits, stats.Buckets);
int border = 10;
int colWidth = Math.Min(30, (int)(1000 / count));
int width = count * colWidth;
Width = width + 2 * border;
Bitmap bmp = new Bitmap(Width, height + border * 2 + 20 + Chrome);
Graphics g = Graphics.FromImage(bmp);
g.Clear(Color.FromArgb(128, 5, 75, 35));
bool anythingSelected = false;
double selectedAmount = 0;
double totalAmount = 0;
if (stats.Selected != null)
{
for (int i = 0; i < stats.Buckets; i++)
{
if (stats.Selected[i])
{
anythingSelected = true;
selectedAmount += stats.Histogram[i];
}
totalAmount += stats.Histogram[i];
}
}
// Draw title
string text = (stats.DomainColumn > -1 ? layer.Table.Header[stats.DomainColumn] + " : " : "") + layer.Table.Header[stats.TargetColumn] + " " + stats.DomainStatType.ToString() + ((stats.DomainStatType == StatTypes.Ratio) ? (" to " + layer.Table.Header[stats.DemoninatorColumn]) : "");
title = text;
if (anythingSelected && stats.DomainStatType != StatTypes.Ratio)
{
text += String.Format(" {0:p1} Selected", selectedAmount / totalAmount);
}
g.DrawString(text, UiTools.StandardGargantuan, Brushes.White, new PointF(border, 0));
string sort = "AZ";
switch (sortType)
{
case 0:
sort = "AZ";
break;
case 1:
sort = "ZA";
break;
case 2:
sort = "09";
break;
case 3:
sort = "90";
break;
}
if (chartType == ChartTypes.BarChart)
{
g.DrawString(sort, UiTools.StandardLarge, Brushes.White, new PointF(Width - 25, 0));
}
System.Drawing.StringFormat drawFormat = new System.Drawing.StringFormat();
drawFormat.FormatFlags = StringFormatFlags.NoWrap | StringFormatFlags.DirectionVertical;
drawFormat.Alignment = StringAlignment.Near;
drawFormat.LineAlignment = StringAlignment.Center;
SolidBrush brush = new SolidBrush(Color.FromArgb(20, 128, 255));
Brush selectedBrush = Brushes.Yellow;
//Brushes.White;
Pen pen = new Pen(Color.FromArgb(20, 128, 255));
double logMax = Math.Log(stats.HistogramMax);
int end = Math.Min(stats.Buckets, ScrollPosition + MaxUnits);
if (stats.Histogram != null)
{
barHitTest = new Rectangle[stats.Buckets];
for (int i = ScrollPosition; i < end; i++)
{
int pos = i - ScrollPosition;
double val = stats.Histogram[i] / (stats.HistogramMax * 1.05);
if (val < 0)
{
val = 0;
}
barHitTest[i] = new Rectangle((int)(pos * colWidth) + border, border + Chrome, colWidth, (int)(height));
Rectangle rect = new Rectangle((int)(pos * colWidth) + border, (int)(height - (val * height)) + border + Chrome, colWidth, (int)(val * height));
if (stats.Selected[i])
{
g.FillRectangle(selectedBrush, rect);
}
else
{
g.FillRectangle(brush, rect);
}
if (stats.DomainColumn > -1 || (stats.DomainValues != null && stats.DomainValues.Length > pos))
{
g.DrawString(stats.DomainValues[i], UiTools.StandardLarge, Brushes.White, new RectangleF((pos * colWidth) + border, border + Chrome, colWidth, height), drawFormat);
}
}
ScrollBarVisible = false;
if (MaxUnits < stats.Buckets)
{
int ScrollAreaWidth = Width - (2 * border);
// Scroll bars are needed
ScrollBarVisible = true;
int scrollWidth = (int)((double)MaxUnits / (double)stats.Buckets * ScrollAreaWidth) + 2;
int scrollStart = (int)((double)ScrollPosition / (double)stats.Buckets * ScrollAreaWidth);
scrollUnitPixelRatio = (double)ScrollAreaWidth / (double)stats.Buckets;
g.DrawLine(Pens.White, new Point(border, height + 22 + Chrome), new Point(border + ScrollAreaWidth, height + 22 + Chrome));
Rectangle rect = new Rectangle(border + scrollStart, height + 15 + Chrome, scrollWidth, 15);
g.FillRectangle(brush, rect);
}
}
brush.Dispose();
pen.Dispose();
g.Flush();
g.Dispose();
return(bmp);
}
private void WriteValues(SchemaElement schema, IList values, Thrift.PageHeader ph, CompressionMethod compression, ColumnStats stats)
{
byte[] data;
using (var ms = new MemoryStream())
{
using (var writer = new BinaryWriter(ms))
{
if (stats.NullCount > 0)
{
CreateDefinitions(values, schema, out IList newValues, out List <int> definitions);
values = newValues;
_rleWriter.Write(writer, schema, definitions);
}
_plainWriter.Write(writer, schema, values);
data = ms.ToArray();
}
}
ph.Uncompressed_page_size = data.Length;
if (compression != CompressionMethod.None)
{
IDataWriter writer = DataFactory.GetWriter(compression);
using (var ms = new MemoryStream())
{
writer.Write(data, ms);
data = ms.ToArray();
}
ph.Compressed_page_size = data.Length;
}
else
{
ph.Compressed_page_size = ph.Uncompressed_page_size;
}
_thrift.Write(ph);
_output.Write(data, 0, data.Length);
}
private List <PageTag> WriteValues(SchemaElement schema, IList values, Thrift.PageHeader ph, CompressionMethod compression, ColumnStats stats)
{
var result = new List <PageTag>();
byte[] dictionaryPageBytes = null;
int dictionaryPageCount = 0;
byte[] dataPageBytes;
//flatten values if the field is repeatable
if (schema.IsRepeated)
{
values = FlattenRepeatables(values, schema);
}
using (var ms = new MemoryStream())
{
using (var writer = new BinaryWriter(ms))
{
//write repetitions
if (schema.IsRepeated)
{
List <int> repetitions = CreateRepetitions(values, schema);
_rleWriter.Write(writer, _definitionsSchema, repetitions, out IList nullExtra);
}
//write definitions
if (schema.HasNulls || schema.IsRepeated)
{
CreateDefinitions(values, schema, out IList newValues, out List <int> definitions);
values = newValues;
_rleWriter.Write(writer, _definitionsSchema, definitions, out IList nullExtra);
}
//write data
if (!_writerOptions.UseDictionaryEncoding || !_dicWriter.Write(writer, schema, values, out IList dicValues))
{
_plainWriter.Write(writer, schema, values, out IList plainExtra);
}
else
{
dictionaryPageCount = dicValues.Count;
ph.Data_page_header.Encoding = Thrift.Encoding.PLAIN_DICTIONARY;
using (var dms = new MemoryStream())
using (var dwriter = new BinaryWriter(dms))
{
_plainWriter.Write(dwriter, schema, dicValues, out IList t0);
dictionaryPageBytes = dms.ToArray();
}
}
dataPageBytes = ms.ToArray();
}
}
if (dictionaryPageBytes != null)
{
Thrift.PageHeader dph = _meta.CreateDictionaryPage(dictionaryPageCount);
dictionaryPageBytes = Compress(dph, dictionaryPageBytes, compression);
int dictionaryHeaderSize = Write(dph, dictionaryPageBytes);
result.Add(new PageTag {
HeaderSize = dictionaryHeaderSize, HeaderMeta = dph
});
}
dataPageBytes = Compress(ph, dataPageBytes, compression);
int dataHeaderSize = Write(ph, dataPageBytes);
result.Add(new PageTag {
HeaderSize = dataHeaderSize, HeaderMeta = ph
});
return(result);
}
/// <summary>
/// Search the mapped columns to find the specified column, and determine the min
/// </summary>
/// <param name="NKDColumnID"></param>
/// <returns></returns>
internal ColumnStats GetDimensions(int columnID)
{
ColumnStats cs = columnManager.GetXYZColumnData(columnID);
return(cs);
}
