private ColumnStats GetColumnStatsXYZ(DataColumn <string> col) { ColumnStats stats = new ColumnStats(); List <float> vals = new List <float>(); // try convert to float columns then run stats foreach (string v in col.GetDataList()) { float val = float.MinValue; bool parsed = float.TryParse(v, out val); if (parsed) { vals.Add(val); } } if (vals.Count > 0) { DataColumn <float> dcf = new DataColumn <float>("tmp"); dcf.SetDataList(vals); stats = GetColumnStatsXYZ(dcf); } return(stats); }
public static Bitmap GetHistogramBitmap(ColumnStats stats) { Bitmap bmp = new Bitmap(stats.Buckets, 150); Graphics g = Graphics.FromImage(bmp); g.Clear(Color.FromArgb(128, 68, 82, 105)); Pen pen = new Pen(Color.FromArgb(127, 137, 157)); double logMax = Math.Log(stats.HistogramMax); if (stats.Histogram != null) { for (int i = 0; i < stats.Histogram.Length; i++) { double height = Math.Log(stats.Histogram[i]) / logMax; if (height < 0) { height = 0; } g.DrawLine(Pens.White, new System.Drawing.Point(i, 150), new System.Drawing.Point(i, (int)(150 - (height * 150)))); } } pen.Dispose(); g.Flush(); g.Dispose(); return(bmp); }
private ColumnStats GetColumnStatsXYZ(DataColumn <float> col) { ColumnStats stats = new ColumnStats(); //float total = 0; float min = float.MaxValue; float max = float.MinValue; int ct = 0; foreach (float cv in col.GetDataList()) { ct++; min = Math.Min(cv, min); max = Math.Max(cv, max); // total += cv; } float diff = max - min; stats.min = min; stats.max = max; stats.diff = diff; stats.count = ct; stats.message = "Min " + min + ", max " + max + ", length: " + diff + " ,samples counted " + ct; return(stats); }
public void ComputeChart() { if (chartType == ChartTypes.Histogram) { if (!Stats.Computed) { Stats = layer.GetSingleColumnHistogram(TargetColumn); } } else if (chartType == ChartTypes.BarChart) { if (!Stats.Computed) { Stats = layer.GetDomainValueBarChart(domainColumn, targetColumn, denominatorColumn, statType); } } else if (chartType == ChartTypes.TimeChart) { if (!Stats.Computed) { Stats = layer.GetDateHistogram(TargetColumn, DateFilter); } } }
private Thrift.ColumnChunk Write(SchemaElement schema, IList values, CompressionMethod compression, ColumnStats stats) { Thrift.ColumnChunk chunk = _meta.AddColumnChunk(compression, _output, schema, values.Count); Thrift.PageHeader ph = _meta.CreateDataPage(values.Count); List <PageTag> pages = WriteValues(schema, values, ph, compression, stats); //the following counters must include both data size and header size chunk.Meta_data.Total_compressed_size = pages.Sum(p => p.HeaderMeta.Compressed_page_size + p.HeaderSize); chunk.Meta_data.Total_uncompressed_size = pages.Sum(p => p.HeaderMeta.Uncompressed_page_size + p.HeaderSize); return(chunk); }
private Bitmap GetChartImageBitmap(Earth3d window) { Bitmap bmp = null; if (chartType == ChartTypes.Histogram) { if (!Stats.Computed) { Stats = layer.GetSingleColumnHistogram(TargetColumn); } bmp = GetBarChartBitmap(Stats); texture = Texture11.FromBitmap(bmp, 0); } else if (chartType == ChartTypes.BarChart) { if (!Stats.Computed) { Stats = layer.GetDomainValueBarChart(domainColumn, targetColumn, denominatorColumn, statType); } bmp = GetBarChartBitmap(Stats); texture = Texture11.FromBitmap(bmp, 0); } else if (chartType == ChartTypes.TimeChart) { if (!Stats.Computed) { Stats = layer.GetDateHistogram(TargetColumn, DateFilter); } bmp = GetBarChartBitmap(Stats); texture = Texture11.FromBitmap(bmp, 0); } Width = bmp.Width; Height = bmp.Height; Top = (int)window.RenderContext11.ViewPort.Height - (Height + 120); Left = (int)window.RenderContext11.ViewPort.Width / 2 - (Width / 2); return(bmp); }
private Thrift.ColumnChunk Write(SchemaElement schema, IList values, CompressionMethod compression, ColumnStats stats) { Thrift.ColumnChunk chunk = _meta.AddColumnChunk(compression, _output, schema, values.Count); var ph = new Thrift.PageHeader(Thrift.PageType.DATA_PAGE, 0, 0); ph.Data_page_header = new Thrift.DataPageHeader { Encoding = Thrift.Encoding.PLAIN, Definition_level_encoding = Thrift.Encoding.RLE, Repetition_level_encoding = Thrift.Encoding.BIT_PACKED, Num_values = values.Count }; WriteValues(schema, values, ph, compression, stats); return(chunk); }
internal ColumnStats GetXYZColumnData(int colNum) { ColumnStats stats = null; try { Object ob = columnData[colNum]; if (ob.GetType() == floatTypeColumn) { DataColumn <float> col = (DataColumn <float>)ob; stats = GetColumnStatsXYZ(col); } else { DataColumn <string> col = (DataColumn <string>)ob; stats = GetColumnStatsXYZ(col); } } catch (Exception ex) { } return(stats); }
/// <summary> /// Carry out the block model import /// </summary> /// <param name="SelectedBMFile"></param> /// <param name="SelectedFormatBMFile"></param> /// <param name="importMap"></param> /// <param name="rawFileReader"></param> /// <returns></returns> internal bool DoBMImport(string SelectedBMFile, string SelectedFormatBMFile, ImportDataMap importMap, RawFileReader rawFileReader, string NKDProjectID, string modelAlias) { BaseImportTools bit = new BaseImportTools(); int cxColumnID = importMap.GetColumnIDMappedTo("CentroidX"); int cyColumnID = importMap.GetColumnIDMappedTo("CentroidY"); int czColumnID = importMap.GetColumnIDMappedTo("CentroidZ"); ColumnStats xOrigin = rawFileReader.GetDimensions(cxColumnID); ColumnStats yOrigin = rawFileReader.GetDimensions(cyColumnID); ColumnStats zOrigin = rawFileReader.GetDimensions(czColumnID); int approxNumLines = xOrigin.count; Stream bmFileStream = new FileStream(SelectedBMFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); // Stream bmFileStream = new FileStream(SelectedBMFile, FileMode.Open); ModelImportStatus mos = new ModelImportStatus(); Guid newModelGuid = Guid.NewGuid(); Guid authorGuid = new Guid(); List <string> status = bit.PerformBMImport(mos, newModelGuid, bmFileStream, null, importMap, xOrigin.min, yOrigin.min, zOrigin.min, backgroundWorker, approxNumLines, NKDProjectID, modelAlias, authorGuid, ConnectionString); return(true); }
private UIElement GetColumnVisual(ColumnStats column, Style headerStyle, Style promptStyle, Style valueStyle, Style sliderStyle) { Grid grid = new Grid();// { HorizontalAlignment = System.Windows.HorizontalAlignment.Stretch, Background = Brushes.Yellow, }; grid.ColumnDefinitions.Add(new ColumnDefinition() { Width = new GridLength(10) }); grid.ColumnDefinitions.Add(new ColumnDefinition() { Width = new GridLength(1, GridUnitType.Auto), SharedSizeGroup = "itemPrompt" }); grid.ColumnDefinitions.Add(new ColumnDefinition() { Width = new GridLength(5) }); grid.ColumnDefinitions.Add(new ColumnDefinition() { Width = new GridLength(1, GridUnitType.Star), SharedSizeGroup = "itemValue" }); #region header TextBlock text = new TextBlock() { Text = column.Name, Style = headerStyle, }; grid.RowDefinitions.Add(new RowDefinition() { Height = new GridLength() }); Grid.SetColumn(text, 0); Grid.SetColumnSpan(text, 4); Grid.SetRow(text, grid.RowDefinitions.Count - 1); grid.Children.Add(text); #endregion #region datatype // Type text = new TextBlock() { Text = column.FieldStats.FieldType.ToString(), Style = promptStyle, }; grid.RowDefinitions.Add(new RowDefinition() { Height = new GridLength(2) }); grid.RowDefinitions.Add(new RowDefinition() { Height = new GridLength() }); Grid.SetColumn(text, 1); Grid.SetRow(text, grid.RowDefinitions.Count - 1); grid.Children.Add(text); // Checkbox if (column.FieldStats.FieldType != SOMFieldType.AlphaNumeric && column.FieldStats.FieldType != SOMFieldType.AnyText) { CheckBox checkbox = new CheckBox() { Content = "Force text", Tag = column, }; checkbox.Checked += ColumnTextCheckbox_Checked; checkbox.Unchecked += ColumnTextCheckbox_Checked; Grid.SetColumn(checkbox, 3); Grid.SetRow(checkbox, grid.RowDefinitions.Count - 1); grid.Children.Add(checkbox); } #endregion #region vector width // Prompt text = new TextBlock() { Text = "vector width", Style = promptStyle, }; grid.RowDefinitions.Add(new RowDefinition() { Height = new GridLength(2) }); grid.RowDefinitions.Add(new RowDefinition() { Height = new GridLength() }); Grid.SetColumn(text, 1); Grid.SetRow(text, grid.RowDefinitions.Count - 1); grid.Children.Add(text); // Value SliderShowValues slider = new SliderShowValues() { Minimum = 1, Maximum = 10, Value = 5, IsInteger = true, Tag = column, Style = sliderStyle, }; slider.ValueChanged += ColumnSlider_ValueChanged; Grid.SetColumn(slider, 3); Grid.SetRow(slider, grid.RowDefinitions.Count - 1); grid.Children.Add(slider); #endregion // counts AddDescriptionRow(grid, "count", column.FieldStats.Count.ToString("N0"), promptStyle, valueStyle); AddDescriptionRow(grid, "unique", column.FieldStats.UniqueCount.ToString("N0"), promptStyle, valueStyle); #region datatype specific fields switch (column.FieldStats.FieldType) { case SOMFieldType.AlphaNumeric: case SOMFieldType.AnyText: ShowUsedColumnStats_Text(grid, column.FieldStats, promptStyle, valueStyle); break; case SOMFieldType.DateTime: ShowUsedColumnStats_Datetime(grid, column.FieldStats, promptStyle, valueStyle); break; case SOMFieldType.FloatingPoint: case SOMFieldType.Integer: ShowUsedColumnStats_Number(grid, column.FieldStats, promptStyle, valueStyle); break; default: throw new ApplicationException("Unknown SOMFieldType: " + column.FieldStats.FieldType.ToString()); } #endregion return new Border() { Child = grid, Padding = new Thickness(4), }; }
private static ColumnSetStats DoWork_ColumnStats(ColumnStatsRequest req, CancellationToken cancel) { var indices = GetIndices(req.Names, req.Results.ColumnNames); if (indices.Item2 != null) { return new ColumnSetStats(req.Names, indices.Item2); } ColumnStats[] columns = new ColumnStats[req.Names.Length]; for (int cntr = 0; cntr < req.Names.Length; cntr++) { int index = indices.Item1[cntr]; string[] values = req.Results.Results. Select(o => o[index]). ToArray(); SOMFieldStats stats = SelfOrganizingMapsDB.GetFieldStats(values); SOMFieldStats textStats = null; switch (stats.FieldType) { case SOMFieldType.AlphaNumeric: case SOMFieldType.AnyText: textStats = stats; break; default: textStats = SelfOrganizingMapsDB.GetFieldStats(values, SOMFieldType.AnyText); break; } columns[cntr] = new ColumnStats(req.Names[cntr], index, stats, textStats); } return new ColumnSetStats(columns); }
public static RowInput[] GetSOMInputs(ColumnStats[] columns, QueryResults results, bool randomSample) { var finalColumns = GetFinalColumnStats(columns); IEnumerable<int> indices = randomSample ? UtilityCore.RandomRange(0, results.Results.Length, StaticRandom.Next(100, 200)) : Enumerable.Range(0, results.Results.Length); return indices. Select(o => { return new RowInput() { Row = results.Results[o], Weights = finalColumns. Select(p => SelfOrganizingMapsDB.ConvertToVector(results.Results[o][p.Item1.Index], p.Item2, p.Item3)). SelectMany(p => p). ToArray(), }; }). ToArray(); }
public ColumnSetStats(ColumnStats[] columns) { this.Names = columns.Select(o => o.Name).ToArray(); this.Columns = columns; this.Exception = null; }
private static Tuple<ColumnStats, SOMFieldStats, SOMConvertToVectorProps>[] GetFinalColumnStats(ColumnStats[] columns) { var retVal = new Tuple<ColumnStats, SOMFieldStats, SOMConvertToVectorProps>[columns.Length]; for (int cntr = 0; cntr < columns.Length; cntr++) { // Field SOMFieldStats field = null; if (columns[cntr].ForceText) { field = columns[cntr].FieldStatsText; } else { field = columns[cntr].FieldStats; } //TODO: look at overrides // Convert SOMConvertToVectorProps convertProps; switch (field.FieldType) { case SOMFieldType.AlphaNumeric: case SOMFieldType.AnyText: convertProps = new SOMConvertToVectorProps(columns[cntr].Width, columns[cntr].Override_Text_Justify ?? TextAlignment.Center); break; case SOMFieldType.DateTime: convertProps = SelfOrganizingMapsDB.GetConvertToProps(field.Date_Min.Value, field.Date_Max.Value, columns[cntr].Width); break; case SOMFieldType.FloatingPoint: case SOMFieldType.Integer: convertProps = SelfOrganizingMapsDB.GetConvertToProps(field.Numeric_Min.Value, field.Numeric_Max.Value, columns[cntr].Width); break; default: throw new ApplicationException("Unknown SOMFieldType: " + field.FieldType.ToString()); } // Build it retVal[cntr] = Tuple.Create(columns[cntr], field, convertProps); } return retVal; }
public Bitmap GetBarChartBitmap(ColumnStats stats) { int Chrome = 25; int height = 150; int count = Math.Min(MaxUnits, stats.Buckets); int border = 10; int colWidth = Math.Min(30, (int)(1000 / count)); int width = count * colWidth; Width = width + 2 * border; Bitmap bmp = new Bitmap(Width, height + border * 2 + 20 + Chrome); Graphics g = Graphics.FromImage(bmp); g.Clear(Color.FromArgb(128, 5, 75, 35)); bool anythingSelected = false; double selectedAmount = 0; double totalAmount = 0; if (stats.Selected != null) { for (int i = 0; i < stats.Buckets; i++) { if (stats.Selected[i]) { anythingSelected = true; selectedAmount += stats.Histogram[i]; } totalAmount += stats.Histogram[i]; } } // Draw title string text = (stats.DomainColumn > -1 ? layer.Table.Header[stats.DomainColumn] + " : " : "") + layer.Table.Header[stats.TargetColumn] + " " + stats.DomainStatType.ToString() + ((stats.DomainStatType == StatTypes.Ratio) ? (" to " + layer.Table.Header[stats.DemoninatorColumn]) : ""); title = text; if (anythingSelected && stats.DomainStatType != StatTypes.Ratio) { text += String.Format(" {0:p1} Selected", selectedAmount / totalAmount); } g.DrawString(text, UiTools.StandardGargantuan, Brushes.White, new PointF(border, 0)); string sort = "AZ"; switch (sortType) { case 0: sort = "AZ"; break; case 1: sort = "ZA"; break; case 2: sort = "09"; break; case 3: sort = "90"; break; } if (chartType == ChartTypes.BarChart) { g.DrawString(sort, UiTools.StandardLarge, Brushes.White, new PointF(Width - 25, 0)); } System.Drawing.StringFormat drawFormat = new System.Drawing.StringFormat(); drawFormat.FormatFlags = StringFormatFlags.NoWrap | StringFormatFlags.DirectionVertical; drawFormat.Alignment = StringAlignment.Near; drawFormat.LineAlignment = StringAlignment.Center; SolidBrush brush = new SolidBrush(Color.FromArgb(20, 128, 255)); Brush selectedBrush = Brushes.Yellow; //Brushes.White; Pen pen = new Pen(Color.FromArgb(20, 128, 255)); double logMax = Math.Log(stats.HistogramMax); int end = Math.Min(stats.Buckets, ScrollPosition + MaxUnits); if (stats.Histogram != null) { barHitTest = new Rectangle[stats.Buckets]; for (int i = ScrollPosition; i < end; i++) { int pos = i - ScrollPosition; double val = stats.Histogram[i] / (stats.HistogramMax * 1.05); if (val < 0) { val = 0; } barHitTest[i] = new Rectangle((int)(pos * colWidth) + border, border + Chrome, colWidth, (int)(height)); Rectangle rect = new Rectangle((int)(pos * colWidth) + border, (int)(height - (val * height)) + border + Chrome, colWidth, (int)(val * height)); if (stats.Selected[i]) { g.FillRectangle(selectedBrush, rect); } else { g.FillRectangle(brush, rect); } if (stats.DomainColumn > -1 || (stats.DomainValues != null && stats.DomainValues.Length > pos)) { g.DrawString(stats.DomainValues[i], UiTools.StandardLarge, Brushes.White, new RectangleF((pos * colWidth) + border, border + Chrome, colWidth, height), drawFormat); } } ScrollBarVisible = false; if (MaxUnits < stats.Buckets) { int ScrollAreaWidth = Width - (2 * border); // Scroll bars are needed ScrollBarVisible = true; int scrollWidth = (int)((double)MaxUnits / (double)stats.Buckets * ScrollAreaWidth) + 2; int scrollStart = (int)((double)ScrollPosition / (double)stats.Buckets * ScrollAreaWidth); scrollUnitPixelRatio = (double)ScrollAreaWidth / (double)stats.Buckets; g.DrawLine(Pens.White, new Point(border, height + 22 + Chrome), new Point(border + ScrollAreaWidth, height + 22 + Chrome)); Rectangle rect = new Rectangle(border + scrollStart, height + 15 + Chrome, scrollWidth, 15); g.FillRectangle(brush, rect); } } brush.Dispose(); pen.Dispose(); g.Flush(); g.Dispose(); return(bmp); }
private void WriteValues(SchemaElement schema, IList values, Thrift.PageHeader ph, CompressionMethod compression, ColumnStats stats) { byte[] data; using (var ms = new MemoryStream()) { using (var writer = new BinaryWriter(ms)) { if (stats.NullCount > 0) { CreateDefinitions(values, schema, out IList newValues, out List <int> definitions); values = newValues; _rleWriter.Write(writer, schema, definitions); } _plainWriter.Write(writer, schema, values); data = ms.ToArray(); } } ph.Uncompressed_page_size = data.Length; if (compression != CompressionMethod.None) { IDataWriter writer = DataFactory.GetWriter(compression); using (var ms = new MemoryStream()) { writer.Write(data, ms); data = ms.ToArray(); } ph.Compressed_page_size = data.Length; } else { ph.Compressed_page_size = ph.Uncompressed_page_size; } _thrift.Write(ph); _output.Write(data, 0, data.Length); }
private List <PageTag> WriteValues(SchemaElement schema, IList values, Thrift.PageHeader ph, CompressionMethod compression, ColumnStats stats) { var result = new List <PageTag>(); byte[] dictionaryPageBytes = null; int dictionaryPageCount = 0; byte[] dataPageBytes; //flatten values if the field is repeatable if (schema.IsRepeated) { values = FlattenRepeatables(values, schema); } using (var ms = new MemoryStream()) { using (var writer = new BinaryWriter(ms)) { //write repetitions if (schema.IsRepeated) { List <int> repetitions = CreateRepetitions(values, schema); _rleWriter.Write(writer, _definitionsSchema, repetitions, out IList nullExtra); } //write definitions if (schema.HasNulls || schema.IsRepeated) { CreateDefinitions(values, schema, out IList newValues, out List <int> definitions); values = newValues; _rleWriter.Write(writer, _definitionsSchema, definitions, out IList nullExtra); } //write data if (!_writerOptions.UseDictionaryEncoding || !_dicWriter.Write(writer, schema, values, out IList dicValues)) { _plainWriter.Write(writer, schema, values, out IList plainExtra); } else { dictionaryPageCount = dicValues.Count; ph.Data_page_header.Encoding = Thrift.Encoding.PLAIN_DICTIONARY; using (var dms = new MemoryStream()) using (var dwriter = new BinaryWriter(dms)) { _plainWriter.Write(dwriter, schema, dicValues, out IList t0); dictionaryPageBytes = dms.ToArray(); } } dataPageBytes = ms.ToArray(); } } if (dictionaryPageBytes != null) { Thrift.PageHeader dph = _meta.CreateDictionaryPage(dictionaryPageCount); dictionaryPageBytes = Compress(dph, dictionaryPageBytes, compression); int dictionaryHeaderSize = Write(dph, dictionaryPageBytes); result.Add(new PageTag { HeaderSize = dictionaryHeaderSize, HeaderMeta = dph }); } dataPageBytes = Compress(ph, dataPageBytes, compression); int dataHeaderSize = Write(ph, dataPageBytes); result.Add(new PageTag { HeaderSize = dataHeaderSize, HeaderMeta = ph }); return(result); }
/// <summary> /// Search the mapped columns to find the specified column, and determine the min /// </summary> /// <param name="NKDColumnID"></param> /// <returns></returns> internal ColumnStats GetDimensions(int columnID) { ColumnStats cs = columnManager.GetXYZColumnData(columnID); return(cs); }