private void lbxFields_SelectedIndexChanged(object sender, EventArgs e)
{
btnGetUniqueValues.Enabled = true;
string field = lbxFields.SelectedItem as string;
if (field == null)
{
return;
}
IComparable min = null;
lblMin.Text = string.Empty;
IComparable max = null;
lblMax.Text = string.Empty;
if (_attributeSource != null)
{
int numPages = (int)Math.Ceiling((double)_attributeSource.NumRows() / 10000);
for (int page = 0; page < numPages; page++)
{
var table = _attributeSource.GetAttributes(page * 10000, 10000);
GetMinMax(table.Rows.Cast <DataRow>(), field, ref min, ref max);
}
}
if (_table != null)
{
GetMinMax(_table.Rows.Cast <DataRow>(), field, ref min, ref max);
}
if (min != null)
{
lblMin.Text = min.ToString();
}
if (max != null)
{
lblMax.Text = max.ToString();
}
}
/// <summary>
/// Gets a list of all unique values of the attribute field.
/// </summary>
private List<Break> GetUniqueValues(string fieldName, IAttributeSource source, ICancelProgressHandler progressHandler)
{
ArrayList lst;
bool hugeCountOk = false;
if (_cachedUniqueValues.ContainsKey(fieldName))
{
lst = _cachedUniqueValues[fieldName];
}
else
{
lst = new ArrayList();
AttributePager ap = new AttributePager(source, 5000);
ProgressMeter pm = new ProgressMeter(progressHandler, "Discovering Unique Values", source.NumRows());
for (int row = 0; row < source.NumRows(); row++)
{
object val = ap.Row(row)[fieldName] ?? "[NULL]";
if (val.ToString() == string.Empty) val = "[NULL]";
if (lst.Contains(val)) continue;
lst.Add(val);
if (lst.Count > 1000 && !hugeCountOk)
{
CancelEventArgs args = new CancelEventArgs(true);
if (TooManyCategories != null) TooManyCategories(this, args);
if (args.Cancel) break;
hugeCountOk = true;
}
pm.CurrentValue = row;
if (progressHandler.Cancel) break;
}
lst.Sort();
if (lst.Count < EditorSettings.MaxSampleCount)
{
_cachedUniqueValues[fieldName] = lst;
}
}
List<Break> result = new List<Break>();
if (lst != null)
{
foreach (object item in lst)
{
result.Add(new Break(item.ToString()));
}
}
return result;
}
/// <summary>
/// Gets the values from a file based data source rather than an in memory object.
/// </summary>
/// <param name="source">Source to get the values from.</param>
/// <param name="progressHandler">The progress handler.</param>
public void GetValues(IAttributeSource source, ICancelProgressHandler progressHandler)
{
int pageSize = 100000;
Values = new List <double>();
string normField = EditorSettings.NormField;
string fieldName = EditorSettings.FieldName;
if (source.NumRows() < EditorSettings.MaxSampleCount)
{
int numPages = (int)Math.Ceiling((double)source.NumRows() / pageSize);
for (int ipage = 0; ipage < numPages; ipage++)
{
int numRows = (ipage == numPages - 1) ? source.NumRows() % pageSize : pageSize;
DataTable table = source.GetAttributes(ipage * pageSize, numRows);
if (!string.IsNullOrEmpty(EditorSettings.ExcludeExpression))
{
DataRow[] rows = table.Select("NOT (" + EditorSettings.ExcludeExpression + ")");
foreach (DataRow row in rows)
{
double val;
if (!double.TryParse(row[fieldName].ToString(), out val))
{
continue;
}
if (double.IsNaN(val))
{
continue;
}
if (normField != null)
{
double norm;
if (!double.TryParse(row[normField].ToString(), out norm) || double.IsNaN(val))
{
continue;
}
Values.Add(val / norm);
continue;
}
Values.Add(val);
}
}
else
{
foreach (DataRow row in table.Rows)
{
double val;
if (!double.TryParse(row[fieldName].ToString(), out val))
{
continue;
}
if (double.IsNaN(val))
{
continue;
}
if (normField != null)
{
double norm;
if (!double.TryParse(row[normField].ToString(), out norm) || double.IsNaN(val))
{
continue;
}
Values.Add(val / norm);
continue;
}
Values.Add(val);
}
}
}
}
else
{
Dictionary <int, double> randomValues = new();
pageSize = 10000;
int count = EditorSettings.MaxSampleCount;
// Specified seed is required for consistently recreating the break values
Random rnd = new(9999);
AttributePager ap = new(source, pageSize);
int countPerPage = count / ap.NumPages();
ProgressMeter pm = new(progressHandler, "Sampling " + count + " random values", count);
for (int iPage = 0; iPage < ap.NumPages(); iPage++)
{
for (int i = 0; i < countPerPage; i++)
{
double val;
double norm = 1;
int index;
bool failed = false;
do
{
index = rnd.Next(ap.StartIndex, ap.StartIndex + pageSize);
DataRow dr = ap.Row(index);
if (!double.TryParse(dr[fieldName].ToString(), out val))
{
failed = true;
}
if (normField == null)
{
continue;
}
if (!double.TryParse(dr[normField].ToString(), out norm))
{
failed = true;
}
}while (randomValues.ContainsKey(index) || double.IsNaN(val) || failed);
if (normField != null)
{
Values.Add(val / norm);
}
else
{
Values.Add(val);
}
randomValues.Add(index, val);
pm.CurrentValue = i + (iPage * countPerPage);
}
if (progressHandler != null && progressHandler.Cancel)
{
break;
}
}
}
Values.Sort();
Statistics.Calculate(Values);
}
/// <summary>
/// Gets the values from a file based data source rather than an in memory object.
/// </summary>
/// <param name="source"></param>
/// <param name="progressHandler"></param>
public void GetValues(IAttributeSource source, ICancelProgressHandler progressHandler)
{
int pageSize = 100000;
Values = new List<double>();
string normField = EditorSettings.NormField;
string fieldName = EditorSettings.FieldName;
if (source.NumRows() < EditorSettings.MaxSampleCount)
{
int numPages = (int)Math.Ceiling((double)source.NumRows() / pageSize);
for (int ipage = 0; ipage < numPages; ipage++)
{
int numRows = (ipage == numPages - 1) ? source.NumRows() % pageSize : pageSize;
DataTable table = source.GetAttributes(ipage * pageSize, numRows);
if (!string.IsNullOrEmpty(EditorSettings.ExcludeExpression))
{
DataRow[] rows = table.Select("NOT (" + EditorSettings.ExcludeExpression + ")");
foreach (DataRow row in rows)
{
double val;
if (!double.TryParse(row[fieldName].ToString(), out val)) continue;
if (double.IsNaN(val)) continue;
if (normField != null)
{
double norm;
if (!double.TryParse(row[normField].ToString(), out norm) || double.IsNaN(val))
continue;
Values.Add(val / norm);
continue;
}
Values.Add(val);
}
}
else
{
foreach (DataRow row in table.Rows)
{
double val;
if (!double.TryParse(row[fieldName].ToString(), out val)) continue;
if (double.IsNaN(val)) continue;
if (normField != null)
{
double norm;
if (!double.TryParse(row[normField].ToString(), out norm) || double.IsNaN(val))
continue;
Values.Add(val / norm);
continue;
}
Values.Add(val);
}
}
}
}
else
{
Dictionary<int, double> randomValues = new Dictionary<int, double>();
pageSize = 10000;
int count = EditorSettings.MaxSampleCount;
Random rnd = new Random();
AttributePager ap = new AttributePager(source, pageSize);
int countPerPage = count / ap.NumPages();
ProgressMeter pm = new ProgressMeter(progressHandler, "Sampling " + count + " random values", count);
for (int iPage = 0; iPage < ap.NumPages(); iPage++)
{
for (int i = 0; i < countPerPage; i++)
{
double val;
double norm = 1;
int index;
bool failed = false;
do
{
index = rnd.Next(ap.StartIndex, ap.StartIndex + pageSize);
DataRow dr = ap.Row(index);
if (!double.TryParse(dr[fieldName].ToString(), out val)) failed = true;
if (normField == null) continue;
if (!double.TryParse(dr[normField].ToString(), out norm))
failed = true;
} while (randomValues.ContainsKey(index) || double.IsNaN(val) || failed);
if (normField != null)
{
Values.Add(val / norm);
}
else
{
Values.Add(val);
}
randomValues.Add(index, val);
pm.CurrentValue = i + iPage * countPerPage;
}
//Application.DoEvents();
if (progressHandler != null && progressHandler.Cancel)
{
break;
}
}
}
Values.Sort();
Statistics.Calculate(Values);
}
/// <summary>
/// Gets a list of all unique values of the attribute field.
/// </summary>
/// <param name="fieldName">Name of the field that gets checked.</param>
/// <param name="source">Attribute source that contains the field.</param>
/// <param name="progressHandler">The progress handler.</param>
/// <returns>A list with the unique values.</returns>
private List <Break> GetUniqueValues(string fieldName, IAttributeSource source, ICancelProgressHandler progressHandler)
{
ArrayList lst;
bool hugeCountOk = false;
if (_cachedUniqueValues.ContainsKey(fieldName))
{
lst = _cachedUniqueValues[fieldName];
}
else
{
lst = new ArrayList();
AttributePager ap = new(source, 5000);
ProgressMeter pm = new(progressHandler, "Discovering Unique Values", source.NumRows());
for (int row = 0; row < source.NumRows(); row++)
{
object val = ap.Row(row)[fieldName] ?? "[NULL]";
if (val.ToString() == string.Empty)
{
val = "[NULL]";
}
if (lst.Contains(val))
{
continue;
}
lst.Add(val);
if (lst.Count > 1000 && !hugeCountOk)
{
CancelEventArgs args = new(true);
TooManyCategories?.Invoke(this, args);
if (args.Cancel)
{
break;
}
hugeCountOk = true;
}
pm.CurrentValue = row;
if (progressHandler.Cancel)
{
break;
}
}
lst.Sort();
if (lst.Count < EditorSettings.MaxSampleCount)
{
_cachedUniqueValues[fieldName] = lst;
}
}
List <Break> result = new();
if (lst != null)
{
foreach (object item in lst)
{
result.Add(new Break(item.ToString()));
}
}
return(result);
}
private void BtnGetUniqueValuesClick(object sender, EventArgs e)
{
// Sorting should be done as the original objects, not as strings.
var lst = new HashSet <object>();
string fieldName = lbxFields.SelectedItem.ToString();
bool useAll = false;
bool isString = true;
if (_attributeSource != null)
{
isString = _attributeSource.GetColumn(fieldName).DataType == typeof(string);
int numPages = (int)Math.Ceiling((double)_attributeSource.NumRows() / 10000);
for (int page = 0; page < numPages; page++)
{
DataTable table = _attributeSource.GetAttributes(page * 10000, 10000);
foreach (DataRow dr in table.Rows)
{
if (dr[fieldName] is DBNull)
{
continue;
}
if (lst.Contains(dr[fieldName]))
{
continue;
}
lst.Add(dr[fieldName]);
}
if (lst.Count <= 10000 || useAll)
{
continue;
}
if (MessageBox.Show(SymbologyFormsMessageStrings.SQLQueryControl_MoreThan10000UniqueValues, SymbologyFormsMessageStrings.SQLQueryControl_LargeNumberOfUniqueValues, MessageBoxButtons.YesNo) == DialogResult.Yes)
{
useAll = true;
}
else
{
break;
}
}
}
else if (_table != null)
{
isString = _table.Columns[fieldName].DataType == typeof(string);
foreach (DataRow dr in _table.Rows)
{
if (dr[fieldName] is DBNull)
{
continue;
}
if (lst.Contains(dr[fieldName]))
{
continue;
}
lst.Add(dr[fieldName]);
}
}
var text = new object[lst.Count];
int i = 0;
foreach (var o in lst.OrderBy(_ => _))
{
if (isString)
{
text[i++] = "'" + ((string)o).Replace("'", "''") + "'";
}
else
{
text[i++] = o.ToString();
}
}
lbxUniqueValues.SuspendLayout();
lbxUniqueValues.Items.Clear();
lbxUniqueValues.Items.AddRange(text);
lbxUniqueValues.ResumeLayout();
lbxUniqueValues.Enabled = true;
lbxUniqueValues.BackColor = Color.White;
btnGetUniqueValues.Enabled = false;
}
/// <summary>
/// Creates a new instance of AttributePager
/// </summary>
public AttributePager(IAttributeSource source, int pageSize)
{
_numRows = source.NumRows();
_source = source;
_pageSize = pageSize;
}
/// <summary>
/// Gets a list of all unique values of the attribute field.
/// </summary>
private List<Break> GetUniqueValues(string fieldName, IAttributeSource source, ICancelProgressHandler progressHandler)
{
ArrayList lst;
bool _hugeCountOk = false;
if(_cachedUniqueValues.ContainsKey(fieldName))
{
lst = _cachedUniqueValues[fieldName];
}
else
{
lst = new ArrayList();
AttributePager ap = new AttributePager(source, 5000);
ProgressMeter pm = new ProgressMeter(progressHandler, "Discovering Unique Values", source.NumRows());
for (int row = 0; row < source.NumRows(); row++)
{
object val = ap.Row(row)[fieldName] ?? "[NULL]";
if (val.ToString() == "") val = "[NULL]";
if (lst.Contains(val)) continue;
lst.Add(val);
if (lst.Count > 1000 && !_hugeCountOk)
{
if (MessageBox.Show(
"There are more than 1000 unique values, which can result in very slow performance. Do you wish to try to create unique categories from all the values anyway?",
"Extreme Number of Categories", MessageBoxButtons.YesNo) == DialogResult.No)
{
break;
}
_hugeCountOk = true;
}
pm.CurrentValue = row;
if (progressHandler.Cancel) break;
}
lst.Sort();
if (lst.Count < EditorSettings.MaxSampleCount)
{
_cachedUniqueValues[fieldName] = lst;
}
}
List<Break> result = new List<Break>();
if (lst != null)
{
foreach (object item in lst)
{
result.Add(new Break(item.ToString()));
}
}
return result;
}
private void lbxFields_SelectedIndexChanged(object sender, EventArgs e)
{
btnGetUniqueValues.Enabled = true;
string field = lbxFields.SelectedItem as string;
if (field == null)
{
return;
}
IComparable min = null;
lblMin.Text = string.Empty;
IComparable max = null;
lblMax.Text = string.Empty;
if (_attributeSource != null)
{
int numPages = (int)Math.Ceiling((double)_attributeSource.NumRows() / 10000);
for (int page = 0; page < numPages; page++)
{
DataTable table = _attributeSource.GetAttributes(page * 10000, 10000);
foreach (DataRow dr in table.Rows)
{
if ((dr[field] is DBNull))
{
continue;
}
if (min == null)
{
min = dr[field] as IComparable;
}
else
{
if (min.CompareTo(dr[field]) > 0)
{
min = dr[field] as IComparable;
}
}
if (max == null)
{
max = dr[field] as IComparable;
}
else
{
if (max.CompareTo(dr[field]) < 0)
{
max = dr[field] as IComparable;
}
}
}
}
}
if (_table != null)
{
foreach (DataRow dr in _table.Rows)
{
if ((dr[field] is DBNull))
{
continue;
}
if (min == null)
{
min = dr[field] as IComparable;
}
else
{
if (min.CompareTo(dr[field]) > 0)
{
min = dr[field] as IComparable;
}
}
if (max == null)
{
max = dr[field] as IComparable;
}
else
{
if (max.CompareTo(dr[field]) < 0)
{
max = dr[field] as IComparable;
}
}
}
}
if (min != null)
{
lblMin.Text = min.ToString();
}
if (max != null)
{
lblMax.Text = max.ToString();
}
}