/// <inheritdoc />
public int[] GetCounts(string[] expressions, ICancelProgressHandler progressHandler, int maxSampleSize)
{
int numSelected = Count;
int[] counts = new int[expressions.Length];
bool requiresRun = false;
for (int ie = 0; ie < expressions.Length; ie++)
{
if (string.IsNullOrEmpty(expressions[ie]))
{
counts[ie] = numSelected;
}
else
{
requiresRun = true;
}
}
if (!requiresRun)
{
return(counts);
}
AttributePager ap = new AttributePager(_layer.DataSet, 100000);
int numinTable = 0;
DataTable result = new DataTable();
result.Columns.AddRange(_layer.DataSet.GetColumns());
FastDrawnState[] drawnStates = _layer.DrawnStates;
for (int shp = 0; shp < drawnStates.Length; shp++)
{
if (drawnStates[shp].Selected)
{
result.Rows.Add(ap.Row(shp).ItemArray);
numinTable++;
if (numinTable > 100000)
{
for (int ie = 0; ie < expressions.Length; ie++)
{
if (string.IsNullOrEmpty(expressions[ie]))
{
continue;
}
counts[ie] += result.Select(expressions[ie]).Length;
}
result.Clear();
}
}
}
for (int ie = 0; ie < expressions.Length; ie++)
{
if (string.IsNullOrEmpty(expressions[ie]))
{
continue;
}
counts[ie] += result.Select(expressions[ie]).Length;
}
result.Clear();
return(counts);
}
/// <summary>
/// Gets the count of members that match the expression
/// </summary>
/// <param name="expressions">The string expression to test</param>
/// <param name="progressHandler">THe progress handler that can also cancel the counting</param>
/// <param name="maxSampleSize">The integer maximum sample size from which to draw counts. If this is negative, it will not be used.</param>
/// <returns>The integer count of the members that match the expression.</returns>
public override int[] GetCounts(string[] expressions, ICancelProgressHandler progressHandler, int maxSampleSize)
{
if (AttributesPopulated) return base.GetCounts(expressions, progressHandler, maxSampleSize);
int[] counts = new int[expressions.Length];
// The most common case would be no filter expression, in which case the count is simply the number of shapes.
bool requiresRun = false;
for (int iex = 0; iex < expressions.Length; iex++)
{
if (!string.IsNullOrEmpty(expressions[iex]))
{
requiresRun = true;
}
else
{
counts[iex] = NumRows();
}
}
if (!requiresRun) return counts;
AttributePager ap = new AttributePager(this, 5000);
ProgressMeter pm = new ProgressMeter(progressHandler, "Calculating Counts", ap.NumPages());
// Don't bother to use a sampling approach if the number of rows is on the same order of magnitude as the number of samples.
if (maxSampleSize > 0 && maxSampleSize < NumRows() / 2)
{
DataTable sample = new DataTable();
sample.Columns.AddRange(GetColumns());
Dictionary<int, int> usedRows = new Dictionary<int, int>();
int samplesPerPage = maxSampleSize / ap.NumPages();
Random rnd = new Random(DateTime.Now.Millisecond);
for (int page = 0; page < ap.NumPages(); page++)
{
for (int i = 0; i < samplesPerPage; i++)
{
int row;
do
{
row = rnd.Next(ap.StartIndex, ap.StartIndex + ap.PageSize);
} while (usedRows.ContainsKey(row));
usedRows.Add(row, row);
sample.Rows.Add(ap.Row(row).ItemArray);
}
ap.MoveNext();
pm.CurrentValue = page;
if (progressHandler.Cancel) break;
//Application.DoEvents();
}
for (int i = 0; i < expressions.Length; i++)
{
try
{
DataRow[] dr = sample.Select(expressions[i]);
counts[i] += dr.Length;
}
catch (Exception ex)
{
Debug.WriteLine(ex);
}
}
pm.Reset();
return counts;
}
for (int page = 0; page < ap.NumPages(); page++)
{
for (int i = 0; i < expressions.Length; i++)
{
DataRow[] dr = ap[page].Select(expressions[i]);
counts[i] += dr.Length;
}
pm.CurrentValue = page;
if (progressHandler.Cancel) break;
//Application.DoEvents();
}
pm.Reset();
return counts;
}
/// <summary>
/// Gets a list of all unique values of the attribute field.
/// </summary>
private List<Break> GetUniqueValues(string fieldName, IAttributeSource source, ICancelProgressHandler progressHandler)
{
ArrayList lst;
bool hugeCountOk = false;
if (_cachedUniqueValues.ContainsKey(fieldName))
{
lst = _cachedUniqueValues[fieldName];
}
else
{
lst = new ArrayList();
AttributePager ap = new AttributePager(source, 5000);
ProgressMeter pm = new ProgressMeter(progressHandler, "Discovering Unique Values", source.NumRows());
for (int row = 0; row < source.NumRows(); row++)
{
object val = ap.Row(row)[fieldName] ?? "[NULL]";
if (val.ToString() == string.Empty) val = "[NULL]";
if (lst.Contains(val)) continue;
lst.Add(val);
if (lst.Count > 1000 && !hugeCountOk)
{
CancelEventArgs args = new CancelEventArgs(true);
if (TooManyCategories != null) TooManyCategories(this, args);
if (args.Cancel) break;
hugeCountOk = true;
}
pm.CurrentValue = row;
if (progressHandler.Cancel) break;
}
lst.Sort();
if (lst.Count < EditorSettings.MaxSampleCount)
{
_cachedUniqueValues[fieldName] = lst;
}
}
List<Break> result = new List<Break>();
if (lst != null)
{
foreach (object item in lst)
{
result.Add(new Break(item.ToString()));
}
}
return result;
}
/// <summary>
/// Gets the values from a file based data source rather than an in memory object.
/// </summary>
/// <param name="source"></param>
/// <param name="progressHandler"></param>
public void GetValues(IAttributeSource source, ICancelProgressHandler progressHandler)
{
int pageSize = 100000;
Values = new List<double>();
string normField = EditorSettings.NormField;
string fieldName = EditorSettings.FieldName;
if (source.NumRows() < EditorSettings.MaxSampleCount)
{
int numPages = (int)Math.Ceiling((double)source.NumRows() / pageSize);
for (int ipage = 0; ipage < numPages; ipage++)
{
int numRows = (ipage == numPages - 1) ? source.NumRows() % pageSize : pageSize;
DataTable table = source.GetAttributes(ipage * pageSize, numRows);
if (!string.IsNullOrEmpty(EditorSettings.ExcludeExpression))
{
DataRow[] rows = table.Select("NOT (" + EditorSettings.ExcludeExpression + ")");
foreach (DataRow row in rows)
{
double val;
if (!double.TryParse(row[fieldName].ToString(), out val)) continue;
if (double.IsNaN(val)) continue;
if (normField != null)
{
double norm;
if (!double.TryParse(row[normField].ToString(), out norm) || double.IsNaN(val))
continue;
Values.Add(val / norm);
continue;
}
Values.Add(val);
}
}
else
{
foreach (DataRow row in table.Rows)
{
double val;
if (!double.TryParse(row[fieldName].ToString(), out val)) continue;
if (double.IsNaN(val)) continue;
if (normField != null)
{
double norm;
if (!double.TryParse(row[normField].ToString(), out norm) || double.IsNaN(val))
continue;
Values.Add(val / norm);
continue;
}
Values.Add(val);
}
}
}
}
else
{
Dictionary<int, double> randomValues = new Dictionary<int, double>();
pageSize = 10000;
int count = EditorSettings.MaxSampleCount;
Random rnd = new Random();
AttributePager ap = new AttributePager(source, pageSize);
int countPerPage = count / ap.NumPages();
ProgressMeter pm = new ProgressMeter(progressHandler, "Sampling " + count + " random values", count);
for (int iPage = 0; iPage < ap.NumPages(); iPage++)
{
for (int i = 0; i < countPerPage; i++)
{
double val;
double norm = 1;
int index;
bool failed = false;
do
{
index = rnd.Next(ap.StartIndex, ap.StartIndex + pageSize);
DataRow dr = ap.Row(index);
if (!double.TryParse(dr[fieldName].ToString(), out val)) failed = true;
if (normField == null) continue;
if (!double.TryParse(dr[normField].ToString(), out norm))
failed = true;
} while (randomValues.ContainsKey(index) || double.IsNaN(val) || failed);
if (normField != null)
{
Values.Add(val / norm);
}
else
{
Values.Add(val);
}
randomValues.Add(index, val);
pm.CurrentValue = i + iPage * countPerPage;
}
//Application.DoEvents();
if (progressHandler != null && progressHandler.Cancel)
{
break;
}
}
}
Values.Sort();
Statistics.Calculate(Values);
}
/// <summary>
/// Gets the values from a file based data source rather than an in memory object.
/// </summary>
/// <param name="source">Source to get the values from.</param>
/// <param name="progressHandler">The progress handler.</param>
public void GetValues(IAttributeSource source, ICancelProgressHandler progressHandler)
{
int pageSize = 100000;
Values = new List <double>();
string normField = EditorSettings.NormField;
string fieldName = EditorSettings.FieldName;
if (source.NumRows() < EditorSettings.MaxSampleCount)
{
int numPages = (int)Math.Ceiling((double)source.NumRows() / pageSize);
for (int ipage = 0; ipage < numPages; ipage++)
{
int numRows = (ipage == numPages - 1) ? source.NumRows() % pageSize : pageSize;
DataTable table = source.GetAttributes(ipage * pageSize, numRows);
if (!string.IsNullOrEmpty(EditorSettings.ExcludeExpression))
{
DataRow[] rows = table.Select("NOT (" + EditorSettings.ExcludeExpression + ")");
foreach (DataRow row in rows)
{
double val;
if (!double.TryParse(row[fieldName].ToString(), out val))
{
continue;
}
if (double.IsNaN(val))
{
continue;
}
if (normField != null)
{
double norm;
if (!double.TryParse(row[normField].ToString(), out norm) || double.IsNaN(val))
{
continue;
}
Values.Add(val / norm);
continue;
}
Values.Add(val);
}
}
else
{
foreach (DataRow row in table.Rows)
{
double val;
if (!double.TryParse(row[fieldName].ToString(), out val))
{
continue;
}
if (double.IsNaN(val))
{
continue;
}
if (normField != null)
{
double norm;
if (!double.TryParse(row[normField].ToString(), out norm) || double.IsNaN(val))
{
continue;
}
Values.Add(val / norm);
continue;
}
Values.Add(val);
}
}
}
}
else
{
Dictionary <int, double> randomValues = new Dictionary <int, double>();
pageSize = 10000;
int count = EditorSettings.MaxSampleCount;
// Specified seed is required for consistently recreating the break values
Random rnd = new Random(9999);
AttributePager ap = new AttributePager(source, pageSize);
int countPerPage = count / ap.NumPages();
ProgressMeter pm = new ProgressMeter(progressHandler, "Sampling " + count + " random values", count);
for (int iPage = 0; iPage < ap.NumPages(); iPage++)
{
for (int i = 0; i < countPerPage; i++)
{
double val;
double norm = 1;
int index;
bool failed = false;
do
{
index = rnd.Next(ap.StartIndex, ap.StartIndex + pageSize);
DataRow dr = ap.Row(index);
if (!double.TryParse(dr[fieldName].ToString(), out val))
{
failed = true;
}
if (normField == null)
{
continue;
}
if (!double.TryParse(dr[normField].ToString(), out norm))
{
failed = true;
}
}while (randomValues.ContainsKey(index) || double.IsNaN(val) || failed);
if (normField != null)
{
Values.Add(val / norm);
}
else
{
Values.Add(val);
}
randomValues.Add(index, val);
pm.CurrentValue = i + (iPage * countPerPage);
}
if (progressHandler != null && progressHandler.Cancel)
{
break;
}
}
}
Values.Sort();
Statistics.Calculate(Values);
}
/// <summary>
/// Gets a list of all unique values of the attribute field.
/// </summary>
/// <param name="fieldName">Name of the field that gets checked.</param>
/// <param name="source">Attribute source that contains the field.</param>
/// <param name="progressHandler">The progress handler.</param>
/// <returns>A list with the unique values.</returns>
private List <Break> GetUniqueValues(string fieldName, IAttributeSource source, ICancelProgressHandler progressHandler)
{
ArrayList lst;
bool hugeCountOk = false;
if (_cachedUniqueValues.ContainsKey(fieldName))
{
lst = _cachedUniqueValues[fieldName];
}
else
{
lst = new ArrayList();
AttributePager ap = new AttributePager(source, 5000);
ProgressMeter pm = new ProgressMeter(progressHandler, "Discovering Unique Values", source.NumRows());
for (int row = 0; row < source.NumRows(); row++)
{
object val = ap.Row(row)[fieldName] ?? "[NULL]";
if (val.ToString() == string.Empty)
{
val = "[NULL]";
}
if (lst.Contains(val))
{
continue;
}
lst.Add(val);
if (lst.Count > 1000 && !hugeCountOk)
{
CancelEventArgs args = new CancelEventArgs(true);
TooManyCategories?.Invoke(this, args);
if (args.Cancel)
{
break;
}
hugeCountOk = true;
}
pm.CurrentValue = row;
if (progressHandler.Cancel)
{
break;
}
}
lst.Sort();
if (lst.Count < EditorSettings.MaxSampleCount)
{
_cachedUniqueValues[fieldName] = lst;
}
}
List <Break> result = new List <Break>();
if (lst != null)
{
foreach (object item in lst)
{
result.Add(new Break(item.ToString()));
}
}
return(result);
}
/// <summary>
/// Gets a list of all unique values of the attribute field.
/// </summary>
private List<Break> GetUniqueValues(string fieldName, IAttributeSource source, ICancelProgressHandler progressHandler)
{
ArrayList lst;
bool _hugeCountOk = false;
if(_cachedUniqueValues.ContainsKey(fieldName))
{
lst = _cachedUniqueValues[fieldName];
}
else
{
lst = new ArrayList();
AttributePager ap = new AttributePager(source, 5000);
ProgressMeter pm = new ProgressMeter(progressHandler, "Discovering Unique Values", source.NumRows());
for (int row = 0; row < source.NumRows(); row++)
{
object val = ap.Row(row)[fieldName] ?? "[NULL]";
if (val.ToString() == "") val = "[NULL]";
if (lst.Contains(val)) continue;
lst.Add(val);
if (lst.Count > 1000 && !_hugeCountOk)
{
if (MessageBox.Show(
"There are more than 1000 unique values, which can result in very slow performance. Do you wish to try to create unique categories from all the values anyway?",
"Extreme Number of Categories", MessageBoxButtons.YesNo) == DialogResult.No)
{
break;
}
_hugeCountOk = true;
}
pm.CurrentValue = row;
if (progressHandler.Cancel) break;
}
lst.Sort();
if (lst.Count < EditorSettings.MaxSampleCount)
{
_cachedUniqueValues[fieldName] = lst;
}
}
List<Break> result = new List<Break>();
if (lst != null)
{
foreach (object item in lst)
{
result.Add(new Break(item.ToString()));
}
}
return result;
}
