private static void fillPcaReport(string modelPath, Forms.RunningProcess.frmRunningProcessDialog rp, double proportion = 0.1, double alpha = 0.05)
{
dataPrepPrincipleComponents pca = new dataPrepPrincipleComponents();
pca.buildModel(modelPath);
double[] meanVector = pca.MeanVector;
double[] std = pca.StdVector;
int numSamp = sampleSizeMaxMean(meanVector, std, proportion, alpha);
rp.addMessage("\nTotal Number of Samples = " + numSamp.ToString());
}
private void buildModel()
{
if (!checkTables())
{
//Console.WriteLine("CheckTables = false");
return;
}
if (!getSampleRatios())
{
//Console.WriteLine("Sample Ratios = false");
return;
}
pca = new dataPrepPrincipleComponents(Sample1, Variables);
cluster = new dataPrepClusterKmean(Sample1, Variables, numberOfBins);
foreach (string s in cntDic.Keys)
{
buildSamples(s);
double[] pValueArr = new double[Variables.Length];
double[] sValueArr = new double[Variables.Length];
//double[] s1Arr = sample1[0];
//double[] s2Arr = sample2[0];
//TwoSampleKolmogorovSmirnovTest test = new TwoSampleKolmogorovSmirnovTest(s1Arr, s2Arr, TwoSampleKolmogorovSmirnovTestHypothesis.SamplesDistributionsAreUnequal);
//getCdfProp(s, 0, test);
//pValue = test.PValue;
//sValue = test.Statistic;
for (int i = 0; i < Variables.Length; i++)
{
double[] s1Arr = sample1[i];
double[] s2Arr = sample2[i];
TwoSampleKolmogorovSmirnovTest test = new TwoSampleKolmogorovSmirnovTest(s1Arr, s2Arr, TwoSampleKolmogorovSmirnovTestHypothesis.SamplesDistributionsAreUnequal);
//Console.WriteLine(test.Significant.ToString());
getCdfProp(s, i, test);
double pValueS = test.PValue;
double sValueS = test.Statistic;
pValueArr[i] = pValueS;
sValueArr[i] = sValueS;
}
pDic.Add(s, pValueArr);
sDic.Add(s, sValueArr);
}
}
private void buildModel(string mdlPath)
{
using (System.IO.StreamReader sr = new System.IO.StreamReader(mdlPath))
{
dataPrepBase.modelTypes mType = (dataPrepBase.modelTypes)Enum.Parse(typeof(dataPrepBase.modelTypes), sr.ReadLine());
if (mType != dataPrepBase.modelTypes.KS)
{
System.Windows.Forms.MessageBox.Show("Not a KS Model!!", "Error", System.Windows.Forms.MessageBoxButtons.OK, System.Windows.Forms.MessageBoxIcon.Error);
return;
}
Variables = sr.ReadLine().Split(new char[] { ',' });
StrataField = sr.ReadLine();
Oridinate = System.Convert.ToBoolean(sr.ReadLine());
string[] lbl = sr.ReadLine().Split(new char[] { ',' });
string[] pArr = sr.ReadLine().Split(new char[] { ',' });
string[] sArr = sr.ReadLine().Split(new char[] { ',' });
string[] cArr1 = sr.ReadLine().Split(new char[] { ',' });
string[] cArr2 = sr.ReadLine().Split(new char[] { ',' });
for (int i = 0; i < lbl.Length; i++)
{
string l = lbl[i];
double[] p = (from string str in pArr[i].Split(new char[] { ';' }) select System.Convert.ToDouble(str)).ToArray();
double[] s = (from string str in sArr[i].Split(new char[] { ';' }) select System.Convert.ToDouble(str)).ToArray();
int c1 = System.Convert.ToInt32(cArr1[i]);
int c2 = System.Convert.ToInt32(cArr2[i]);
pDic.Add(l, p);
sDic.Add(l, s);
cntDic.Add(l, new int[] { c1, c2 });
}
double[][] minmax1, minmax2, bp1, bp2;
for (int i = 0; i < lbl.Length; i++)
{
string l = lbl[i];
double[] min1 = (from string s in sr.ReadLine().Split(new char[] { ',' }) select System.Convert.ToDouble(s)).ToArray();
double[] max1 = (from string s in sr.ReadLine().Split(new char[] { ',' }) select System.Convert.ToDouble(s)).ToArray();
double[] min2 = (from string s in sr.ReadLine().Split(new char[] { ',' }) select System.Convert.ToDouble(s)).ToArray();
double[] max2 = (from string s in sr.ReadLine().Split(new char[] { ',' }) select System.Convert.ToDouble(s)).ToArray();
minmax1 = new double[2][];
minmax1[0] = min1;
minmax1[1] = max1;
minMaxDic1.Add(l, minmax1);
minmax2 = new double[2][];
minmax2[0] = min2;
minmax2[1] = max2;
minMaxDic2.Add(l, minmax2);
bp1 = new double[Variables.Length][];
bp2 = new double[Variables.Length][];
for (int j = 0; j < Variables.Length; j++)
{
bp1[j] = (from string s in sr.ReadLine().Split(new char[] { ',' }) select System.Convert.ToDouble(s)).ToArray();
bp2[j] = (from string s in sr.ReadLine().Split(new char[] { ',' }) select System.Convert.ToDouble(s)).ToArray();
}
binPropDic1.Add(l, bp1);
binPropDic2.Add(l, bp2);
int[] clusCnt = (from string s in sr.ReadLine().Split(new char[] { ',' }) select System.Convert.ToInt32(s)).ToArray();
clusCountDic.Add(l, clusCnt);
int[] clusSampCnt = (from string s in sr.ReadLine().Split(new char[] { ',' }) select System.Convert.ToInt32(s)).ToArray();
clusSampleCountDic.Add(l, clusSampCnt);
}
pca = new dataPrepPrincipleComponents();
string pcPath = System.IO.Path.GetDirectoryName(mdlPath) + "\\" + System.IO.Path.GetFileNameWithoutExtension(mdlPath) + "_pca.mdl";
pca.buildModel(pcPath);
cluster = new dataPrepClusterKmean();
string clusterPath = System.IO.Path.GetDirectoryName(mdlPath) + "\\" + System.IO.Path.GetFileNameWithoutExtension(mdlPath) + "_cluster.mdl";
cluster.buildModel(clusterPath);
numberOfBins = cluster.Classes;
sr.Close();
}
}
private void createPcaModel(string[] paramArr)
{
Statistics.dataPrepPrincipleComponents dpPca = null;
if (paramArr.Length < 4)
{
IRaster rs = rsUtil.createRaster(getRaster(paramArr[1]));
dpPca = new Statistics.dataPrepPrincipleComponents(rs);
}
else
{
ITable table = getTable(paramArr[1]);
string[] variables = paramArr[2].Split(new char[] { ',' });
dpPca = new Statistics.dataPrepPrincipleComponents(table, variables);
}
dpPca.writeModel(paramArr[paramArr.Length - 1]);
}
private static void fillPcaReport(string modelPath, Forms.RunningProcess.frmRunningProcessDialog rp, double proportion = 0.1, double alpha = 0.05)
{
dataPrepPrincipleComponents pca = new dataPrepPrincipleComponents();
pca.buildModel(modelPath);
double[] meanVector = pca.MeanVector;
double[] std = pca.StdVector;
int numSamp = sampleSizeMaxMean(meanVector, std, proportion, alpha);
rp.addMessage("\nTotal Number of Samples = " + numSamp.ToString());
}
public void selectPcaFeaturesToSample(ITable inputTable, string pcaModelPath, double proportionOfMean, double alpha)
{
IObjectClassInfo2 objInfo2 = (IObjectClassInfo2)inputTable;
if (!objInfo2.CanBypassEditSession())
{
System.Windows.Forms.MessageBox.Show("Input Table participates in a composite relationship. Please export this table as a new table and try again!");
return;
}
Statistics.dataPrepPrincipleComponents pca = new Statistics.dataPrepPrincipleComponents();
pca.buildModel(pcaModelPath);
System.Random rd = new Random();
double tSamples = System.Convert.ToDouble(esriUtil.Statistics.dataPrepSampleSize.sampleSizeMaxMean(pca.MeanVector, pca.StdVector, proportionOfMean, alpha));
int tRecords = inputTable.RowCount(null);
double gR = tSamples / tRecords;
string sampleFldName = geoUtil.createField(inputTable, "sample", esriFieldType.esriFieldTypeSmallInteger, false);
IQueryFilter qf0 = new QueryFilterClass();
IQueryFilter qf = new QueryFilterClass();
qf.SubFields = sampleFldName;
IWorkspace wks = ((IDataset)inputTable).Workspace;
IWorkspaceEdit wksE = (IWorkspaceEdit)wks;
if (wksE.IsBeingEdited())
{
wksE.StopEditing(true);
}
try
{
ICursor cur = inputTable.Update(qf, false);
int sIndex = cur.FindField(sampleFldName);
IRow rw = cur.NextRow();
while (rw != null)
{
double rNum = rd.NextDouble();
int ss = 0;
double r = gR;
if (rNum < r)
{
ss = 1;
}
rw.set_Value(sIndex, ss);
cur.UpdateRow(rw);
rw = cur.NextRow();
}
System.Runtime.InteropServices.Marshal.ReleaseComObject(cur);
}
catch (Exception e)
{
System.Windows.Forms.MessageBox.Show(e.ToString());
}
}
private void buildModel(string mdlPath)
{
using (System.IO.StreamReader sr = new System.IO.StreamReader(mdlPath))
{
dataPrepBase.modelTypes mType = (dataPrepBase.modelTypes)Enum.Parse(typeof(dataPrepBase.modelTypes), sr.ReadLine());
if (mType != dataPrepBase.modelTypes.KS)
{
System.Windows.Forms.MessageBox.Show("Not a KS Model!!", "Error", System.Windows.Forms.MessageBoxButtons.OK, System.Windows.Forms.MessageBoxIcon.Error);
return;
}
Variables = sr.ReadLine().Split(new char[] { ',' });
StrataField = sr.ReadLine();
Oridinate = System.Convert.ToBoolean(sr.ReadLine());
string[] lbl = sr.ReadLine().Split(new char[] { ',' });
string[] pArr = sr.ReadLine().Split(new char[] { ',' });
string[] sArr = sr.ReadLine().Split(new char[] {','});
string[] cArr1 = sr.ReadLine().Split(new char[] { ',' });
string[] cArr2 = sr.ReadLine().Split(new char[] { ',' });
for (int i = 0; i < lbl.Length; i++)
{
string l = lbl[i];
double[] p = (from string str in pArr[i].Split(new char[]{';'}) select System.Convert.ToDouble(str)).ToArray();
double[] s = (from string str in sArr[i].Split(new char[] { ';' }) select System.Convert.ToDouble(str)).ToArray();
int c1 = System.Convert.ToInt32(cArr1[i]);
int c2 = System.Convert.ToInt32(cArr2[i]);
pDic.Add(l, p);
sDic.Add(l, s);
cntDic.Add(l, new int[]{c1,c2});
}
double[][] minmax1, minmax2, bp1, bp2;
for (int i = 0; i < lbl.Length; i++)
{
string l = lbl[i];
double[] min1 = (from string s in sr.ReadLine().Split(new char[] { ',' }) select System.Convert.ToDouble(s)).ToArray();
double[] max1 = (from string s in sr.ReadLine().Split(new char[] { ',' }) select System.Convert.ToDouble(s)).ToArray();
double[] min2 = (from string s in sr.ReadLine().Split(new char[] { ',' }) select System.Convert.ToDouble(s)).ToArray();
double[] max2 = (from string s in sr.ReadLine().Split(new char[] { ',' }) select System.Convert.ToDouble(s)).ToArray();
minmax1 = new double[2][];
minmax1[0] = min1;
minmax1[1] = max1;
minMaxDic1.Add(l, minmax1);
minmax2 = new double[2][];
minmax2[0] = min2;
minmax2[1] = max2;
minMaxDic2.Add(l,minmax2);
bp1 = new double[Variables.Length][];
bp2 = new double[Variables.Length][];
for (int j = 0; j < Variables.Length; j++)
{
bp1[j] = (from string s in sr.ReadLine().Split(new char[] { ',' }) select System.Convert.ToDouble(s)).ToArray();
bp2[j] = (from string s in sr.ReadLine().Split(new char[] { ',' }) select System.Convert.ToDouble(s)).ToArray();
}
binPropDic1.Add(l, bp1);
binPropDic2.Add(l, bp2);
int[] clusCnt = (from string s in sr.ReadLine().Split(new char[] { ',' }) select System.Convert.ToInt32(s)).ToArray();
clusCountDic.Add(l, clusCnt);
int[] clusSampCnt = (from string s in sr.ReadLine().Split(new char[] { ',' }) select System.Convert.ToInt32(s)).ToArray();
clusSampleCountDic.Add(l, clusSampCnt);
}
pca = new dataPrepPrincipleComponents();
string pcPath = System.IO.Path.GetDirectoryName(mdlPath) + "\\" + System.IO.Path.GetFileNameWithoutExtension(mdlPath) + "_pca.mdl";
pca.buildModel(pcPath);
cluster = new dataPrepClusterKmean();
string clusterPath = System.IO.Path.GetDirectoryName(mdlPath) + "\\" + System.IO.Path.GetFileNameWithoutExtension(mdlPath) + "_cluster.mdl";
cluster.buildModel(clusterPath);
numberOfBins = cluster.Classes;
sr.Close();
}
}
private void buildModel()
{
if (!checkTables())
{
//Console.WriteLine("CheckTables = false");
return;
}
if (!getSampleRatios())
{
//Console.WriteLine("Sample Ratios = false");
return;
}
pca = new dataPrepPrincipleComponents(Sample1, Variables);
cluster = new dataPrepClusterKmean(Sample1, Variables, numberOfBins);
foreach(string s in cntDic.Keys)
{
buildSamples(s);
double[] pValueArr = new double[Variables.Length];
double[] sValueArr = new double[Variables.Length];
//double[] s1Arr = sample1[0];
//double[] s2Arr = sample2[0];
//TwoSampleKolmogorovSmirnovTest test = new TwoSampleKolmogorovSmirnovTest(s1Arr, s2Arr, TwoSampleKolmogorovSmirnovTestHypothesis.SamplesDistributionsAreUnequal);
//getCdfProp(s, 0, test);
//pValue = test.PValue;
//sValue = test.Statistic;
for (int i = 0; i < Variables.Length; i++)
{
double[] s1Arr = sample1[i];
double[] s2Arr = sample2[i];
TwoSampleKolmogorovSmirnovTest test = new TwoSampleKolmogorovSmirnovTest(s1Arr, s2Arr, TwoSampleKolmogorovSmirnovTestHypothesis.SamplesDistributionsAreUnequal);
//Console.WriteLine(test.Significant.ToString());
getCdfProp(s, i, test);
double pValueS = test.PValue;
double sValueS = test.Statistic;
pValueArr[i] = pValueS;
sValueArr[i] = sValueS;
}
pDic.Add(s, pValueArr);
sDic.Add(s, sValueArr);
}
}