private void initPreprocess()
{
if (!this._isInited)
{
foreach (var p in Preprocesser.GerProcesser())
{
if (p.Filter == null)
{
continue;
}
if (p.Filter.FType == FilterType.SpecFilter)
{
this.treeMethod.Nodes[0].Nodes.Add(new MyTreenode()
{
Name = p.Filter.Name,
Text = p.Filter.Name,
Preocesser = p,
ToolTipText = "双击可添加预处理方法"
});
}
else
{
this.treeMethod.Nodes[1].Nodes.Add(new MyTreenode()
{
Name = p.Filter.Name,
Text = p.Filter.Name,
Preocesser = p,
ToolTipText = "双击可添加预处理方法"
});
}
}
}
this.treeMethod.ExpandAll();
this._isInited = true;
}
public void Train(SpecBase lib, bool needFilter = true)
{
//过滤掉性质有NaN的数据
lib.FilterNaN();
int[] idxs = lib.Specs.Select((d, idx) => new { s = d, idx = idx }).Where(d => d.s.Usage != UsageTypeEnum.Ignore).Select(d => d.idx).ToArray();
this._lib = Serialize.DeepClone <SpecBase>(lib.SubLib(idxs));
if (needFilter && this._filters != null)
{
this._lib.SetX(Preprocesser.Process(this._filters, this._lib), true);
}
//PCA分解
var handler = Tools.ModelHandler;
if (this._maxRank < 1)
{
this._maxRank = 10;
}
this._maxRank = Math.Min(this._maxRank, this._lib.X.Dimensions[1]);
var r = handler.IdentifyTrain(3, this._lib.X, this._maxRank);
this._p = (MWNumericArray)r[0];
this._w = (MWNumericArray)r[1];
this._t = (MWNumericArray)r[2];
this._trained = true;
}
public FittingResult Predict(Spectrum specInput, bool needFilter = true, int numOfId = 5, int topK = 1)
{
var spec = specInput.Clone();
var y = new MWNumericArray(specInput.Data.Lenght, 1, specInput.Data.Y);
//进行预处理
if (needFilter && this._filters != null)
{
y = Preprocesser.ProcessForPredict(this._filters, y);
spec.Data.Y = (double[])y.ToVector(MWArrayComponent.Real);
}
var handler = Tools.ModelHandler;
var rlst = handler.FitPredictor(5, this._lib.X, y, this._wind, (MWNumericArray)this._region.VarIndex, topK);
var tq = (MWNumericArray)rlst[0];
var sq = (MWNumericArray)rlst[1];
var ratio = (MWNumericArray)rlst[2];
var idx = (MWNumericArray)rlst[3];
var fit = (MWNumericArray)rlst[4];
var fitspec = spec.Clone();
fitspec.Components = this._lib.Components.Clone();
fitspec.Data.Y = (double[])fit.ToVector(MWArrayComponent.Real);
var r = new FittingResult()
{
SpecOriginal = spec,
SQ = sq.ToScalarDouble(),
TQ = tq.ToScalarDouble(),
FitSpec = fitspec,
MinSQ = this._minSQ,
MinTQ = this._TQ,
Wind = this._wind,
VarIndex = this._region.VarIndex
};
var flst = new List <FittingSpec>();
for (int i = 1; i <= ratio.NumberOfElements; i++)
{
if ((double)ratio[i] <= double.Epsilon)
{
break;
}
flst.Add(new FittingSpec()
{
Spec = this._lib[(int)idx[i] - 1],
Rate = (double)ratio[i]
});
}
r.Specs = flst.ToArray();
this.getResult(ref r);
r.Result = r.TQ >= this._TQ && r.SQ >= this._minSQ;
return(r);
}
public void Train(SpecBase lib, bool needFilter = true)
{
//过滤掉性质有NaN的数据
lib.FilterNaN();
int[] idxs = lib.Specs.Select((d, idx) => new { s = d, idx = idx }).Where(d => d.s.Usage != UsageTypeEnum.Ignore).Select(d => d.idx).ToArray();
this._lib = Serialize.DeepClone <SpecBase>(lib.SubLib(idxs));
if (needFilter && this._filters != null)
{
this._lib.SetX(Preprocesser.Process(this._filters, this._lib), true);
}
this._trained = true;
}
private void addRow(Preprocesser p, bool needfire = true)
{
p.StatuChange += new EventHandler(p_StatuChange);
var row = new MyDataRow()
{
Preocesser = p
};
row.CreateCells(this.gridMethodLst, row.Preocesser.Filter.Name, row.Preocesser.Filter.ArgusToString(), row.Preocesser.Statu.GetDescription());
row.Cells[1].ToolTipText = "点击可重新设置参数";
//row.Cells[1].
this.gridMethodLst.Rows.Add(row);
if (!p.Filter.EditEnable)
{
row.DefaultCellStyle.BackColor = Color.FromArgb(250, 250, 250);
}
if (needfire)
{
this.fireProcessorChanged();
}
}
public DialogResult ShowDialog(Preprocesser p, NIR.Component c)
{
if (p == null || p.SpecsInput == null || p.SpecsInput.Count == 0)
{
return(System.Windows.Forms.DialogResult.Abort);
}
var filter = p.Filter as VarRegionManu;
if (filter == null)
{
return(System.Windows.Forms.DialogResult.Abort);
}
this._processer = p;
this.varRegionControl1.Drawchart(p.SpecsInput, c);
this.varRegionControl1.XaxisRegion = filter.XaxisRegion.Select(d => new SelectRange()
{
Begin = d.X,
End = d.Y
}).ToList();
var result = this.ShowDialog();
return(result);
}
public IdentifyResult[] CrossValidation(SpecBase lib, bool needFilter = true, int numOfId = 5)
{
//过滤掉性质有NaN的数据
lib.FilterNaN();
var clib = lib.SubLib(UsageTypeEnum.Calibrate);
if (clib.Count == 0)
{
return(new List <IdentifyResult>().ToArray());
}
var cx = clib.X;
if (needFilter && this._filters != null)
{
cx = Preprocesser.Process(this._filters, cx);
}
if (!this._trained)
{
this.Train(clib);
}
var handler = Tools.ModelHandler;
//Tools.Save(cx, "x.mat", "x");
var rlst = handler.IdentifyCrossValidation(3, cx, this._wind, this._maxNum, this._maxRank);
var lst = new IdentifyResult[cx.Dimensions[1]];
clib.X = cx;
for (int i = 0; i < lst.Length; i++)
{
var SQ = Tools.SelectColumn((MWNumericArray)rlst[0], i + 1);
var tq = Tools.SelectColumn((MWNumericArray)rlst[1], i + 1);
var idx = Tools.SelectColumn((MWNumericArray)rlst[2], i + 1);
var item = new IdentifyResult()
{
MinSQ = this._minSQ,
MinTQ = this._TQ,
};
item.Spec = clib[i].Clone();
item.Components = item.Spec.Components.Clone();
var itemlst = new List <IdentifyItem>();
for (int k = 0; k < Math.Min(this._maxNum, idx.NumberOfElements); k++)
{
var adidx = (int)idx[k + 1] - 1;
if (adidx >= clib.Count || adidx < 0)
{
break;
}
itemlst.Add(new IdentifyItem()
{
Parent = item,
SQ = (double)SQ[k + 1],
TQ = (double)tq[k + 1],
Spec = clib[adidx],
SpecOriginal = item.Spec,
Wind = this._wind
});
}
item.Items = itemlst.ToArray();
item = GetPredictValue(item, item.Items.Length, numOfId);
item.IsId = item.Items.Where(d => d.Result).Count() > 0;
lst[i] = item;
}
return(lst);
}
public IdentifyResult Predict(Spectrum speciii, bool needFilter = true, int numOfId = 5, int topK = 1)
{
if (this._lib == null || speciii == null)
{
throw new ArgumentNullException("");
}
var spec = speciii.Clone();
var y = new MWNumericArray(speciii.Data.Lenght, 1, speciii.Data.Y);
//进行预处理
if (needFilter && this._filters != null)
{
y = Preprocesser.ProcessForPredict(this._filters, y);
spec.Data.Y = (double[])y.ToVector(MWArrayComponent.Real);
}
var handler = Tools.ModelHandler;
//Tools.ToolHandler.Save(this._lib.X, "x", "x");
//Tools.ToolHandler.Save(y, "y", "y");
//Tools.ToolHandler.Save(this._baseLib.X, "xa", "xa");
//this.LibBase.FullPath = "aaaa.lib";
//this.LibBase.Save();
this._maxNum = Math.Min(this._maxNum, this._lib.X.Dimensions[1]);
var rlst = handler.IdentifyPredictor(3, this._lib.X, y, this._wind, this._maxNum, this._p, this._w, this._t, topK);
var minSQ = (MWNumericArray)rlst[0];
var minTQ = (MWNumericArray)rlst[1];
var ad = (MWNumericArray)rlst[2];
var r = new IdentifyResult()
{
Spec = spec,
MinSQ = this.MinSQ,
MinTQ = this.TQ
};
//设置结果
var itemlst = new List <IdentifyItem>();
// r.Items = new IdentifyItem[ad.NumberOfElements];
for (int i = 0; i < this._maxNum; i++)
{
var adidx = (int)ad[i + 1] - 1;
if (adidx >= this._lib.Count || adidx < 0)
{
break;
}
itemlst.Add(new IdentifyItem()
{
Parent = r,
TQ = (double)minTQ[i + 1],
SQ = (double)minSQ[i + 1],
Spec = this._lib[adidx],
Wind = this._wind,
SpecOriginal = r.Spec
});
}
r.Items = itemlst.ToArray();
r = GetPredictValue(r, r.Items.Length, numOfId);
r.IsId = r.Items.Where(d => d.Result).Count() > 0;
return(r);
}
public FittingResult[] CrossValidation(SpecBase lib, bool needFilter = true, int numOfId = 5)
{
//过滤掉性质有NaN的数据
lib.FilterNaN();
var clib = lib.SubLib(UsageTypeEnum.Calibrate);
if (clib.Count == 0)
{
return(new List <FittingResult>().ToArray());
}
var cx = clib.X;
if (needFilter && this._filters != null)
{
cx = Preprocesser.ProcessForPredict(this._filters, cx);
}
var handler = Tools.ModelHandler;
//RIPP.NIR.Data.Tools.Save(cx, "datacx.mat","cxripp");
var rlst = handler.FitCroosValidation(5, cx, this._wind, (MWNumericArray)this._region.VarIndex);
var allTQ = (MWNumericArray)rlst[0];
var allSQ = (MWNumericArray)rlst[1];
var allRatio = (MWNumericArray)rlst[2];
var allIdx = (MWNumericArray)rlst[3];
var allFit = (MWNumericArray)rlst[4];
clib.X = cx;
var lst = new FittingResult[cx.Dimensions[1]];
for (int i = 0; i < lst.Length; i++)
{
var ratio = Tools.SelectColumn(allRatio, i + 1);
var idx = Tools.SelectColumn(allIdx, i + 1);
var fit = Tools.SelectColumn(allFit, i + 1);
var fitspec = clib[i].Clone();
fitspec.Components = clib.Components.Clone();
fitspec.Data.Y = (double[])fit.ToVector(MWArrayComponent.Real);
var item = new FittingResult()
{
SQ = (double)allSQ[i + 1],
TQ = (double)allTQ[i + 1],
MinSQ = this._minSQ,
MinTQ = this._TQ,
FitSpec = fitspec,
Wind = this._wind,
VarIndex = this._region.VarIndex
};
item.SpecOriginal = clib[i].Clone();
var flst = new List <FittingSpec>();
for (int k = 1; k <= ratio.NumberOfElements; k++)
{
if ((double)ratio[k] <= double.Epsilon)
{
break;
}
flst.Add(new FittingSpec()
{
Spec = clib[(int)idx[k] - 1],
Rate = (double)ratio[k]
});
}
item.Specs = flst.ToArray();
this.getResult(ref item);
item.Result = item.TQ >= this._TQ && item.SQ >= this._minSQ;
lst[i] = item;
}
return(lst);
}
/// <summary>
/// 训练
/// </summary>
/// <param name="lib">光谱库</param>
/// <param name="needFilter">是否需要前处理</param>
public void Train(SpecBase libb, bool needFilter = true)
{
var lib = libb.Clone();
//过滤掉性质有NaN的数据
lib.FilterNaN(this._comp);
foreach (var s in lib.Specs)
{
if (this._OutlierNames.Contains(s.Name))
{
s.Usage = UsageTypeEnum.Ignore;
}
}
// lib.Specs = lib.Specs.Where(d => !this._OutlierNames.Contains(d.Name)).ToList();
var sublib = lib.SubLib(UsageTypeEnum.Calibrate);
var cx = sublib.X;
var cy = sublib.GetY(this._comp, true);
if (needFilter && this._filters != null)
{
cx = Preprocesser.Process(this._filters, cx);
}
var handler = Tools.ModelHandler;
var pls = handler.PLS1Train(11, cx, cy, this._maxFactor, (int)this._method);
this._Scores = (MWNumericArray)pls[0];
this._Loads = (MWNumericArray)pls[1];
this._Weights = (MWNumericArray)pls[2];
this._Bias = (MWNumericArray)pls[3];
this._Score_Length = (MWNumericArray)pls[4];
this._centerSpecData = (MWNumericArray)pls[5];
this._centerCompValue = ((MWNumericArray)pls[6]).ToScalarDouble();
this._Mdt = (double[])((MWNumericArray)pls[7]).ToVector(MWArrayComponent.Real);
this._NNdt = (double[])((MWNumericArray)pls[8]).ToVector(MWArrayComponent.Real);
this._sec = (double[])((MWNumericArray)pls[9]).ToVector(MWArrayComponent.Real);
this._cr = (double[])((MWNumericArray)pls[10]).ToVector(MWArrayComponent.Real);
if (this._annType != PLSAnnEnum.None)
{
double mse = 0;
//中心化Scores,cy
var toolHandler = new RIPPMatlab.Tools();
this._ScoresMean = (MWNumericArray)toolHandler.Mean(1, this._Scores)[0];
var mScores = (MWNumericArray)toolHandler.Centring(1, this._Scores, this._ScoresMean)[0];
var my = (MWNumericArray)toolHandler.Centring(1, cy, this._centerCompValue)[0];
//
var guidlib = lib.SubLib(UsageTypeEnum.Guide);
bool needgruid = this._annArgus.IsGuard && guidlib.Specs.Count > 0;
MWArray[] ann;
if (needgruid) //有监控
{
var gx = guidlib.X;
var gy = guidlib.GetY(this._comp, true);
if (needFilter && this._filters != null)
{
gx = Preprocesser.Process(this._filters, gx);
}
var gpls = handler.PLS1Predictor(5, gx, this.Scores, this.Loads, this.Weights, this.Bias, this.Score_Length, this.CenterSpecData, this.CenterCompValue, (int)this._method);
var gScoresm = (MWNumericArray)toolHandler.Centring(1, gpls[4], this._ScoresMean)[0];
var gym = (MWNumericArray)toolHandler.Centring(1, gy, this._centerCompValue)[0];
ann = handler.bann2(6,
mScores,
my,
gScoresm,
gym,
this._annArgus.FuncTrain.GetDescription(),
this._annArgus.NumHidden,
this._annArgus.F1.GetDescription(),
this._annArgus.F2.GetDescription(),
this._annArgus.Epochs,
this._annArgus.Target,
this._annArgus.TimesAvg,
this._annArgus.TimesRepeat);
this._sem = ((MWNumericArray)ann[4]).ToScalarDouble();
this._mr = ((MWNumericArray)ann[5]).ToScalarDouble();
}
else
{
ann = handler.bann1(4,
mScores,
my,
this._annArgus.FuncTrain.GetDescription(),
this._annArgus.NumHidden,
this._annArgus.F1.GetDescription(),
this._annArgus.F2.GetDescription(),
this._annArgus.Epochs,
this._annArgus.Target,
this._annArgus.TimesAvg,
this._annArgus.TimesRepeat);
}
this._annModel.w1 = (MWNumericArray)ann[0];
this._annModel.b1 = (MWNumericArray)ann[1];
this._annModel.w2 = (MWNumericArray)ann[2];
this._annModel.b2 = (MWNumericArray)ann[3];
////重新计算CR
//double ye = 0, yea = 0;
//var yp = (MWNumericArray)ann[4];
//for (int k = 0; k < cy.NumberOfElements; k++)
//{
// ye += (cy[k + 1].ToScalarDouble() - this._centerCompValue - yp[k + 1].ToScalarDouble()) * (cy[k + 1].ToScalarDouble() - this._centerCompValue - yp[k + 1].ToScalarDouble());
// yea += (cy[k + 1].ToScalarDouble() - this._centerCompValue) * (cy[k + 1].ToScalarDouble() - this._centerCompValue);
//}
//this._cr[this._maxFactor - 1] = ye / (ye + yea);
}
this._trained = true;
lib.Dispose();
sublib.Dispose();
cx.Dispose();
cy.Dispose();
GC.Collect();
}
public PLS1Result[] CrossValidation(SpecBase libb, bool needFilter = true, int numOfId = 5)
{
var lib = libb.Clone();
//过滤掉性质有NaN的数据
lib.FilterNaN(this._comp);
foreach (var s in lib.Specs)
{
if (this._OutlierNames.Contains(s.Name))
{
s.Usage = UsageTypeEnum.Ignore;
}
}
//lib.Specs = lib.Specs.Where(d => !this._OutlierNames.Contains(d.Name)).ToList();
if (!this._trained)
{
this.Train(lib, needFilter);
}
var clib = lib.SubLib(UsageTypeEnum.Calibrate);
if (clib.Count == 0)
{
return(null);
}
if (needFilter && this._filters != null)
{
clib.X = Preprocesser.Process(this._filters, clib.X);
}
var handler = Tools.ModelHandler;
MWNumericArray Ylast, SR, MD, nd;
if (this._annType != PLSAnnEnum.None)
{
var model = Serialize.DeepClone <PLSSubModel>(this);
model.Train(lib, true);
var pls = handler.PLS1Predictor(5, clib.X, model.Scores, model.Loads, model.Weights, model.Bias, model.Score_Length, model.CenterSpecData, model.CenterCompValue, (int)this._method);
var toolHandler = Tools.ToolHandler;
Ylast = (MWNumericArray)pls[0];
SR = (MWNumericArray)pls[1];
MD = (MWNumericArray)pls[2];
nd = (MWNumericArray)pls[3];
var mscores = (MWNumericArray)toolHandler.Centring(1, pls[4], model.ScoresMean)[0];
double[] annrsult;
var annp = handler.annp(1,
mscores,
model.ANNModel.w1,
model.ANNModel.b1,
model.ANNModel.w2,
model.ANNModel.b2,
model.ANNAgrus.F1.GetDescription(),
model.ANNAgrus.F2.GetDescription())[0];
annrsult = (double[])((MWNumericArray)annp).ToVector(MWArrayComponent.Real);
for (int row = 0; row < annrsult.Length; row++)
{
Ylast[row + 1, this._factor] = annrsult[row] + model.CenterCompValue;
}
}
else
{
var pls = handler.PLS1CrossValidation(4, clib.X, clib.GetY(this._comp, true), this._maxFactor, (int)this._method);
Ylast = (MWNumericArray)pls[0];
SR = (MWNumericArray)pls[1];
MD = (MWNumericArray)pls[2];
nd = (MWNumericArray)pls[3];
}
var items = new List <PLS1Result>();
for (int i = 0; i < clib.Count; i++)
{
var c = this._comp.Clone();
var s = clib[i];
if (s.Components != null && s.Components.Contains(c.Name))
{
c.ActualValue = s.Components[c.Name].ActualValue;
}
var r = new PLS1Result
{
MDMin = this._MDMin,
NDMin = this._NNDMin,
SRMin = this._SRMin,
Comp = c,
Spec = s,
YLast = (double[])Tools.SelectRow(Ylast, i + 1).ToVector(MWArrayComponent.Real),
SR = (double[])Tools.SelectRow(SR, i + 1).ToVector(MWArrayComponent.Real),
MahDist = (double[])Tools.SelectRow(MD, i + 1).ToVector(MWArrayComponent.Real),
ND = (double[])Tools.SelectRow(nd, i + 1).ToVector(MWArrayComponent.Real),
Factor = this.Factor
};
if (!double.IsNaN(r.YLast[0]))
{
items.Add(r);
}
}
lib.Dispose();
clib.Dispose();
GC.Collect();
return(items.ToArray());
}
/// <summary>
/// 预测
/// </summary>
/// <param name="spec"></param>
/// <param name="needFilter">是否需要前处理</param>
/// <returns></returns>
public PLS1Result Predict(Spectrum spec, bool needFilter = true, int numOfId = 5, int topK = 1)
{
if (spec == null)
{
throw new ArgumentNullException("");
}
var s = spec.Clone();
if (s.Data == null || s.Data.Y == null || s.Data.Y.Length == 0)
{
return(null);
}
var x = new MWNumericArray(s.Data.Lenght, 1, s.Data.Y);
//进行预处理
if (needFilter && this._filters != null)
{
x = Preprocesser.ProcessForPredict(this._filters, x);
s.Data.Y = (double[])x.ToVector(MWArrayComponent.Real);
}
var c = this._comp.Clone();
//定义变量
double[] SR, MD, ND, Ylast;
var handler = Tools.ModelHandler;
if (this._annType != PLSAnnEnum.None)
{
var r = handler.PLS1Predictor(5, x, this._Scores, this._Loads, this._Weights, this._Bias, this._Score_Length, this._centerSpecData, this._centerCompValue, (int)this._method);
var toolHandler = Tools.ToolHandler;
// SR,MD,nd,XScores
Ylast = (double[])((MWNumericArray)r[0]).ToVector(MWArrayComponent.Real);
SR = (double[])((MWNumericArray)r[1]).ToVector(MWArrayComponent.Real);
MD = (double[])((MWNumericArray)r[2]).ToVector(MWArrayComponent.Real);
ND = (double[])((MWNumericArray)r[3]).ToVector(MWArrayComponent.Real);
var mscores = (MWNumericArray)toolHandler.Centring(1, r[4], this._ScoresMean)[0];
var annp = (MWNumericArray)handler.annp(1,
mscores,
this._annModel.w1,
this._annModel.b1,
this._annModel.w2,
this._annModel.b2,
this._annArgus.F1.GetDescription(),
this._annArgus.F2.GetDescription())[0];
Ylast[this._factor - 1] = annp.ToScalarDouble() + this._centerCompValue;
}
else
{
var r = handler.PLS1Predictor(5, x, this._Scores, this._Loads, this._Weights, this._Bias, this._Score_Length, this._centerSpecData, this._centerCompValue, (int)this._method);
Ylast = (double[])((MWNumericArray)r[0]).ToVector(MWArrayComponent.Real);
SR = (double[])((MWNumericArray)r[1]).ToVector(MWArrayComponent.Real);
MD = (double[])((MWNumericArray)r[2]).ToVector(MWArrayComponent.Real);
ND = (double[])((MWNumericArray)r[3]).ToVector(MWArrayComponent.Real);
}
c.PredictedValue = Ylast[this.Factor - 1];
// c.PredictedValue = this._Nonnegative && c.PredictedValue < 0 ? 0 : c.PredictedValue;
c.PredictedValue = this._Nonnegative && c.PredictedValue < 0 ? Math.Pow(10, -c.Eps)
: c.PredictedValue;
if (s.Components != null && s.Components.Contains(c.Name))
{
c.ActualValue = s.Components[c.Name].ActualValue;
c.Error = c.PredictedValue - c.ActualValue;
}
//加入判定条件，设置样式
int errorcount = 0;
if (MD[this._factor - 1] > this._MDMin)
{
errorcount++;
}
else if (SR[this._factor - 1] > this._SRMin)
{
errorcount++;
}
else if (ND[this._factor - 1] > this._NNDMin)
{
errorcount++;
}
switch (errorcount)
{
case 0:
c.State = ComponentStatu.Pass;
break;
case 1:
c.State = ComponentStatu.Green;
break;
case 2:
c.State = ComponentStatu.Blue;
break;
default:
c.State = ComponentStatu.Red;
break;
}
PLS1Result item = new PLS1Result()
{
Spec = s,
MDMin = this._MDMin,
NDMin = this._NNDMin,
SRMin = this._SRMin,
Comp = c,
MahDist = MD,
ND = ND,
SR = SR,
YLast = Ylast,
Factor = this.Factor
};
return(item);
}