public void Train(SpecBase lib, bool needFilter = true)
{
//过滤掉性质有NaN的数据
lib.FilterNaN();
int[] idxs = lib.Specs.Select((d, idx) => new { s = d, idx = idx }).Where(d => d.s.Usage != UsageTypeEnum.Ignore).Select(d => d.idx).ToArray();
this._lib = Serialize.DeepClone <SpecBase>(lib.SubLib(idxs));
if (needFilter && this._filters != null)
{
this._lib.SetX(Preprocesser.Process(this._filters, this._lib), true);
}
//PCA分解
var handler = Tools.ModelHandler;
if (this._maxRank < 1)
{
this._maxRank = 10;
}
this._maxRank = Math.Min(this._maxRank, this._lib.X.Dimensions[1]);
var r = handler.IdentifyTrain(3, this._lib.X, this._maxRank);
this._p = (MWNumericArray)r[0];
this._w = (MWNumericArray)r[1];
this._t = (MWNumericArray)r[2];
this._trained = true;
}
public void Train(SpecBase lib, bool needFilter = true)
{
//过滤掉性质有NaN的数据
lib.FilterNaN();
int[] idxs = lib.Specs.Select((d, idx) => new { s = d, idx = idx }).Where(d => d.s.Usage != UsageTypeEnum.Ignore).Select(d => d.idx).ToArray();
this._lib = Serialize.DeepClone <SpecBase>(lib.SubLib(idxs));
if (needFilter && this._filters != null)
{
this._lib.SetX(Preprocesser.Process(this._filters, this._lib), true);
}
this._trained = true;
}
public IdentifyResult[] CrossValidation(SpecBase lib, bool needFilter = true, int numOfId = 5)
{
//过滤掉性质有NaN的数据
lib.FilterNaN();
var clib = lib.SubLib(UsageTypeEnum.Calibrate);
if (clib.Count == 0)
{
return(new List <IdentifyResult>().ToArray());
}
var cx = clib.X;
if (needFilter && this._filters != null)
{
cx = Preprocesser.Process(this._filters, cx);
}
if (!this._trained)
{
this.Train(clib);
}
var handler = Tools.ModelHandler;
//Tools.Save(cx, "x.mat", "x");
var rlst = handler.IdentifyCrossValidation(3, cx, this._wind, this._maxNum, this._maxRank);
var lst = new IdentifyResult[cx.Dimensions[1]];
clib.X = cx;
for (int i = 0; i < lst.Length; i++)
{
var SQ = Tools.SelectColumn((MWNumericArray)rlst[0], i + 1);
var tq = Tools.SelectColumn((MWNumericArray)rlst[1], i + 1);
var idx = Tools.SelectColumn((MWNumericArray)rlst[2], i + 1);
var item = new IdentifyResult()
{
MinSQ = this._minSQ,
MinTQ = this._TQ,
};
item.Spec = clib[i].Clone();
item.Components = item.Spec.Components.Clone();
var itemlst = new List <IdentifyItem>();
for (int k = 0; k < Math.Min(this._maxNum, idx.NumberOfElements); k++)
{
var adidx = (int)idx[k + 1] - 1;
if (adidx >= clib.Count || adidx < 0)
{
break;
}
itemlst.Add(new IdentifyItem()
{
Parent = item,
SQ = (double)SQ[k + 1],
TQ = (double)tq[k + 1],
Spec = clib[adidx],
SpecOriginal = item.Spec,
Wind = this._wind
});
}
item.Items = itemlst.ToArray();
item = GetPredictValue(item, item.Items.Length, numOfId);
item.IsId = item.Items.Where(d => d.Result).Count() > 0;
lst[i] = item;
}
return(lst);
}
public FittingResult[] Validation(SpecBase lib, bool needFilter = true, int numOfId = 5)
{
//过滤掉性质有NaN的数据
lib.FilterNaN();
var clib = lib.SubLib(UsageTypeEnum.Calibrate);
var vlib = lib.SubLib(UsageTypeEnum.Validate);
if (clib.Count == 0 || vlib.Count == 0)
{
return(new List <FittingResult>().ToArray());
}
var cx = clib.X;
var vx = vlib.X;
if (needFilter && this._filters != null)
{
cx = Preprocesser.Process(this._filters, cx);
clib.X = cx;
vx = Preprocesser.ProcessForPredict(this._filters, vx);
vlib.X = vx;
}
var handler = Tools.ModelHandler;
var rlst = handler.FitValidation(5, cx, vx, this._wind, (MWNumericArray)this._region.VarIndex);
var allTQ = (MWNumericArray)rlst[0];
var allSQ = (MWNumericArray)rlst[1];
var allRatio = (MWNumericArray)rlst[2];
var allIdx = (MWNumericArray)rlst[3];
var allFit = (MWNumericArray)rlst[4];
var lst = new FittingResult[vx.Dimensions[1]];
clib.X = cx;
vlib.X = vx;
for (int i = 0; i < lst.Length; i++)
{
var ratio = Tools.SelectColumn(allRatio, i + 1);
var idx = Tools.SelectColumn(allIdx, i + 1);
var fit = Tools.SelectColumn(allFit, i + 1);
var fitspec = vlib[i].Clone();
fitspec.Components = clib.Components.Clone();
fitspec.Data.Y = (double[])fit.ToVector(MWArrayComponent.Real);
var item = new FittingResult()
{
SQ = (double)allSQ[i + 1],
TQ = (double)allTQ[i + 1],
FitSpec = fitspec,
Wind = this._wind,
VarIndex = this._region.VarIndex
};
item.SpecOriginal = vlib[i].Clone();
var flst = new List <FittingSpec>();
for (int k = 1; k <= ratio.NumberOfElements; k++)
{
if ((double)ratio[k] <= double.Epsilon)
{
break;
}
flst.Add(new FittingSpec()
{
Spec = clib[(int)idx[k] - 1],
Rate = (double)ratio[k]
});
}
item.Specs = flst.ToArray();
this.getResult(ref item);
item.Result = item.TQ >= this._TQ && item.SQ >= this._minSQ;
lst[i] = item;
}
return(lst);
}
/// <summary>
/// 训练
/// </summary>
/// <param name="lib">光谱库</param>
/// <param name="needFilter">是否需要前处理</param>
public void Train(SpecBase libb, bool needFilter = true)
{
var lib = libb.Clone();
//过滤掉性质有NaN的数据
lib.FilterNaN(this._comp);
foreach (var s in lib.Specs)
{
if (this._OutlierNames.Contains(s.Name))
{
s.Usage = UsageTypeEnum.Ignore;
}
}
// lib.Specs = lib.Specs.Where(d => !this._OutlierNames.Contains(d.Name)).ToList();
var sublib = lib.SubLib(UsageTypeEnum.Calibrate);
var cx = sublib.X;
var cy = sublib.GetY(this._comp, true);
if (needFilter && this._filters != null)
{
cx = Preprocesser.Process(this._filters, cx);
}
var handler = Tools.ModelHandler;
var pls = handler.PLS1Train(11, cx, cy, this._maxFactor, (int)this._method);
this._Scores = (MWNumericArray)pls[0];
this._Loads = (MWNumericArray)pls[1];
this._Weights = (MWNumericArray)pls[2];
this._Bias = (MWNumericArray)pls[3];
this._Score_Length = (MWNumericArray)pls[4];
this._centerSpecData = (MWNumericArray)pls[5];
this._centerCompValue = ((MWNumericArray)pls[6]).ToScalarDouble();
this._Mdt = (double[])((MWNumericArray)pls[7]).ToVector(MWArrayComponent.Real);
this._NNdt = (double[])((MWNumericArray)pls[8]).ToVector(MWArrayComponent.Real);
this._sec = (double[])((MWNumericArray)pls[9]).ToVector(MWArrayComponent.Real);
this._cr = (double[])((MWNumericArray)pls[10]).ToVector(MWArrayComponent.Real);
if (this._annType != PLSAnnEnum.None)
{
double mse = 0;
//中心化Scores,cy
var toolHandler = new RIPPMatlab.Tools();
this._ScoresMean = (MWNumericArray)toolHandler.Mean(1, this._Scores)[0];
var mScores = (MWNumericArray)toolHandler.Centring(1, this._Scores, this._ScoresMean)[0];
var my = (MWNumericArray)toolHandler.Centring(1, cy, this._centerCompValue)[0];
//
var guidlib = lib.SubLib(UsageTypeEnum.Guide);
bool needgruid = this._annArgus.IsGuard && guidlib.Specs.Count > 0;
MWArray[] ann;
if (needgruid) //有监控
{
var gx = guidlib.X;
var gy = guidlib.GetY(this._comp, true);
if (needFilter && this._filters != null)
{
gx = Preprocesser.Process(this._filters, gx);
}
var gpls = handler.PLS1Predictor(5, gx, this.Scores, this.Loads, this.Weights, this.Bias, this.Score_Length, this.CenterSpecData, this.CenterCompValue, (int)this._method);
var gScoresm = (MWNumericArray)toolHandler.Centring(1, gpls[4], this._ScoresMean)[0];
var gym = (MWNumericArray)toolHandler.Centring(1, gy, this._centerCompValue)[0];
ann = handler.bann2(6,
mScores,
my,
gScoresm,
gym,
this._annArgus.FuncTrain.GetDescription(),
this._annArgus.NumHidden,
this._annArgus.F1.GetDescription(),
this._annArgus.F2.GetDescription(),
this._annArgus.Epochs,
this._annArgus.Target,
this._annArgus.TimesAvg,
this._annArgus.TimesRepeat);
this._sem = ((MWNumericArray)ann[4]).ToScalarDouble();
this._mr = ((MWNumericArray)ann[5]).ToScalarDouble();
}
else
{
ann = handler.bann1(4,
mScores,
my,
this._annArgus.FuncTrain.GetDescription(),
this._annArgus.NumHidden,
this._annArgus.F1.GetDescription(),
this._annArgus.F2.GetDescription(),
this._annArgus.Epochs,
this._annArgus.Target,
this._annArgus.TimesAvg,
this._annArgus.TimesRepeat);
}
this._annModel.w1 = (MWNumericArray)ann[0];
this._annModel.b1 = (MWNumericArray)ann[1];
this._annModel.w2 = (MWNumericArray)ann[2];
this._annModel.b2 = (MWNumericArray)ann[3];
////重新计算CR
//double ye = 0, yea = 0;
//var yp = (MWNumericArray)ann[4];
//for (int k = 0; k < cy.NumberOfElements; k++)
//{
// ye += (cy[k + 1].ToScalarDouble() - this._centerCompValue - yp[k + 1].ToScalarDouble()) * (cy[k + 1].ToScalarDouble() - this._centerCompValue - yp[k + 1].ToScalarDouble());
// yea += (cy[k + 1].ToScalarDouble() - this._centerCompValue) * (cy[k + 1].ToScalarDouble() - this._centerCompValue);
//}
//this._cr[this._maxFactor - 1] = ye / (ye + yea);
}
this._trained = true;
lib.Dispose();
sublib.Dispose();
cx.Dispose();
cy.Dispose();
GC.Collect();
}
public PLS1Result[] CrossValidation(SpecBase libb, bool needFilter = true, int numOfId = 5)
{
var lib = libb.Clone();
//过滤掉性质有NaN的数据
lib.FilterNaN(this._comp);
foreach (var s in lib.Specs)
{
if (this._OutlierNames.Contains(s.Name))
{
s.Usage = UsageTypeEnum.Ignore;
}
}
//lib.Specs = lib.Specs.Where(d => !this._OutlierNames.Contains(d.Name)).ToList();
if (!this._trained)
{
this.Train(lib, needFilter);
}
var clib = lib.SubLib(UsageTypeEnum.Calibrate);
if (clib.Count == 0)
{
return(null);
}
if (needFilter && this._filters != null)
{
clib.X = Preprocesser.Process(this._filters, clib.X);
}
var handler = Tools.ModelHandler;
MWNumericArray Ylast, SR, MD, nd;
if (this._annType != PLSAnnEnum.None)
{
var model = Serialize.DeepClone <PLSSubModel>(this);
model.Train(lib, true);
var pls = handler.PLS1Predictor(5, clib.X, model.Scores, model.Loads, model.Weights, model.Bias, model.Score_Length, model.CenterSpecData, model.CenterCompValue, (int)this._method);
var toolHandler = Tools.ToolHandler;
Ylast = (MWNumericArray)pls[0];
SR = (MWNumericArray)pls[1];
MD = (MWNumericArray)pls[2];
nd = (MWNumericArray)pls[3];
var mscores = (MWNumericArray)toolHandler.Centring(1, pls[4], model.ScoresMean)[0];
double[] annrsult;
var annp = handler.annp(1,
mscores,
model.ANNModel.w1,
model.ANNModel.b1,
model.ANNModel.w2,
model.ANNModel.b2,
model.ANNAgrus.F1.GetDescription(),
model.ANNAgrus.F2.GetDescription())[0];
annrsult = (double[])((MWNumericArray)annp).ToVector(MWArrayComponent.Real);
for (int row = 0; row < annrsult.Length; row++)
{
Ylast[row + 1, this._factor] = annrsult[row] + model.CenterCompValue;
}
}
else
{
var pls = handler.PLS1CrossValidation(4, clib.X, clib.GetY(this._comp, true), this._maxFactor, (int)this._method);
Ylast = (MWNumericArray)pls[0];
SR = (MWNumericArray)pls[1];
MD = (MWNumericArray)pls[2];
nd = (MWNumericArray)pls[3];
}
var items = new List <PLS1Result>();
for (int i = 0; i < clib.Count; i++)
{
var c = this._comp.Clone();
var s = clib[i];
if (s.Components != null && s.Components.Contains(c.Name))
{
c.ActualValue = s.Components[c.Name].ActualValue;
}
var r = new PLS1Result
{
MDMin = this._MDMin,
NDMin = this._NNDMin,
SRMin = this._SRMin,
Comp = c,
Spec = s,
YLast = (double[])Tools.SelectRow(Ylast, i + 1).ToVector(MWArrayComponent.Real),
SR = (double[])Tools.SelectRow(SR, i + 1).ToVector(MWArrayComponent.Real),
MahDist = (double[])Tools.SelectRow(MD, i + 1).ToVector(MWArrayComponent.Real),
ND = (double[])Tools.SelectRow(nd, i + 1).ToVector(MWArrayComponent.Real),
Factor = this.Factor
};
if (!double.IsNaN(r.YLast[0]))
{
items.Add(r);
}
}
lib.Dispose();
clib.Dispose();
GC.Collect();
return(items.ToArray());
}