/// <summary> /// Calculates the gradient of the log-likelihood function for Weibull with features. /// Provided below is the derivation in Latex. Paste it into an online /// Latex editor like https://www.overleaf.com/7650945xdjrrdzrmsdd#/26769609/ /// to read and and understand where the formulas came from. /// Let $f_i$ be the vector of features for the $i^{th}$ data point and $x_i$ be the duration it took to recover. /// Then, the Likelihood and log-likelihood functions are given by - /// \[L = \prod_{i=1}^n \log(pdf(x_i, W.f_i)) \] /// \[ll = \sum_{i=1}^n \log(pdf(x_i, W.f_i))\] /// \[\frac{\partial(ll)}{\partial W} = \frac{1}{pdf(x_i, W.f_i)} f_i(\partial \theta_i)^T \] /// Where, /// \[\theta_i = W.f_i = [\kappa, \lambda]\] here $\kappa$, once the sigmoid is applied to it /// to ensure it being positive, is the shape paramter of the Weibull and $\lambda$ once the /// sigmoid is applied is the scale parameter. /// \[\partial \theta_i = \frac{\partial(pdf(x,\theta_i))}{\partial(\theta_i)}\] /// </summary> /// <param name="w">The matrix of parameters (2 x # of features)</param> /// <param name="fSamples">The features corresponding to the organic recoveries. /// Number of rows should be same as this.OrganicRecoveryDurations.Length</param> /// <param name="fCensored">The features corresponding to the reboots. /// Number of rows should be the same as this.InorganicRecoveryDurations.Length</param> /// <param name="eps">Since we divide by the PDF and survival functions, we need to make sure /// they don't get lower than a threshold or the gradients blow up. /// This parameter is a lower bound on them.</param> /// <param name="bailOutSurvivalValue">The value to turn to for survival when gradients start blowing up.</param> /// <returns>The gradient of the log-likelihood with the matrix of parameters, w</returns> public Matrix <double> GradLL(Matrix <double> w, Matrix <double> fSamples, Matrix <double> fCensored, double eps = 1e-8, double bailOutSurvivalValue = 10.0) { IReadOnlyCollection <double> t = this.OrganicRecoveryDurations; IReadOnlyCollection <double> x = this.InorganicRecoveryDurations; Matrix <double> gradW = Matrix <double> .Build.Dense(w.RowCount, w.ColumnCount); Sigmoid sShape = new Sigmoid(this.ShapeUpperBound); Sigmoid sScale = new Sigmoid(this.ScaleUpperBound); for (int i = 0; i < fSamples.RowCount; i++) { Vector <double> currentRow = fSamples.Row(i); Vector <double> theta = w.Multiply(currentRow); // A 2 dim vector that will be converted to the 2 Weibull params. double shape = sShape.Transform(theta[0]); // To prevent the parameters from becoming negative, we use sigmoids. double scale = sScale.Transform(theta[1]); double pdf = this.PDF(t.ElementAt(i), shape, scale); Vector <double> pdfGrad = this.GradPDF(t.ElementAt(i), shape, scale, pdf); Vector <double> sigmoidGrad = Vector <double> .Build.DenseOfArray(new double[] { sShape.Grad(theta[0]), sScale.Grad(theta[1]) }); Vector <double> delTheta = pdfGrad.PointwiseMultiply(sigmoidGrad); // Since we used the sigmoids, the product rule dictates that we point-wise multiply the derivatives. // Since we are dividing the matrix by the pdf, we need to be careful it doesn't blow up. if (pdf > eps) { gradW = gradW.Add(delTheta.OuterProduct(currentRow).Divide(pdf)); // del/del(W) log(lik(x,W.f)) = 1/lik(x,W.f) * f. (del(lik(x))/del(W.f))^T } else { double survival = this.Survival(bailOutSurvivalValue, shape, scale); Vector <double> survivalGrad = this.GradSurvival(bailOutSurvivalValue, shape, scale, survival); delTheta = survivalGrad.PointwiseMultiply(sigmoidGrad); gradW = gradW.Add(delTheta.OuterProduct(currentRow).Divide(survival)); } } for (int i = 0; i < fCensored.RowCount; i++) { Vector <double> currentRow = fCensored.Row(i); Vector <double> theta = w.Multiply(currentRow); double shape = sShape.Transform(theta[0]); double scale = sScale.Transform(theta[1]); double survival = this.Survival(x.ElementAt(i), shape, scale); Vector <double> survivalGrad = this.GradSurvival(x.ElementAt(i), shape, scale, survival); Vector <double> sigmoidGrad = Vector <double> .Build.DenseOfArray(new double[] { sShape.Grad(theta[0]), sScale.Grad(theta[1]) }); Vector <double> delTheta = survivalGrad.PointwiseMultiply(sigmoidGrad); // Since we are dividing the matrix by the survival, we need to be careful it doesn't blow up. if (survival > eps) { gradW = gradW.Add(delTheta.OuterProduct(currentRow).Divide(survival)); } else { survival = this.Survival(bailOutSurvivalValue, shape, scale); survivalGrad = this.GradSurvival(bailOutSurvivalValue, shape, scale, survival); delTheta = survivalGrad.PointwiseMultiply(sigmoidGrad); gradW = gradW.Add(delTheta.OuterProduct(currentRow).Divide(survival)); } } return(gradW); }
/// <summary> /// Calculating gradient of the loglikelihood using GradLPDF and GradLSurvival. /// </summary> /// <param name="w">The current weight where we want to evaluate the gradient.</param> /// <param name="fSamples">Organic recover time data.</param> /// <param name="fCensored">Inorganic recover time data.</param> /// <param name="eps">Threshold for small pdf.</param> /// <param name="bailOutSurvivalValue">Threshold for small survival function.</param> /// <returns>The gradient of the log-likelihood function with respect to w (the weights). /// </returns> public Matrix <double> GradLL2(Matrix <double> w, Matrix <double> fSamples, Matrix <double> fCensored, double eps = 1e-8, double bailOutSurvivalValue = 10.0) { List <double> t = this.OrganicRecoveryDurations; List <double> x = this.InorganicRecoveryDurations; Matrix <double> gradW = Matrix <double> .Build.Dense(w.RowCount, w.ColumnCount); Sigmoid sShape = new Sigmoid(this.ShapeUpperBound); Sigmoid sScale = new Sigmoid(this.ScaleUpperBound); for (int i = 0; i < fSamples.RowCount; i++) { Vector <double> currentRow = fSamples.Row(i); // A 2 dim vector that will be converted to the 2 Weibull params. Vector <double> theta = w.Multiply(currentRow); // To prevent the parameters from becoming negative, we use sigmoids. double shape = sShape.Transform(theta[0]); double scale = sScale.Transform(theta[1]); //// double pdf = this.PDF(t.ElementAt(i), kappa, lambda); Vector <double> lpdfGrad = this.GradLPDF(t.ElementAt(i), shape, scale); Vector <double> sigmoidGrad = Vector <double> .Build.DenseOfArray(new double[] { sShape.Grad(theta[0]), sScale.Grad(theta[1]) }); // Since we used the sigmoids, the product rule dictates that // we point-wise multiply the derivatives. Vector <double> delTheta = lpdfGrad.PointwiseMultiply(sigmoidGrad); // currentRow is just feature vector. gradW = gradW.Add(delTheta.OuterProduct(currentRow)); if (double.IsNaN(gradW[0, 0]) || double.IsPositiveInfinity(gradW[0, 0]) || double.IsNegativeInfinity(gradW[0, 0])) { // Hopefully, we will never enter this code path. throw new Exception("The moment we feared has arrived, gradient has blown" + "up due to samples data." + "First, try tightening the upper bounds for the shape and scale parameters" + "and if that doesn't work, add a break point here. My suspicion in delTheta"); } /* Since we are dividing the matrix by the pdf, we need to be * careful it doesn't blow up. * if (pdf > eps) * { * gradW = gradW.Add(delTheta.OuterProduct(currentRow).Divide(pdf)); * // del/del(W) log(lik(x,W.f)) = 1/lik(x,W.f) * f. (del(lik(x))/del(W.f))^T * } * else * { * double survival = this.Survival(bailOutSurvivalValue, kappa, lambda); * Vector<double> survivalGrad = this.GradSurvival(bailOutSurvivalValue, * kappa, lambda, survival); * delTheta = survivalGrad.PointwiseMultiply(sigmoidGrad); * gradW = gradW.Add(delTheta.OuterProduct(currentRow).Divide(survival)); * } */ } for (int i = 0; i < fCensored.RowCount; i++) { Vector <double> currentRow = fCensored.Row(i); Vector <double> theta = w.Multiply(currentRow); // To prevent the parameters from becoming negative, we use sigmoids. double shape = sShape.Transform(theta[0]); double scale = sScale.Transform(theta[1]); //// double survival = this.Survival(t.ElementAt(i), kappa, lambda); Vector <double> lsurvivalGrad = this.GradLSurvival(x.ElementAt(i), shape, scale); Vector <double> sigmoidGrad = Vector <double> .Build.DenseOfArray( new double[] { sShape.Grad(theta[0]), sScale.Grad(theta[1]) }); Vector <double> delTheta = lsurvivalGrad.PointwiseMultiply(sigmoidGrad); gradW = gradW.Add(delTheta.OuterProduct(currentRow)); if (double.IsNaN(gradW[0, 0]) || double.IsPositiveInfinity(gradW[0, 0]) || double.IsNegativeInfinity(gradW[0, 0])) { throw new Exception("The moment we feared has arrived, gradient has blown" + "up due to censored data." + "First, try tightening the upper bounds for the shape and scale parameters" + "and if that doesn't work, add a break point here. My suspicion in delTheta"); } /* * Since we are dividing the matrix by the survival, we need to be * careful it doesn't blow up. * if (survival > eps) * { * } * else * { * survival = this.Survival(bailOutSurvivalValue, kappa, lambda); * survivalGrad = this.GradSurvival(bailOutSurvivalValue, kappa, lambda, survival); * delTheta = survivalGrad.PointwiseMultiply(sigmoidGrad); * gradW = gradW.Add(delTheta.OuterProduct(currentRow).Divide(survival)); * } */ } return(gradW); }