private static void Main()
//****************************************************************************80
//
// Purpose:
//
// MAIN is the main program for PWL_APPROX_1D_TEST.
//
// Discussion:
//
// PWL_APPROX_1D_TEST tests the PWL_APPROX_1D library.
//
// Licensing:
//
// This code is distributed under the GNU LGPL license.
//
// Modified:
//
// 09 October 2012
//
// Author:
//
// John Burkardt
//
{
int[] nc_test = { 2, 4, 8, 16 };
const int nc_test_num = 4;
int[] nd_test = { 16, 64 };
const int nd_test_num = 2;
int prob;
Console.WriteLine("");
Console.WriteLine("PWL_APPROX_1D_TEST:");
Console.WriteLine(" Test the PWL_APPROX_1D library.");
Console.WriteLine(" The QR_SOLVE library is needed.");
Console.WriteLine(" The R8LIB library is needed.");
Console.WriteLine(" The test also needs the TEST_INTERP_1D library.");
int prob_num = ProbabilityFunctions.p00_prob_num();
for (prob = 1; prob <= prob_num; prob++)
{
int j;
for (j = 0; j < nc_test_num; j++)
{
int nc = nc_test[j];
int k;
for (k = 0; k < nd_test_num; k++)
{
int nd = nd_test[k];
test01(prob, nc, nd);
}
}
}
Console.WriteLine("");
Console.WriteLine("PWL_APPROX_1D_TEST:");
Console.WriteLine(" Normal end of execution.");
Console.WriteLine("");
}
private protected GamTrainerBase(IHostEnvironment env,
string name,
SchemaShape.Column label,
string featureColumnName,
string weightCrowGroupColumnName,
int numberOfIterations,
double learningRate,
int maximumBinCountPerFeature)
: base(Contracts.CheckRef(env, nameof(env)).Register(name), TrainerUtils.MakeR4VecFeature(featureColumnName), label, TrainerUtils.MakeR4ScalarWeightColumn(weightCrowGroupColumnName))
{
GamTrainerOptions = new TOptions();
GamTrainerOptions.NumberOfIterations = numberOfIterations;
GamTrainerOptions.LearningRate = learningRate;
GamTrainerOptions.MaximumBinCountPerFeature = maximumBinCountPerFeature;
GamTrainerOptions.LabelColumnName = label.Name;
GamTrainerOptions.FeatureColumnName = featureColumnName;
if (weightCrowGroupColumnName != null)
{
GamTrainerOptions.ExampleWeightColumnName = weightCrowGroupColumnName;
}
Info = new TrainerInfo(normalization: false, calibration: NeedCalibration, caching: false, supportValid: true);
_gainConfidenceInSquaredStandardDeviations = Math.Pow(ProbabilityFunctions.Probit(1 - (1 - GamTrainerOptions.GainConfidenceLevel) * 0.5), 2);
_entropyCoefficient = GamTrainerOptions.EntropyCoefficient * 1e-6;
InitializeThreads();
}
private protected GamTrainerBase(IHostEnvironment env,
string name,
SchemaShape.Column label,
string featureColumn,
string weightColumn,
int numIterations,
double learningRate,
int maxBins)
: base(Contracts.CheckRef(env, nameof(env)).Register(name), TrainerUtils.MakeR4VecFeature(featureColumn), label, TrainerUtils.MakeR4ScalarWeightColumn(weightColumn))
{
Args = new TArgs();
Args.NumIterations = numIterations;
Args.LearningRates = learningRate;
Args.MaxBins = maxBins;
Args.LabelColumn = label.Name;
Args.FeatureColumn = featureColumn;
if (weightColumn != null)
{
Args.WeightColumn = weightColumn;
}
Info = new TrainerInfo(normalization: false, calibration: NeedCalibration, caching: false, supportValid: true);
_gainConfidenceInSquaredStandardDeviations = Math.Pow(ProbabilityFunctions.Probit(1 - (1 - Args.GainConfidenceLevel) * 0.5), 2);
_entropyCoefficient = Args.EntropyCoefficient * 1e-6;
InitializeThreads();
}
public static double NormalCdf(double x, double mean, double variance)
{
double centered = x - mean;
double ztrans = centered / (Math.Sqrt(variance) * Math.Sqrt(2));
return(0.5 * (1 + ProbabilityFunctions.Erf(ztrans)));
}
public static bool TryGetBiasStatistics(LinearModelStatistics stats, Single bias, out Single stdError, out Single zScore, out Single pValue)
{
if (!stats._coeffStdError.HasValue)
{
stdError = 0;
zScore = 0;
pValue = 0;
return(false);
}
const Double sqrt2 = 1.41421356237; // Math.Sqrt(2);
stdError = stats._coeffStdError.Value.Values[0];
Contracts.Assert(stdError == stats._coeffStdError.Value.GetItemOrDefault(0));
zScore = bias / stdError;
pValue = 1 - (Single)ProbabilityFunctions.Erf(Math.Abs(zScore / sqrt2));
return(true);
}
public Transition(double onRoadDistanceInMeters, double haversineDistanceInMeters, DirectedRoad from, DirectedRoad to, MapMatcherParameters parameters)
{
OnRoadDistInMeters = onRoadDistanceInMeters;
HaversineDistInMeters = haversineDistanceInMeters;
From = from;
To = to;
// COMPUTE TRANSITION PROBABILITY
double diffInMeters = Math.Abs(haversineDistanceInMeters - onRoadDistanceInMeters);
if (diffInMeters > parameters.TransitionDiffThreshold)
{
Probability = 0;
}
else
{
Probability = ProbabilityFunctions.ExponentialDistribution(diffInMeters, parameters.Beta);
}
}
private IEnumerable <CoefficientStatistics> GetUnorderedCoefficientStatistics(LinearBinaryPredictor parent, RoleMappedSchema schema)
{
Contracts.AssertValue(_env);
_env.CheckValue(parent, nameof(parent));
if (!_coeffStdError.HasValue)
{
yield break;
}
var weights = parent.Weights2 as IReadOnlyList <Single>;
_env.Assert(_paramCount == 1 || weights != null);
_env.Assert(_coeffStdError.Value.Length == weights.Count + 1);
var names = default(VBuffer <ReadOnlyMemory <char> >);
MetadataUtils.GetSlotNames(schema, RoleMappedSchema.ColumnRole.Feature, weights.Count, ref names);
Single[] stdErrorValues = _coeffStdError.Value.Values;
const Double sqrt2 = 1.41421356237; // Math.Sqrt(2);
bool denseStdError = _coeffStdError.Value.IsDense;
int[] stdErrorIndices = _coeffStdError.Value.Indices;
Single[] zScores = new Single[_paramCount - 1];
for (int i = 1; i < _paramCount; i++)
{
int wi = denseStdError ? i - 1 : stdErrorIndices[i] - 1;
_env.Assert(0 <= wi && wi < weights.Count);
var name = names.GetItemOrDefault(wi).ToString();
if (string.IsNullOrEmpty(name))
{
name = $"f{wi}";
}
var weight = weights[wi];
var stdError = stdErrorValues[i];
var zScore = zScores[i - 1] = weight / stdError;
var pValue = 1 - (Single)ProbabilityFunctions.Erf(Math.Abs(zScore / sqrt2));
yield return(new CoefficientStatistics(name, weight, stdError, zScore, pValue));
}
}
private Double ComputeKernelPValue(Double rawScore)
{
int i;
int n = RawScoreBuffer.Count;
if (n == 0)
{
return(0.5);
}
Double pValue = 0;
Double bandWidth = Math.Sqrt(2) * ((n == 1) ? 1 : Math.Sqrt(_sumSquaredDist) / n);
bandWidth = Math.Max(bandWidth, 1e-6);
Double diff;
for (i = 0; i < n; ++i)
{
diff = rawScore - RawScoreBuffer[i];
pValue -= ProbabilityFunctions.Erf(diff / bandWidth);
_sumSquaredDist += diff * diff;
}
pValue = 0.5 + pValue / (2 * n);
if (RawScoreBuffer.IsFull)
{
for (i = 1; i < n; ++i)
{
diff = RawScoreBuffer[0] - RawScoreBuffer[i];
_sumSquaredDist -= diff * diff;
}
diff = RawScoreBuffer[0] - rawScore;
_sumSquaredDist -= diff * diff;
}
return(pValue);
}
private protected GamTrainerBase(IHostEnvironment env, TOptions options, string name, SchemaShape.Column label)
: base(Contracts.CheckRef(env, nameof(env)).Register(name), TrainerUtils.MakeR4VecFeature(options.FeatureColumnName),
label, TrainerUtils.MakeR4ScalarWeightColumn(options.ExampleWeightColumnName))
{
Contracts.CheckValue(env, nameof(env));
Host.CheckValue(options, nameof(options));
Host.CheckParam(options.LearningRate > 0, nameof(options.LearningRate), "Must be positive.");
Host.CheckParam(options.NumberOfThreads == null || options.NumberOfThreads > 0, nameof(options.NumberOfThreads), "Must be positive.");
Host.CheckParam(0 <= options.EntropyCoefficient && options.EntropyCoefficient <= 1, nameof(options.EntropyCoefficient), "Must be in [0, 1].");
Host.CheckParam(0 <= options.GainConfidenceLevel && options.GainConfidenceLevel < 1, nameof(options.GainConfidenceLevel), "Must be in [0, 1).");
Host.CheckParam(0 < options.MaximumBinCountPerFeature, nameof(options.MaximumBinCountPerFeature), "Must be positive.");
Host.CheckParam(0 < options.NumberOfIterations, nameof(options.NumberOfIterations), "Must be positive.");
Host.CheckParam(0 < options.MinimumExampleCountPerLeaf, nameof(options.MinimumExampleCountPerLeaf), "Must be positive.");
GamTrainerOptions = options;
Info = new TrainerInfo(normalization: false, calibration: NeedCalibration, caching: false, supportValid: true);
_gainConfidenceInSquaredStandardDeviations = Math.Pow(ProbabilityFunctions.Probit(1 - (1 - GamTrainerOptions.GainConfidenceLevel) * 0.5), 2);
_entropyCoefficient = GamTrainerOptions.EntropyCoefficient * 1e-6;
InitializeThreads();
}
private protected GamTrainerBase(IHostEnvironment env, TArgs args, string name, SchemaShape.Column label)
: base(Contracts.CheckRef(env, nameof(env)).Register(name), TrainerUtils.MakeR4VecFeature(args.FeatureColumn),
label, TrainerUtils.MakeR4ScalarWeightColumn(args.WeightColumn))
{
Contracts.CheckValue(env, nameof(env));
Host.CheckValue(args, nameof(args));
Host.CheckParam(args.LearningRates > 0, nameof(args.LearningRates), "Must be positive.");
Host.CheckParam(args.NumThreads == null || args.NumThreads > 0, nameof(args.NumThreads), "Must be positive.");
Host.CheckParam(0 <= args.EntropyCoefficient && args.EntropyCoefficient <= 1, nameof(args.EntropyCoefficient), "Must be in [0, 1].");
Host.CheckParam(0 <= args.GainConfidenceLevel && args.GainConfidenceLevel < 1, nameof(args.GainConfidenceLevel), "Must be in [0, 1).");
Host.CheckParam(0 < args.MaxBins, nameof(args.MaxBins), "Must be posittive.");
Host.CheckParam(0 < args.NumIterations, nameof(args.NumIterations), "Must be positive.");
Host.CheckParam(0 < args.MinDocuments, nameof(args.MinDocuments), "Must be positive.");
Args = args;
Info = new TrainerInfo(normalization: false, calibration: NeedCalibration, caching: false, supportValid: true);
_gainConfidenceInSquaredStandardDeviations = Math.Pow(ProbabilityFunctions.Probit(1 - (1 - Args.GainConfidenceLevel) * 0.5), 2);
_entropyCoefficient = Args.EntropyCoefficient * 1e-6;
InitializeThreads();
}
public float GetT(T keyFromState)
{
return(ProbabilityFunctions.getT(map.IndexOfKey(keyFromState), map.Keys.Count));
}
public static double erfinv(double x) => ProbabilityFunctions.Erfinv(x);
/// <summary>
/// Find the actual period based on best frequency and second best frequency:
/// Pick the best frequency by inspecting the auto-correlation energy (pick the highest) in time-domain.
/// In the normal case, usually, when the time series is with period T, then the best frequency is N/T,
/// while the second frequency would be N/2T, because period = T implies period = nT, where n is an integer.
/// In such a case, smaller period will win out on the autu-correlation energy list, due to the property
/// of auto-correlation.
/// </summary>
/// <param name="values">The auto-correlation function of the augmented time series</param>
/// <param name="bestFrequency">The best frequency candidate</param>
/// <param name="secondFrequency">The second best frequency candidate</param>
/// <param name="timeSeriesLength">The length of the original time series, this is used for post
/// processing to reduce false positive
/// </param>
/// <param name="randomnessThreshold">Randomness threshold that specifies how confidently the input
/// values follow a predictable pattern recurring as seasonal data.
/// </param>
/// <returns>The period detected by check best frequency and second best frequency</returns>
private static int FindActualPeriod(Complex[] values, int bestFrequency, int secondFrequency, int timeSeriesLength, double randomnessThreshold)
{
int firstPeriod = -1;
int secondPeriod = -1;
double firstTimeDomainEnergy = -1;
double secondTimeDomainEnergy = -1;
firstPeriod = FindBestPeriod(values, bestFrequency, timeSeriesLength, out firstTimeDomainEnergy);
if (secondFrequency != -1)
{
secondPeriod = FindBestPeriod(values, secondFrequency, timeSeriesLength, out secondTimeDomainEnergy);
}
if (firstPeriod == -1 && secondPeriod == -1)
{
return(-1);
}
int truePeriod;
double trueTimeDomainEnergy;
if (firstPeriod == -1)
{
truePeriod = secondPeriod;
trueTimeDomainEnergy = secondTimeDomainEnergy;
}
else if (secondPeriod == -1)
{
truePeriod = firstPeriod;
trueTimeDomainEnergy = firstTimeDomainEnergy;
}
else
{
if (firstPeriod == secondPeriod)
{
truePeriod = firstPeriod;
trueTimeDomainEnergy = firstTimeDomainEnergy;
}
else
{
// hueristic: if the second frequency is with higher energy in time domain, we think it is a better candidate
if (secondTimeDomainEnergy > firstTimeDomainEnergy * 1.05)
{
truePeriod = secondPeriod;
trueTimeDomainEnergy = secondTimeDomainEnergy;
}
else
{
truePeriod = firstPeriod;
trueTimeDomainEnergy = firstTimeDomainEnergy;
}
}
}
trueTimeDomainEnergy /= values[0].Real;
/* This is a key equation, which is named the "testing for randomness with the correlogram". /ref: http://www.ltrr.arizona.edu/~dmeko/notes_3.pdf
* 1.96 is for the 2-sigma, which has 95% statistical confidence. 2.58 is for 99% confidence, 2.85 for 99.5% confidence
* increasing the threshold aims to mitigate the fake seasonal component caused by outliers. in practice, if there exist true seasonal component,
* such as BirdStrike/Appdownloads, the energy is far larger than threshold, hence change threshold from 2.85 to 4.0 have no impact (tested);
*/
double randomnessValue = ProbabilityFunctions.Probit(randomnessThreshold);
double threshold = randomnessValue / Math.Sqrt(timeSeriesLength);
if (trueTimeDomainEnergy < threshold || values[truePeriod].Real < MinEnergyThreshold)
{
return(-1);
}
return(truePeriod);
}
private static void normal_solve_test()
//****************************************************************************80
//
// Purpose:
//
// NORMAL_SOLVE_TEST tests NORMAL_SOLVE.
//
// Licensing:
//
// This code is distributed under the GNU LGPL license.
//
// Modified:
//
// 20 April 2012
//
// Author:
//
// John Burkardt
//
{
int flag = 0;
int prob;
Console.WriteLine("");
Console.WriteLine("NORMAL_SOLVE_TEST");
Console.WriteLine(" NORMAL_SOLVE is a function with a simple interface which");
Console.WriteLine(" solves a linear system A*x = b in the least squares sense.");
Console.WriteLine(" Compare a tabulated solution X1 to the NORMAL_SOLVE result X2.");
Console.WriteLine("");
Console.WriteLine(" NORMAL_SOLVE cannot be applied when N < M,");
Console.WriteLine(" or if the matrix does not have full column rank.");
int prob_num = ProbabilityFunctions.p00_prob_num();
Console.WriteLine("");
Console.WriteLine(" Number of problems = " + prob_num + "");
Console.WriteLine("");
Console.WriteLine(" Index M N ||B|| ||X1 - X2|| ||X1|| ||X2|| ||R1|| ||R2||");
Console.WriteLine("");
for (prob = 1; prob <= prob_num; prob++)
{
//
// Get problem size.
//
int m = ProbabilityFunctions.p00_m(prob);
int n = ProbabilityFunctions.p00_n(prob);
//
// Retrieve problem data.
//
double[] a = ProbabilityFunctions.p00_a(prob, m, n);
double[] b = ProbabilityFunctions.p00_b(prob, m);
double[] x1 = ProbabilityFunctions.p00_x(prob, n);
double b_norm = typeMethods.r8vec_norm(m, b);
double x1_norm = typeMethods.r8vec_norm(n, x1);
double[] r1 = typeMethods.r8mat_mv_new(m, n, a, x1);
int i;
for (i = 0; i < m; i++)
{
r1[i] -= b[i];
}
double r1_norm = typeMethods.r8vec_norm(m, r1);
//
// Use NORMAL_SOLVE on the problem.
//
double[] x2 = NormalSolve.normal_solve(m, n, a, b, ref flag);
if (flag != 0)
{
Console.WriteLine(" " + prob.ToString(CultureInfo.InvariantCulture).PadLeft(5)
+ " " + m.ToString(CultureInfo.InvariantCulture).PadLeft(4)
+ " " + n.ToString(CultureInfo.InvariantCulture).PadLeft(4)
+ " " + b_norm.ToString(CultureInfo.InvariantCulture).PadLeft(12)
+ " " + "------------"
+ " " + x1_norm.ToString(CultureInfo.InvariantCulture).PadLeft(12)
+ " " + "------------"
+ " " + r1_norm.ToString(CultureInfo.InvariantCulture).PadLeft(12)
+ " " + "------------" + "");
}
else
{
double x2_norm = typeMethods.r8vec_norm(n, x2);
double[] r2 = typeMethods.r8mat_mv_new(m, n, a, x2);
for (i = 0; i < m; i++)
{
r2[i] -= b[i];
}
double r2_norm = typeMethods.r8vec_norm(m, r2);
//
// Compare tabulated and computed solutions.
//
double x_diff_norm = typeMethods.r8vec_norm_affine(n, x1, x2);
//
// Report results for this problem.
//
Console.WriteLine(" " + prob.ToString(CultureInfo.InvariantCulture).PadLeft(5)
+ " " + m.ToString(CultureInfo.InvariantCulture).PadLeft(4)
+ " " + n.ToString(CultureInfo.InvariantCulture).PadLeft(4)
+ " " + b_norm.ToString(CultureInfo.InvariantCulture).PadLeft(12)
+ " " + x_diff_norm.ToString(CultureInfo.InvariantCulture).PadLeft(12)
+ " " + x1_norm.ToString(CultureInfo.InvariantCulture).PadLeft(12)
+ " " + x2_norm.ToString(CultureInfo.InvariantCulture).PadLeft(12)
+ " " + r1_norm.ToString(CultureInfo.InvariantCulture).PadLeft(12)
+ " " + r2_norm.ToString(CultureInfo.InvariantCulture).PadLeft(12) + "");
}
}
}
private static void GetUnorderedCoefficientStatistics(LinearModelStatistics stats, ref VBuffer <Single> weights, ref VBuffer <ReadOnlyMemory <char> > names,
ref VBuffer <Single> estimate, ref VBuffer <Single> stdErr, ref VBuffer <Single> zScore, ref VBuffer <Single> pValue, out ValueGetter <VBuffer <ReadOnlyMemory <char> > > getSlotNames)
{
if (!stats._coeffStdError.HasValue)
{
getSlotNames = null;
return;
}
Contracts.Assert(stats._coeffStdError.Value.Length == weights.Length + 1);
var estimateValues = estimate.Values;
if (Utils.Size(estimateValues) < stats.ParametersCount - 1)
{
estimateValues = new Single[stats.ParametersCount - 1];
}
var stdErrorValues = stdErr.Values;
if (Utils.Size(stdErrorValues) < stats.ParametersCount - 1)
{
stdErrorValues = new Single[stats.ParametersCount - 1];
}
var zScoreValues = zScore.Values;
if (Utils.Size(zScoreValues) < stats.ParametersCount - 1)
{
zScoreValues = new Single[stats.ParametersCount - 1];
}
var pValueValues = pValue.Values;
if (Utils.Size(pValueValues) < stats.ParametersCount - 1)
{
pValueValues = new Single[stats.ParametersCount - 1];
}
const Double sqrt2 = 1.41421356237; // Math.Sqrt(2);
bool denseStdError = stats._coeffStdError.Value.IsDense;
int[] stdErrorIndices = stats._coeffStdError.Value.Indices;
for (int i = 1; i < stats.ParametersCount; i++)
{
int wi = denseStdError ? i - 1 : stdErrorIndices[i] - 1;
Contracts.Assert(0 <= wi && wi < weights.Length);
var weight = estimateValues[i - 1] = weights.GetItemOrDefault(wi);
var stdError = stdErrorValues[wi] = stats._coeffStdError.Value.Values[i];
zScoreValues[i - 1] = weight / stdError;
pValueValues[i - 1] = 1 - (Single)ProbabilityFunctions.Erf(Math.Abs(zScoreValues[i - 1] / sqrt2));
}
estimate = new VBuffer <Single>(stats.ParametersCount - 1, estimateValues, estimate.Indices);
stdErr = new VBuffer <Single>(stats.ParametersCount - 1, stdErrorValues, stdErr.Indices);
zScore = new VBuffer <Single>(stats.ParametersCount - 1, zScoreValues, zScore.Indices);
pValue = new VBuffer <Single>(stats.ParametersCount - 1, pValueValues, pValue.Indices);
var slotNames = names;
getSlotNames =
(ref VBuffer <ReadOnlyMemory <char> > dst) =>
{
var values = dst.Values;
if (Utils.Size(values) < stats.ParametersCount - 1)
{
values = new ReadOnlyMemory <char> [stats.ParametersCount - 1];
}
for (int i = 1; i < stats.ParametersCount; i++)
{
int wi = denseStdError ? i - 1 : stdErrorIndices[i] - 1;
values[i - 1] = slotNames.GetItemOrDefault(wi);
}
dst = new VBuffer <ReadOnlyMemory <char> >(stats.ParametersCount - 1, values, dst.Indices);
};
}
public static double StdNormalCdf(double x)
{
return(0.5 * (1 + ProbabilityFunctions.Erf(x * 1 / Math.Sqrt(2))));
}
public Emission(ProjectToRoad projection, double sigmaValue)
{
Probability = ProbabilityFunctions.Gaussian(projection.ProjectedDistance.DistanceInMeters, sigmaValue);
Distance = projection.ProjectedDistance;
}
