public void And(LPSTerm term, InequalityType inequality)
{
constraints_.Add(new LPSConstraint {
Inequality = inequality, Term = term
});
constraintCount_++;
}
/// <summary>
/// Ensures that the input is within an originBound ball of origin, or within 0.0f - 255f,
/// whichever is tightest.
/// </summary>
/// <returns></returns>
public static LPSConstraints OriginBoundFormula(LPSTerm[] input, double[] origin, double originBound)
{
Debug.Assert(input.Length == origin.Length);
LPSConstraints ct = new LPSConstraints();
for (int i = 0; i < input.Length; i++)
{
double ub = Math.Min(Utils.RobustnessOptions.MaxValue, origin[i] + originBound);
double lb = Math.Max(Utils.RobustnessOptions.MinValue, origin[i] - originBound);
if (lb <= ub)
{
var tmp = LPSTerm.Const(ub);
ct.And(input[i], InequalityType.LE, tmp);
tmp = LPSTerm.Const(lb);
ct.And(input[i], InequalityType.GE, tmp);
}
else
{
var tmp = LPSTerm.Const(origin[i] + originBound);
ct.And(input[i], InequalityType.LE, tmp);
tmp = LPSTerm.Const(origin[i] - originBound);
ct.And(input[i], InequalityType.GE, tmp);
}
}
return(ct);
}
public static LPSTerm Const(double d)
{
var v = new LPSTerm();
v.intercept_ = d;
return(v);
}
// (left `binop` right), equivalently: (left - right `binop` 0)
// NB: Allocates, does not overwrite the left or right term.
public void And(LPSTerm left, InequalityType inequality, LPSTerm right)
{
LPSTerm t = LPSTerm.Const(0.0);
t.Add(left);
t.AddMul(right, -1.0);
And(t, inequality);
}
// this += d*v
public void AddMul(LPSTerm v, double d)
{
v.coefficients_.Multiply(d, vcinfo_.tempmultstorage.Value);
coefficients_.Add(vcinfo_.tempmultstorage.Value, coefficients_);
intercept_ += d * v.intercept_;
addmulcounter++;
}
public static LPSTerm[] FromUnderlyingTransposeAlgebra(Matrix <double> outm, Vector <double> outv)
{
LPSTerm[] ret = new LPSTerm[outm.ColumnCount];
for (int i = 0; i < outm.ColumnCount; i++)
{
ret[i] = new LPSTerm(outm.Column(i), outv[i]);
}
return(ret);
}
private static LPSTerm FreshVariable()
{
var tmp = new LPSTerm();
tmp.coefficients_[vcinfo_.varcount_] = 1.0;
tmp.intercept_ = 0.0;
vcinfo_.varcount_++;
return(tmp);
}
private LPSTerm doInnerProduct(LPSState state, LPSTerm[] vs, Vector <double> ds)
{
LPSTerm result = LPSTerm.Const(0.0);
for (int i = 0; i < vs.Length; i++)
{
result.AddMul(vs[i], ds[i]);
}
return(result);
}
public static LPSTerm[] FreshVariables(int howmany)
{
var tmp = new LPSTerm[howmany];
for (int i = 0; i < howmany; i++)
{
tmp[i] = FreshVariable();
}
return(tmp);
}
public override LPSTerm[] EvaluateSymbolic(LPSState state, LPSTerm[] input)
{
LPSTerm[] output = new LPSTerm[OutputDimension];
for (int i = 0; i < OutputDimension; i++)
{
output[i] = doInnerProduct(state, input, weightMatrixRows_[i]);
output[i].Add(interceptVector_[i]);
}
return(output);
}
public static Matrix <double> UnderlyingTransposeMatrix(LPSTerm[] terms)
{
Matrix <double> res = DenseMatrix.Create(LPSTerm.TotalVarCount(), terms.Length, 0.0);
for (int i = 0; i < terms.Length; i++)
{
res.SetColumn(i, terms[i].GetCoefficients());
}
return(res);
}
public static LPSObjective MaxConf(LPSTerm[] output, int origLabel, int newLabel)
{
var tmp = LPSTerm.Const(0.0);
tmp.Add(output[newLabel]);
tmp.Sub(output[origLabel]);
return(new LPSObjective {
term = tmp, type = LPSObjectiveType.Max
});
}
public static void AddQuantizationSafety(LPSConstraints cts, LPSTerm[] input, double[] origin)
{
Random r = new Random();
int i = r.Next(0, origin.Length - 1);
LPSTerm curr = input[i];
// i.e: origin[i] - epsilon < input[i]
var tmp = LPSTerm.Const(origin[i] + 1.0);
cts.And(tmp, InequalityType.LE, curr);
}
public static Matrix <double> UnderlyingMatrix(LPSTerm[] terms)
{
// Stopwatch s = new Stopwatch();
// s.Start();
Matrix <double> res = SparseMatrix.Create(terms.Length, LPSTerm.TotalVarCount(), 0.0);
for (int i = 0; i < terms.Length; i++)
{
res.SetRow(i, terms[i].GetCoefficients());
}
// s.Stop();
// Console.WriteLine("To underlying matrix: {0} milliseconds",s.ElapsedMilliseconds);
return(res);
}
public static LPSTerm[] IdentityMatrix(int howmany)
{
//Matrix<double> coeffs = DenseMatrix.CreateIdentity(howmany);
//Vector<double> interc = DenseVector.Create(howmany, 0.0);
// return new LPSTerm[](coeffs, interc);
LPSTerm[] terms = new LPSTerm[howmany];
int pos = 0;
for (int i = 0; i < howmany; i++)
{
Vector <double> coeffs = SparseVector.Create(howmany, 0.0);
coeffs[pos++] = 1.0;
terms[i] = new LPSTerm(coeffs, 0.0);
}
return(terms);
}
public override LPSTerm[] EvaluateSymbolic(LPSState state, LPSTerm[] input)
{
LPSTerm[] cur = default(NumInstLPSTermArr).CreateVector(input.Length);
// If we have a meanImage ...
if (meanImage_ != null)
{
for (int i = 0; i < input.Length; i++)
{
cur[i].Sub(LPSTerm.Const(meanImage_[i])); // - mean
cur[i].Add(input[i]); // + input
cur[i].Mul(scale_);
}
return(cur);
}
// If we have a meanChannel ...
if (meanChannel_ != null && meanChannel_.Count() > 0)
{
for (int channel = 0; channel < InputCoordinates.ChannelCount; channel++)
{
for (int r = 0; r < InputCoordinates.RowCount; r++)
{
for (int c = 0; c < InputCoordinates.ColumnCount; c++)
{
int index = InputCoordinates.GetIndex(channel, r, c);
cur[index].Sub(LPSTerm.Const(meanChannel_[channel]));
cur[index].Add(input[index]);
cur[index].Mul(scale_);
}
}
}
return(cur);
}
// Finally, if we are only doing scaling ...
for (int i = 0; i < input.Length; i++)
{
cur[i].Add(input[i]);
cur[i].Mul(scale_);
}
return(cur);
}
/// <summary>
/// Create formulae of the form: <code> -epsilon < input[i] - origin[i] < epsilon </code>
/// </summary>
///
public static void AddEpsilonBounds(LPSConstraints cts, LPSTerm[] input, LPSTerm epsilon, double[] origin)
{
for (int i = 0; i < origin.Length; i++)
{
var curr = input[i];
// i.e: origin[i] - epsilon < input[i]
var tmp = LPSTerm.Const(origin[i]);
tmp.Sub(epsilon);
cts.And(tmp, InequalityType.LE, curr);
// and: input[i] < epsilon + origin[i]
tmp = LPSTerm.Const(origin[i]);
tmp.Add(epsilon);
cts.And(curr, InequalityType.LE, tmp);
}
cts.And(epsilon, InequalityType.GT, LPSTerm.Const(0.0)); // Quantization error!
cts.And(epsilon, InequalityType.LE, LPSTerm.Const(Utils.RobustnessOptions.Epsilon));
}
/// <summary>
/// LabelFormula(output,label,confidence) gives back a formula expressing
/// that: for all i s.t. i != label, output[label] - output[i] >= confidence
/// </summary>
/// <param name="output">Output of neural network (before softmax, as given by our evaluator).</param>
/// <param name="label">The label we wish to win.</param>
/// <param name="confidence">A confidence interval for all comparisons (e.g. for quantization etc).</param>
/// <returns>The constraint expressing that our label is indeed the winning one. </returns>
public static LPSConstraints LabelFormula(LPSTerm[] output, int label, double confidence = 0f)
{
LPSConstraints ct = new LPSConstraints();
for (int i = 0; i < output.Length; i++)
{
if (i != label)
{
// Need: output[label] - output[i] >= confidence
// i.e.: output[label] - output[i] - confidence >= 0
var tmp = LPSTerm.Const(0.0); // tmp := 0
tmp.Add(output[label]); // tmp := output[label]
tmp.AddMul(output[i], -1.0); // tmp := output[label] - output[i]
tmp.Add(-1.0 * confidence); // tmp := output[label] - output[i] - confidence
ct.And(tmp, InequalityType.GE);
}
}
return(ct);
}
public bool IsActivationWobbly(LPSTerm input, double[] image)
{
double icpt = input.Intercept;
Vector <double> imagecoeffs = input.GetCoefficients().SubVector(0, image.Length);
double innerprod = imagecoeffs * DenseVector.OfArray(image);
double shouldIncrease = (innerprod + icpt < 0) ? 1.0 : -1.0;
Vector <double> signVec = imagecoeffs.Map(x => (x >= 0) ? 1.0 : -1.0);
// Adversarial image:
Vector <double> adversarial_image = DenseVector.OfArray(image);
for (int i = 0; i < image.Length; i++)
{
adversarial_image[i] += shouldIncrease * signVec[i] * 0.5 * Utils.RobustnessOptions.Epsilon;
}
//Console.WriteLine("Original activation: {0}", innerprod + icpt);
//Console.WriteLine("Adversarial activation: {0}", imagecoeffs * adversarial_image + icpt);
//Console.Read();
return(Math.Sign(innerprod + icpt) != Math.Sign(imagecoeffs * adversarial_image + icpt));
}
public override LPSTerm ApplyKernelSymbolic(LPSState state, LPSTerm[] input, int outIndex, int channel, int row, int column)
{
int[] selections = state.Instrumentation[Index].Selections;
int maxIndex = selections[outIndex];
LPSTerm maxInput = input[maxIndex];
for (int i = 0; i < KernelDimension; i++)
{
for (int j = 0; j < KernelDimension; j++)
{
int x = row - Padding + i;
int y = column - Padding + j;
if (x >= InputCoordinates.RowCount || y >= InputCoordinates.ColumnCount)
{
continue;
}
int curIndex = InputCoordinates.GetIndex(channel, x, y);
if (curIndex == maxIndex)
{
continue;
}
if (curIndex < 0 || curIndex >= input.Length)
{
continue;
}
// maxInput - input[curIndex] >= 0
LPSTerm t = LPSTerm.Const(0.0);
t.Add(maxInput);
t.AddMul(input[curIndex], -1.0);
state.DeferredCts.And(t, InequalityType.GE);
}
}
return(maxInput);
}
public static Tuple <Matrix <double>, Vector <double> > Coalesce(List <Layer> toCoalesce)
{
Debug.Assert(toCoalesce.Count != 0);
int input_dim = toCoalesce.First().InputDimension;
int output_dim = toCoalesce.Last().OutputDimension;
VCInfo tmp = LPSTerm.GetVariableFactoryState();
LPSTerm.ResetVariableFactory(input_dim);
LPSTerm[] identity = LPSTerm.IdentityMatrix(input_dim);
LPSTerm[] v = identity;
for (int i = 0; i < toCoalesce.Count; i++)
{
Layer curr = toCoalesce[i];
var w = curr.EvaluateSymbolic(null, v);
v = w;
}
LPSTerm.RestoreVariableFactory(tmp);
return(new Tuple <Matrix <double>, Vector <double> >(LPSTerm.UnderlyingMatrix(v), LPSTerm.UnderlyingIntercept(v)));
}
public override LPSTerm[] EvaluateSymbolic(LPSState state, LPSTerm[] input)
{
DisjunctionChoice[] disjunctionChoices = state.Instrumentation[Index].DisjunctionConstraints;
Debug.Assert(InputDimension == disjunctionChoices.Length);
LPSTerm[] output = new LPSTerm[OutputDimension];
Random r = new Random(System.DateTime.Now.Millisecond);
// int uncertain = 0;
for (int i = 0; i < OutputDimension; i++)
{
switch (disjunctionChoices[i])
{
case DisjunctionChoice.ACTIVE:
output[i] = input[i];
// If we are supposed to do sampling
if (Utils.RobustnessOptions.LiveConstraintSamplingRatio != 1.0)
{
// Console.WriteLine("Sampling!");
// if we are above threshold defer
if (r.Next(0, 100) > (int)Utils.RobustnessOptions.LiveConstraintSamplingRatio * 100)
{
state.DeferredCts.And(input[i], InequalityType.GE);
}
else
{
state.CurrentCts.And(input[i], InequalityType.GE);
}
}
else
{
state.CurrentCts.And(input[i], InequalityType.GE);
}
break;
case DisjunctionChoice.INACTIVE:
output[i] = LPSTerm.Const(0.0);
// CEGAR version: defer 'dead' constraint
state.DeferredCts.And(input[i], InequalityType.LT);
// Original version
// state.CurrentCts.And(input[i],InequalityType.LT);
break;
default:
throw new Exception("Invalid disjunction choice type!");
}
/* This is more of an experiment really ...
* if (IsActivationWobbly(input[i], state.Origin))
* {
* uncertain++;
* // Mutate state to have new disjunction choices to explore in the future
* disjunctionChoices[i] = Instrumentation.FlipDisjunctionChoice(disjunctionChoices[i]);
* }
*/
}
// Console.WriteLine("** Ultra-sensitive ReLU activations {0}/{1}", uncertain, OutputDimension);
return(output);
}
public void Sub(LPSTerm v)
{
coefficients_ -= v.coefficients_;
intercept_ -= v.intercept_;
}
// this += v
public void Add(LPSTerm v)
{
coefficients_.Add(v.coefficients_, coefficients_);
intercept_ += v.intercept_;
}
public static LPSObjective MinLInf(LPSConstraints cts, LPSTerm[] input, LPSTerm epsilon, double[] origin)
{
return(new LPSObjective {
term = epsilon, type = LPSObjectiveType.Min
});
}